diff --git a/tools/nitpick/README.md b/tools/nitpick/README.md index c7b9050070..62e614c214 100644 --- a/tools/nitpick/README.md +++ b/tools/nitpick/README.md @@ -47,14 +47,20 @@ These steps assume the hifi repository has been cloned to `~/hifi`. ### Windows 1. (First time) download and install Python 3 from https://hifi-qa.s3.amazonaws.com/nitpick/Windows/python-3.7.0-amd64.exe (also located at https://www.python.org/downloads/) 1. Click the "add python to path" checkbox on the python installer - 1. After installation - add the path to python.exe to the Windows PATH environment variable. + 1. After installation: + 1. Open a new terminal + 1. Enter `python` and hit enter + 1. Verify that python is available (the prompt will change to `>>>`) + 1. Type `exit()` and hit enter to close python + 1. Install requests (a python library to download files from URLs) + `pip3 install requests` 1. (First time) download and install AWS CLI from https://hifi-qa.s3.amazonaws.com/nitpick/Windows/AWSCLI64PY3.msi (also available at https://aws.amazon.com/cli/ 1. Open a new command prompt and run `aws configure` 1. Enter the AWS account number 1. Enter the secret key 1. Leave region name and ouput format as default [None] - 1. Install the latest release of Boto3 via pip: + 1. Install the latest release of Boto3 via pip (from a terminal): `pip install boto3` 1. (First time) Download adb (Android Debug Bridge) from *https://dl.google.com/android/repository/platform-tools-latest-windows.zip* @@ -76,11 +82,13 @@ These steps assume the hifi repository has been cloned to `~/hifi`. `open "/Applications/Python 3.7/Install Certificates.command"`. This is needed because the Mac Python supplied no longer links with the deprecated Apple-supplied system OpenSSL libraries but rather supplies a private copy of OpenSSL 1.0.2 which does not automatically access the system default root certificates. 1. Verify that `/usr/local/bin/python3` exists. -1. (First time - AWS interface) Install pip with the script provided by the Python Packaging Authority: In a terminal: `curl -O https://bootstrap.pypa.io/get-pip.py` In a terminal: `python3 get-pip.py --user` + 1. Install requests (a python library to download files from URLs) + `pip3 install requests` +1. (First time - AWS interface) Install pip with the script provided by the Python Packaging Authority: 1. Use pip to install the AWS CLI. `pip3 install awscli --upgrade --user` This will install aws in your user. For user XXX, aws will be located in ~/Library/Python/3.7/bin diff --git a/tools/nitpick/src/AWSInterface.cpp b/tools/nitpick/src/AWSInterface.cpp index 4e83460b9e..7e11bd80fc 100644 --- a/tools/nitpick/src/AWSInterface.cpp +++ b/tools/nitpick/src/AWSInterface.cpp @@ -469,7 +469,7 @@ void AWSInterface::updateAWS() { if (!file.open(QIODevice::WriteOnly | QIODevice::Text)) { QMessageBox::critical(0, "Internal error: " + QString(__FILE__) + ":" + QString::number(__LINE__), - "Could not create 'addTestCases.py'"); + "Could not create 'updateAWS.py'"); exit(-1); } diff --git a/tools/nitpick/src/AdbInterface.h b/tools/nitpick/src/AdbInterface.h index c1ce84c019..a2aa2be8ea 100644 --- a/tools/nitpick/src/AdbInterface.h +++ b/tools/nitpick/src/AdbInterface.h @@ -17,12 +17,6 @@ public: QString getAdbCommand(); private: -#ifdef Q_OS_WIN - const QString _adbExe{ "adb.exe" }; -#else - // Both Mac and Linux use "python" - const QString _adbExe{ "adb" }; -#endif QString _adbCommand; }; diff --git a/tools/nitpick/src/Downloader.cpp b/tools/nitpick/src/Downloader.cpp index 3256e79601..26e2140dbd 100644 --- a/tools/nitpick/src/Downloader.cpp +++ b/tools/nitpick/src/Downloader.cpp @@ -8,32 +8,65 @@ // See the accompanying file LICENSE or http://www.apache.org/licenses/LICENSE-2.0.html // #include "Downloader.h" +#include "PythonInterface.h" -#include +#include +#include +#include +#include +#include -Downloader::Downloader(QUrl fileURL, QObject *parent) : QObject(parent) { - _networkAccessManager.get(QNetworkRequest(fileURL)); - - connect( - &_networkAccessManager, SIGNAL (finished(QNetworkReply*)), - this, SLOT (fileDownloaded(QNetworkReply*)) - ); +Downloader::Downloader() { + PythonInterface* pythonInterface = new PythonInterface(); + _pythonCommand = pythonInterface->getPythonCommand(); } -void Downloader::fileDownloaded(QNetworkReply* reply) { - QNetworkReply::NetworkError error = reply->error(); - if (error != QNetworkReply::NetworkError::NoError) { - QMessageBox::information(0, "Test Aborted", "Failed to download file: " + reply->errorString()); +void Downloader::downloadFiles(const QStringList& URLs, const QString& directoryName, const QStringList& filenames, void *caller) { + if (URLs.size() <= 0) { return; } - _downloadedData = reply->readAll(); + QString filename = directoryName + "/downloadFiles.py"; + if (QFile::exists(filename)) { + QFile::remove(filename); + } + QFile file(filename); - //emit a signal - reply->deleteLater(); - emit downloaded(); -} + if (!file.open(QIODevice::WriteOnly | QIODevice::Text)) { + QMessageBox::critical(0, "Internal error: " + QString(__FILE__) + ":" + QString::number(__LINE__), + "Could not create 'downloadFiles.py'"); + exit(-1); + } -QByteArray Downloader::downloadedData() const { - return _downloadedData; + QTextStream stream(&file); + + stream << "import requests\n"; + + for (int i = 0; i < URLs.size(); ++i) { + stream << "\nurl = '" + URLs[i] + "'\n"; + stream << "r = requests.get(url)\n"; + stream << "open('" + directoryName + '/' + filenames [i] + "', 'wb').write(r.content)\n"; + } + + file.close(); + +#ifdef Q_OS_WIN + QProcess* process = new QProcess(); + connect(process, &QProcess::started, this, [=]() { _busyWindow.exec(); }); + connect(process, SIGNAL(finished(int)), process, SLOT(deleteLater())); + connect(process, static_cast(&QProcess::finished), this, + [=](int exitCode, QProcess::ExitStatus exitStatus) { _busyWindow.hide(); }); + + QStringList parameters = QStringList() << filename; + process->start(_pythonCommand, parameters); +#elif defined Q_OS_MAC + QProcess* process = new QProcess(); + QStringList parameters = QStringList() << "-c" << _pythonCommand + " " + filename; + process->start("sh", parameters); + + // Wait for the last file to download + while (!QFile::exists(directoryName + '/' + filenames[filenames.length() - 1])) { + QThread::msleep(200); + } +#endif } diff --git a/tools/nitpick/src/Downloader.h b/tools/nitpick/src/Downloader.h index 742a88b890..e48c195999 100644 --- a/tools/nitpick/src/Downloader.h +++ b/tools/nitpick/src/Downloader.h @@ -11,38 +11,19 @@ #ifndef hifi_downloader_h #define hifi_downloader_h -#include -#include -#include -#include -#include -#include -#include -#include -#include +#include "BusyWindow.h" #include -#include -#include -#include -#include - class Downloader : public QObject { Q_OBJECT public: - explicit Downloader(QUrl fileURL, QObject *parent = 0); + Downloader(); - QByteArray downloadedData() const; - -signals: - void downloaded(); - -private slots: - void fileDownloaded(QNetworkReply* pReply); + void downloadFiles(const QStringList& URLs, const QString& directoryName, const QStringList& filenames, void *caller); private: - QNetworkAccessManager _networkAccessManager; - QByteArray _downloadedData; + QString _pythonCommand; + BusyWindow _busyWindow; }; #endif // hifi_downloader_h \ No newline at end of file diff --git a/tools/nitpick/src/Nitpick.cpp b/tools/nitpick/src/Nitpick.cpp index 39800c6bc6..03acd4a893 100644 --- a/tools/nitpick/src/Nitpick.cpp +++ b/tools/nitpick/src/Nitpick.cpp @@ -24,8 +24,6 @@ Nitpick::Nitpick(QWidget* parent) : QMainWindow(parent) { _ui.progressBar->setVisible(false); _ui.tabWidget->setCurrentIndex(0); - _signalMapper = new QSignalMapper(); - connect(_ui.actionClose, &QAction::triggered, this, &Nitpick::on_closePushbutton_clicked); connect(_ui.actionAbout, &QAction::triggered, this, &Nitpick::about); connect(_ui.actionContent, &QAction::triggered, this, &Nitpick::content); @@ -40,7 +38,7 @@ Nitpick::Nitpick(QWidget* parent) : QMainWindow(parent) { _ui.plainTextEdit->setReadOnly(true); - setWindowTitle("Nitpick - v3.0.0"); + setWindowTitle("Nitpick - v3.1.0"); clientProfiles << "VR-High" << "Desktop-High" << "Desktop-Low" << "Mobile-Touch" << "VR-Standalone"; _ui.clientProfileComboBox->insertItems(0, clientProfiles); @@ -48,10 +46,8 @@ Nitpick::Nitpick(QWidget* parent) : QMainWindow(parent) { } Nitpick::~Nitpick() { - delete _signalMapper; - - if (_test) { - delete _test; + if (_testCreator) { + delete _testCreator; } if (_testRunnerDesktop) { @@ -64,10 +60,10 @@ Nitpick::~Nitpick() { } void Nitpick::setup() { - if (_test) { - delete _test; + if (_testCreator) { + delete _testCreator; } - _test = new Test(_ui.progressBar, _ui.checkBoxInteractiveMode); + _testCreator = new TestCreator(_ui.progressBar, _ui.checkBoxInteractiveMode); std::vector dayCheckboxes; dayCheckboxes.emplace_back(_ui.mondayCheckBox); @@ -99,9 +95,12 @@ void Nitpick::setup() { timeEditCheckboxes, timeEdits, _ui.workingFolderRunOnDesktopLabel, - _ui.checkBoxServerless, + _ui.checkBoxServerless, + _ui.usePreviousInstallationOnDesktopCheckBox, _ui.runLatestOnDesktopCheckBox, _ui.urlOnDesktopLineEdit, + _ui.runFullSuiteOnDesktopCheckBox, + _ui.scriptURLOnDesktopLineEdit, _ui.runNowPushbutton, _ui.statusLabelOnDesktop ); @@ -118,8 +117,11 @@ void Nitpick::setup() { _ui.downloadAPKPushbutton, _ui.installAPKPushbutton, _ui.runInterfacePushbutton, + _ui.usePreviousInstallationOnMobileCheckBox, _ui.runLatestOnMobileCheckBox, _ui.urlOnMobileLineEdit, + _ui.runFullSuiteOnMobileCheckBox, + _ui.scriptURLOnMobileLineEdit, _ui.statusLabelOnMobile ); } @@ -130,7 +132,7 @@ void Nitpick::startTestsEvaluation(const bool isRunningFromCommandLine, const QString& branch, const QString& user ) { - _test->startTestsEvaluation(isRunningFromCommandLine, isRunningInAutomaticTestRun, snapshotDirectory, branch, user); + _testCreator->startTestsEvaluation(isRunningFromCommandLine, isRunningInAutomaticTestRun, snapshotDirectory, branch, user); } void Nitpick::on_tabWidget_currentChanged(int index) { @@ -149,47 +151,47 @@ void Nitpick::on_tabWidget_currentChanged(int index) { } void Nitpick::on_evaluateTestsPushbutton_clicked() { - _test->startTestsEvaluation(false, false); + _testCreator->startTestsEvaluation(false, false); } void Nitpick::on_createRecursiveScriptPushbutton_clicked() { - _test->createRecursiveScript(); + _testCreator->createRecursiveScript(); } void Nitpick::on_createAllRecursiveScriptsPushbutton_clicked() { - _test->createAllRecursiveScripts(); + _testCreator->createAllRecursiveScripts(); } void Nitpick::on_createTestsPushbutton_clicked() { - _test->createTests(_ui.clientProfileComboBox->currentText()); + _testCreator->createTests(_ui.clientProfileComboBox->currentText()); } void Nitpick::on_createMDFilePushbutton_clicked() { - _test->createMDFile(); + _testCreator->createMDFile(); } void Nitpick::on_createAllMDFilesPushbutton_clicked() { - _test->createAllMDFiles(); + _testCreator->createAllMDFiles(); } void Nitpick::on_createTestAutoScriptPushbutton_clicked() { - _test->createTestAutoScript(); + _testCreator->createTestAutoScript(); } void Nitpick::on_createAllTestAutoScriptsPushbutton_clicked() { - _test->createAllTestAutoScripts(); + _testCreator->createAllTestAutoScripts(); } void Nitpick::on_createTestsOutlinePushbutton_clicked() { - _test->createTestsOutline(); + _testCreator->createTestsOutline(); } void Nitpick::on_createTestRailTestCasesPushbutton_clicked() { - _test->createTestRailTestCases(); + _testCreator->createTestRailTestCases(); } void Nitpick::on_createTestRailRunButton_clicked() { - _test->createTestRailRun(); + _testCreator->createTestRailRun(); } void Nitpick::on_setWorkingFolderRunOnDesktopPushbutton_clicked() { @@ -206,16 +208,25 @@ void Nitpick::on_runNowPushbutton_clicked() { _testRunnerDesktop->run(); } +void Nitpick::on_usePreviousInstallationOnDesktopCheckBox_clicked() { + _ui.runLatestOnDesktopCheckBox->setEnabled(!_ui.usePreviousInstallationOnDesktopCheckBox->isChecked()); + _ui.urlOnDesktopLineEdit->setEnabled(!_ui.usePreviousInstallationOnDesktopCheckBox->isChecked() && !_ui.runLatestOnDesktopCheckBox->isChecked()); +} + void Nitpick::on_runLatestOnDesktopCheckBox_clicked() { _ui.urlOnDesktopLineEdit->setEnabled(!_ui.runLatestOnDesktopCheckBox->isChecked()); } +void Nitpick::on_runFullSuiteOnDesktopCheckBox_clicked() { + _ui.scriptURLOnDesktopLineEdit->setEnabled(!_ui.runFullSuiteOnDesktopCheckBox->isChecked()); +} + void Nitpick::automaticTestRunEvaluationComplete(QString zippedFolderName, int numberOfFailures) { _testRunnerDesktop->automaticTestRunEvaluationComplete(zippedFolderName, numberOfFailures); } void Nitpick::on_updateTestRailRunResultsPushbutton_clicked() { - _test->updateTestRailRunResult(); + _testCreator->updateTestRailRunResult(); } // To toggle between show and hide @@ -247,80 +258,15 @@ void Nitpick::on_closePushbutton_clicked() { } void Nitpick::on_createPythonScriptRadioButton_clicked() { - _test->setTestRailCreateMode(PYTHON); + _testCreator->setTestRailCreateMode(PYTHON); } void Nitpick::on_createXMLScriptRadioButton_clicked() { - _test->setTestRailCreateMode(XML); + _testCreator->setTestRailCreateMode(XML); } void Nitpick::on_createWebPagePushbutton_clicked() { - _test->createWebPage(_ui.updateAWSCheckBox, _ui.awsURLLineEdit); -} - -void Nitpick::downloadFile(const QUrl& url) { - _downloaders.emplace_back(new Downloader(url, this)); - connect(_downloaders[_index], SIGNAL(downloaded()), _signalMapper, SLOT(map())); - - _signalMapper->setMapping(_downloaders[_index], _index); - - ++_index; -} - -void Nitpick::downloadFiles(const QStringList& URLs, const QString& directoryName, const QStringList& filenames, void *caller) { - connect(_signalMapper, SIGNAL(mapped(int)), this, SLOT(saveFile(int))); - - _directoryName = directoryName; - _filenames = filenames; - _caller = caller; - - _numberOfFilesToDownload = URLs.size(); - _numberOfFilesDownloaded = 0; - _index = 0; - - _ui.progressBar->setMinimum(0); - _ui.progressBar->setMaximum(_numberOfFilesToDownload - 1); - _ui.progressBar->setValue(0); - _ui.progressBar->setVisible(true); - - foreach (auto downloader, _downloaders) { - delete downloader; - } - - _downloaders.clear(); - for (int i = 0; i < _numberOfFilesToDownload; ++i) { - downloadFile(URLs[i]); - } -} - -void Nitpick::saveFile(int index) { - try { - QFile file(_directoryName + "/" + _filenames[index]); - file.open(QIODevice::WriteOnly); - file.write(_downloaders[index]->downloadedData()); - file.close(); - } catch (...) { - QMessageBox::information(0, "Test Aborted", "Failed to save file: " + _filenames[index]); - _ui.progressBar->setVisible(false); - return; - } - - ++_numberOfFilesDownloaded; - - if (_numberOfFilesDownloaded == _numberOfFilesToDownload) { - disconnect(_signalMapper, SIGNAL(mapped(int)), this, SLOT(saveFile(int))); - if (_caller == _test) { - _test->finishTestsEvaluation(); - } else if (_caller == _testRunnerDesktop) { - _testRunnerDesktop->downloadComplete(); - } else if (_caller == _testRunnerMobile) { - _testRunnerMobile->downloadComplete(); - } - - _ui.progressBar->setVisible(false); - } else { - _ui.progressBar->setValue(_numberOfFilesDownloaded); - } + _testCreator->createWebPage(_ui.updateAWSCheckBox, _ui.awsURLLineEdit); } void Nitpick::about() { @@ -360,10 +306,19 @@ void Nitpick::on_connectDevicePushbutton_clicked() { _testRunnerMobile->connectDevice(); } +void Nitpick::on_usePreviousInstallationOnMobileCheckBox_clicked() { + _ui.runLatestOnMobileCheckBox->setEnabled(!_ui.usePreviousInstallationOnMobileCheckBox->isChecked()); + _ui.urlOnMobileLineEdit->setEnabled(!_ui.usePreviousInstallationOnMobileCheckBox->isChecked() && !_ui.runLatestOnMobileCheckBox->isChecked()); +} + void Nitpick::on_runLatestOnMobileCheckBox_clicked() { _ui.urlOnMobileLineEdit->setEnabled(!_ui.runLatestOnMobileCheckBox->isChecked()); } +void Nitpick::on_runFullSuiteOnMobileCheckBox_clicked() { + _ui.scriptURLOnMobileLineEdit->setEnabled(!_ui.runFullSuiteOnMobileCheckBox->isChecked()); +} + void Nitpick::on_downloadAPKPushbutton_clicked() { _testRunnerMobile->downloadAPK(); } diff --git a/tools/nitpick/src/Nitpick.h b/tools/nitpick/src/Nitpick.h index 80fef934d6..1e9d7bdee5 100644 --- a/tools/nitpick/src/Nitpick.h +++ b/tools/nitpick/src/Nitpick.h @@ -11,12 +11,10 @@ #define hifi_Nitpick_h #include -#include #include #include "ui_Nitpick.h" -#include "Downloader.h" -#include "Test.h" +#include "TestCreator.h" #include "TestRunnerDesktop.h" #include "TestRunnerMobile.h" @@ -38,9 +36,6 @@ public: void automaticTestRunEvaluationComplete(QString zippedFolderName, int numberOfFailures); - void downloadFile(const QUrl& url); - void downloadFiles(const QStringList& URLs, const QString& directoryName, const QStringList& filenames, void* caller); - void setUserText(const QString& user); QString getSelectedUser(); @@ -75,7 +70,9 @@ private slots: void on_setWorkingFolderRunOnDesktopPushbutton_clicked(); void on_runNowPushbutton_clicked(); + void on_usePreviousInstallationOnDesktopCheckBox_clicked(); void on_runLatestOnDesktopCheckBox_clicked(); + void on_runFullSuiteOnDesktopCheckBox_clicked(); void on_updateTestRailRunResultsPushbutton_clicked(); @@ -87,15 +84,16 @@ private slots: void on_createWebPagePushbutton_clicked(); - void saveFile(int index); - void about(); void content(); // Run on Mobile controls void on_setWorkingFolderRunOnMobilePushbutton_clicked(); void on_connectDevicePushbutton_clicked(); + + void on_usePreviousInstallationOnMobileCheckBox_clicked(); void on_runLatestOnMobileCheckBox_clicked(); + void on_runFullSuiteOnMobileCheckBox_clicked(); void on_downloadAPKPushbutton_clicked(); void on_installAPKPushbutton_clicked(); @@ -105,28 +103,13 @@ private slots: private: Ui::NitpickClass _ui; - Test* _test{ nullptr }; + TestCreator* _testCreator{ nullptr }; TestRunnerDesktop* _testRunnerDesktop{ nullptr }; TestRunnerMobile* _testRunnerMobile{ nullptr }; - std::vector _downloaders; - - // local storage for parameters - folder to store downloaded files in, and a list of their names - QString _directoryName; - QStringList _filenames; - - // Used to enable passing a parameter to slots - QSignalMapper* _signalMapper; - - int _numberOfFilesToDownload{ 0 }; - int _numberOfFilesDownloaded{ 0 }; - int _index{ 0 }; - bool _isRunningFromCommandline{ false }; - void* _caller; - QStringList clientProfiles; }; diff --git a/tools/nitpick/src/Test.cpp b/tools/nitpick/src/Test.cpp deleted file mode 100644 index e8e284bf32..0000000000 --- a/tools/nitpick/src/Test.cpp +++ /dev/null @@ -1,1116 +0,0 @@ -// -// Test.cpp -// -// Created by Nissim Hadar on 2 Nov 2017. -// Copyright 2013 High Fidelity, Inc. -// -// Distributed under the Apache License, Version 2.0. -// See the accompanying file LICENSE or http://www.apache.org/licenses/LICENSE-2.0.html -// -#include "Test.h" - -#include -#include -#include -#include -#include -#include - -#include -#include - -#include "Nitpick.h" -extern Nitpick* nitpick; - -#include - -Test::Test(QProgressBar* progressBar, QCheckBox* checkBoxInteractiveMode) : _awsInterface(NULL) { - _progressBar = progressBar; - _checkBoxInteractiveMode = checkBoxInteractiveMode; - - _mismatchWindow.setModal(true); - - if (nitpick) { - nitpick->setUserText(GIT_HUB_DEFAULT_USER); - nitpick->setBranchText(GIT_HUB_DEFAULT_BRANCH); - } -} - -bool Test::createTestResultsFolderPath(const QString& directory) { - QDateTime now = QDateTime::currentDateTime(); - _testResultsFolderPath = directory + "/" + TEST_RESULTS_FOLDER + "--" + now.toString(DATETIME_FORMAT) + "(local)[" + QHostInfo::localHostName() + "]"; - QDir testResultsFolder(_testResultsFolderPath); - - // Create a new test results folder - return QDir().mkdir(_testResultsFolderPath); -} - -QString Test::zipAndDeleteTestResultsFolder() { - QString zippedResultsFileName { _testResultsFolderPath + ".zip" }; - QFileInfo fileInfo(zippedResultsFileName); - if (fileInfo.exists()) { - QFile::remove(zippedResultsFileName); - } - - QDir testResultsFolder(_testResultsFolderPath); - JlCompress::compressDir(_testResultsFolderPath + ".zip", _testResultsFolderPath); - - testResultsFolder.removeRecursively(); - - //In all cases, for the next evaluation - _testResultsFolderPath = ""; - _failureIndex = 1; - _successIndex = 1; - - return zippedResultsFileName; -} - -int Test::compareImageLists() { - _progressBar->setMinimum(0); - _progressBar->setMaximum(_expectedImagesFullFilenames.length() - 1); - _progressBar->setValue(0); - _progressBar->setVisible(true); - - // Loop over both lists and compare each pair of images - // Quit loop if user has aborted due to a failed test. - bool keepOn{ true }; - int numberOfFailures{ 0 }; - for (int i = 0; keepOn && i < _expectedImagesFullFilenames.length(); ++i) { - // First check that images are the same size - QImage resultImage(_resultImagesFullFilenames[i]); - QImage expectedImage(_expectedImagesFullFilenames[i]); - - double similarityIndex; // in [-1.0 .. 1.0], where 1.0 means images are identical - - bool isInteractiveMode = (!_isRunningFromCommandLine && _checkBoxInteractiveMode->isChecked() && !_isRunningInAutomaticTestRun); - - // similarityIndex is set to -100.0 to indicate images are not the same size - if (isInteractiveMode && (resultImage.width() != expectedImage.width() || resultImage.height() != expectedImage.height())) { - QMessageBox::critical(0, "Internal error: " + QString(__FILE__) + ":" + QString::number(__LINE__), "Images are not the same size"); - similarityIndex = -100.0; - } else { - similarityIndex = _imageComparer.compareImages(resultImage, expectedImage); - } - - TestResult testResult = TestResult{ - (float)similarityIndex, - _expectedImagesFullFilenames[i].left(_expectedImagesFullFilenames[i].lastIndexOf("/") + 1), // path to the test (including trailing /) - QFileInfo(_expectedImagesFullFilenames[i].toStdString().c_str()).fileName(), // filename of expected image - QFileInfo(_resultImagesFullFilenames[i].toStdString().c_str()).fileName() // filename of result image - }; - - _mismatchWindow.setTestResult(testResult); - - if (similarityIndex < THRESHOLD) { - ++numberOfFailures; - - if (!isInteractiveMode) { - appendTestResultsToFile(testResult, _mismatchWindow.getComparisonImage(), true); - } else { - _mismatchWindow.exec(); - - switch (_mismatchWindow.getUserResponse()) { - case USER_RESPONSE_PASS: - break; - case USE_RESPONSE_FAIL: - appendTestResultsToFile(testResult, _mismatchWindow.getComparisonImage(), true); - break; - case USER_RESPONSE_ABORT: - keepOn = false; - break; - default: - assert(false); - break; - } - } - } else { - appendTestResultsToFile(testResult, _mismatchWindow.getComparisonImage(), false); - } - - _progressBar->setValue(i); - } - - _progressBar->setVisible(false); - return numberOfFailures; -} - -int Test::checkTextResults() { - // Create lists of failed and passed tests - QStringList nameFilterFailed; - nameFilterFailed << "*.failed.txt"; - QStringList testsFailed = QDir(_snapshotDirectory).entryList(nameFilterFailed, QDir::Files, QDir::Name); - - QStringList nameFilterPassed; - nameFilterPassed << "*.passed.txt"; - QStringList testsPassed = QDir(_snapshotDirectory).entryList(nameFilterPassed, QDir::Files, QDir::Name); - - // Add results to Test Results folder - foreach(QString currentFilename, testsFailed) { - appendTestResultsToFile(currentFilename, true); - } - - foreach(QString currentFilename, testsPassed) { - appendTestResultsToFile(currentFilename, false); - } - - return testsFailed.length(); -} - -void Test::appendTestResultsToFile(TestResult testResult, QPixmap comparisonImage, bool hasFailed) { - // Critical error if Test Results folder does not exist - if (!QDir().exists(_testResultsFolderPath)) { - QMessageBox::critical(0, "Internal error: " + QString(__FILE__) + ":" + QString::number(__LINE__), "Folder " + _testResultsFolderPath + " not found"); - exit(-1); - } - - // There are separate subfolders for failures and passes - QString resultFolderPath; - if (hasFailed) { - resultFolderPath = _testResultsFolderPath + "/Failure_" + QString::number(_failureIndex) + "--" + - testResult._actualImageFilename.left(testResult._actualImageFilename.length() - 4); - - ++_failureIndex; - } else { - resultFolderPath = _testResultsFolderPath + "/Success_" + QString::number(_successIndex) + "--" + - testResult._actualImageFilename.left(testResult._actualImageFilename.length() - 4); - - ++_successIndex; - } - - if (!QDir().mkdir(resultFolderPath)) { - QMessageBox::critical(0, "Internal error: " + QString(__FILE__) + ":" + QString::number(__LINE__), - "Failed to create folder " + resultFolderPath); - exit(-1); - } - - QFile descriptionFile(resultFolderPath + "/" + TEST_RESULTS_FILENAME); - if (!descriptionFile.open(QIODevice::ReadWrite)) { - QMessageBox::critical(0, "Internal error: " + QString(__FILE__) + ":" + QString::number(__LINE__), "Failed to create file " + TEST_RESULTS_FILENAME); - exit(-1); - } - - // Create text file describing the failure - QTextStream stream(&descriptionFile); - stream << "Test in folder " << testResult._pathname.left(testResult._pathname.length() - 1) << endl; // remove trailing '/' - stream << "Expected image was " << testResult._expectedImageFilename << endl; - stream << "Actual image was " << testResult._actualImageFilename << endl; - stream << "Similarity index was " << testResult._error << endl; - - descriptionFile.close(); - - // Copy expected and actual images, and save the difference image - QString sourceFile; - QString destinationFile; - - sourceFile = testResult._pathname + testResult._expectedImageFilename; - destinationFile = resultFolderPath + "/" + "Expected Image.png"; - if (!QFile::copy(sourceFile, destinationFile)) { - QMessageBox::critical(0, "Internal error: " + QString(__FILE__) + ":" + QString::number(__LINE__), "Failed to copy " + sourceFile + " to " + destinationFile); - exit(-1); - } - - sourceFile = testResult._pathname + testResult._actualImageFilename; - destinationFile = resultFolderPath + "/" + "Actual Image.png"; - if (!QFile::copy(sourceFile, destinationFile)) { - QMessageBox::critical(0, "Internal error: " + QString(__FILE__) + ":" + QString::number(__LINE__), "Failed to copy " + sourceFile + " to " + destinationFile); - exit(-1); - } - - comparisonImage.save(resultFolderPath + "/" + "Difference Image.png"); -} - -void::Test::appendTestResultsToFile(QString testResultFilename, bool hasFailed) { - // The test name includes everything until the penultimate period - QString testNameTemp = testResultFilename.left(testResultFilename.lastIndexOf('.')); - QString testName = testResultFilename.left(testNameTemp.lastIndexOf('.')); - QString resultFolderPath; - if (hasFailed) { - resultFolderPath = _testResultsFolderPath + "/Failure_" + QString::number(_failureIndex) + "--" + testName; - ++_failureIndex; - } else { - resultFolderPath = _testResultsFolderPath + "/Success_" + QString::number(_successIndex) + "--" + testName; - ++_successIndex; - } - - if (!QDir().mkdir(resultFolderPath)) { - QMessageBox::critical(0, "Internal error: " + QString(__FILE__) + ":" + QString::number(__LINE__), - "Failed to create folder " + resultFolderPath); - exit(-1); - } - - QString source = _snapshotDirectory + "/" + testResultFilename; - QString destination = resultFolderPath + "/Result.txt"; - if (!QFile::copy(source, destination)) { - QMessageBox::critical(0, "Internal error: " + QString(__FILE__) + ":" + QString::number(__LINE__), "Failed to copy " + testResultFilename + " to " + resultFolderPath); - exit(-1); - } -} - -void Test::startTestsEvaluation(const bool isRunningFromCommandLine, - const bool isRunningInAutomaticTestRun, - const QString& snapshotDirectory, - const QString& branchFromCommandLine, - const QString& userFromCommandLine -) { - _isRunningFromCommandLine = isRunningFromCommandLine; - _isRunningInAutomaticTestRun = isRunningInAutomaticTestRun; - - if (snapshotDirectory.isNull()) { - // Get list of PNG images in folder, sorted by name - QString previousSelection = _snapshotDirectory; - QString parent = previousSelection.left(previousSelection.lastIndexOf('/')); - if (!parent.isNull() && parent.right(1) != "/") { - parent += "/"; - } - _snapshotDirectory = QFileDialog::getExistingDirectory(nullptr, "Please select folder containing the snapshots", parent, - QFileDialog::ShowDirsOnly); - - // If user canceled then restore previous selection and return - if (_snapshotDirectory == "") { - _snapshotDirectory = previousSelection; - return; - } - } else { - _snapshotDirectory = snapshotDirectory; - _exitWhenComplete = (isRunningFromCommandLine && !isRunningInAutomaticTestRun); - } - - // Quit if test results folder could not be created - if (!createTestResultsFolderPath(_snapshotDirectory)) { - return; - } - - // Create two lists. The first is the test results, the second is the expected images - // The expected images are represented as a URL to enable download from GitHub - // Images that are in the wrong format are ignored. - - QStringList sortedTestResultsFilenames = createListOfAll_imagesInDirectory("png", _snapshotDirectory); - QStringList expectedImagesURLs; - - _resultImagesFullFilenames.clear(); - _expectedImagesFilenames.clear(); - _expectedImagesFullFilenames.clear(); - - QString branch = (branchFromCommandLine.isNull()) ? nitpick->getSelectedBranch() : branchFromCommandLine; - QString user = (userFromCommandLine.isNull()) ? nitpick->getSelectedUser() : userFromCommandLine; - - foreach(QString currentFilename, sortedTestResultsFilenames) { - QString fullCurrentFilename = _snapshotDirectory + "/" + currentFilename; - if (isInSnapshotFilenameFormat("png", currentFilename)) { - _resultImagesFullFilenames << fullCurrentFilename; - - QString expectedImagePartialSourceDirectory = getExpectedImagePartialSourceDirectory(currentFilename); - - // Images are stored on GitHub as ExpectedImage_ddddd.png - // Extract the digits at the end of the filename (excluding the file extension) - QString expectedImageFilenameTail = currentFilename.left(currentFilename.length() - 4).right(NUM_DIGITS); - QString expectedImageStoredFilename = EXPECTED_IMAGE_PREFIX + expectedImageFilenameTail + ".png"; - - QString imageURLString("https://raw.githubusercontent.com/" + user + "/" + GIT_HUB_REPOSITORY + "/" + branch + "/" + - expectedImagePartialSourceDirectory + "/" + expectedImageStoredFilename); - - expectedImagesURLs << imageURLString; - - // The image retrieved from GitHub needs a unique name - QString expectedImageFilename = currentFilename.replace("/", "_").replace(".png", "_EI.png"); - - _expectedImagesFilenames << expectedImageFilename; - _expectedImagesFullFilenames << _snapshotDirectory + "/" + expectedImageFilename; - } - } - - nitpick->downloadFiles(expectedImagesURLs, _snapshotDirectory, _expectedImagesFilenames, (void *)this); -} - -void Test::finishTestsEvaluation() { - // First - compare the pairs of images - int numberOfFailures = compareImageLists(); - - // Next - check text results - numberOfFailures += checkTextResults(); - - if (!_isRunningFromCommandLine && !_isRunningInAutomaticTestRun) { - if (numberOfFailures == 0) { - QMessageBox::information(0, "Success", "All images are as expected"); - } else { - QMessageBox::information(0, "Failure", "One or more images are not as expected"); - } - } - - QString zippedFolderName = zipAndDeleteTestResultsFolder(); - - if (_exitWhenComplete) { - exit(0); - } - - if (_isRunningInAutomaticTestRun) { - nitpick->automaticTestRunEvaluationComplete(zippedFolderName, numberOfFailures); - } -} - -bool Test::isAValidDirectory(const QString& pathname) { - // Only process directories - QDir dir(pathname); - if (!dir.exists()) { - return false; - } - - // Ignore '.', '..' directories - if (pathname[pathname.length() - 1] == '.') { - return false; - } - - return true; -} - -QString Test::extractPathFromTestsDown(const QString& fullPath) { - // `fullPath` includes the full path to the test. We need the portion below (and including) `tests` - QStringList pathParts = fullPath.split('/'); - int i{ 0 }; - while (i < pathParts.length() && pathParts[i] != "tests") { - ++i; - } - - if (i == pathParts.length()) { - QMessageBox::critical(0, "Internal error: " + QString(__FILE__) + ":" + QString::number(__LINE__), "Bad testPathname"); - exit(-1); - } - - QString partialPath; - for (int j = i; j < pathParts.length(); ++j) { - partialPath += "/" + pathParts[j]; - } - - return partialPath; -} - -void Test::includeTest(QTextStream& textStream, const QString& testPathname) { - QString partialPath = extractPathFromTestsDown(testPathname); - QString partialPathWithoutTests = partialPath.right(partialPath.length() - 7); - - textStream << "Script.include(testsRootPath + \"" << partialPathWithoutTests + "\");" << endl; -} - -void Test::createTests(const QString& clientProfile) { - // Rename files sequentially, as ExpectedResult_00000.png, ExpectedResult_00001.png and so on - // Any existing expected result images will be deleted - QString previousSelection = _snapshotDirectory; - QString parent = previousSelection.left(previousSelection.lastIndexOf('/')); - if (!parent.isNull() && parent.right(1) != "/") { - parent += "/"; - } - - _snapshotDirectory = QFileDialog::getExistingDirectory(nullptr, "Please select folder containing the snapshots", parent, - QFileDialog::ShowDirsOnly); - - // If user canceled then restore previous selection and return - if (_snapshotDirectory == "") { - _snapshotDirectory = previousSelection; - return; - } - - previousSelection = _testsRootDirectory; - parent = previousSelection.left(previousSelection.lastIndexOf('/')); - if (!parent.isNull() && parent.right(1) != "/") { - parent += "/"; - } - - _testsRootDirectory = QFileDialog::getExistingDirectory(nullptr, "Please select test root folder", parent, - QFileDialog::ShowDirsOnly); - - // If user canceled then restore previous selection and return - if (_testsRootDirectory == "") { - _testsRootDirectory = previousSelection; - return; - } - - QStringList sortedImageFilenames = createListOfAll_imagesInDirectory("png", _snapshotDirectory); - - int i = 1; - const int maxImages = pow(10, NUM_DIGITS); - foreach (QString currentFilename, sortedImageFilenames) { - QString fullCurrentFilename = _snapshotDirectory + "/" + currentFilename; - if (isInSnapshotFilenameFormat("png", currentFilename)) { - if (i >= maxImages) { - QMessageBox::critical(0, "Error", "More than " + QString::number(maxImages) + " images not supported"); - exit(-1); - } - - // Path to test is extracted from the file name - // Example: - // filename is tests.engine.interaction.pointer.laser.distanceScaleEnd.00000.jpg - // path is <_testDirectory>/engine/interaction/pointer/laser/distanceScaleEnd - // - // Note: we don't use the first part and the last 2 parts of the filename at this stage - // - QStringList pathParts = currentFilename.split("."); - QString fullNewFileName = _testsRootDirectory; - for (int j = 1; j < pathParts.size() - 2; ++j) { - fullNewFileName += "/" + pathParts[j]; - } - - // The image _index is the penultimate component of the path parts (the last being the file extension) - QString newFilename = "ExpectedImage_" + pathParts[pathParts.size() - 2].rightJustified(5, '0') + ".png"; - fullNewFileName += "/" + newFilename; - - try { - if (QFile::exists(fullNewFileName)) { - QFile::remove(fullNewFileName); - } - QFile::copy(fullCurrentFilename, fullNewFileName); - } catch (...) { - QMessageBox::critical(0, "Error", "Could not copy file: " + fullCurrentFilename + " to " + fullNewFileName + "\n"); - exit(-1); - } - ++i; - } - } - - QMessageBox::information(0, "Success", "Test images have been created"); -} - -ExtractedText Test::getTestScriptLines(QString testFileName) { - ExtractedText relevantTextFromTest; - - QFile inputFile(testFileName); - inputFile.open(QIODevice::ReadOnly); - if (!inputFile.isOpen()) { - QMessageBox::critical(0, - "Internal error: " + QString(__FILE__) + ":" + QString::number(__LINE__), - "Failed to open \"" + testFileName - ); - } - - QTextStream stream(&inputFile); - QString line = stream.readLine(); - - // Name of test is the string in the following line: - // nitpick.perform("Apply Material Entities to Avatars", Script.resolvePath("."), function(testType) {... - const QString ws("\\h*"); //white-space character - const QString functionPerformName(ws + "nitpick" + ws + "\\." + ws + "perform"); - const QString quotedString("\\\".+\\\""); - QString regexTestTitle(ws + functionPerformName + "\\(" + quotedString); - QRegularExpression lineContainingTitle = QRegularExpression(regexTestTitle); - - - // Each step is either of the following forms: - // nitpick.addStepSnapshot("Take snapshot"... - // nitpick.addStep("Clean up after test"... - const QString functionAddStepSnapshotName(ws + "nitpick" + ws + "\\." + ws + "addStepSnapshot"); - const QString regexStepSnapshot(ws + functionAddStepSnapshotName + ws + "\\(" + ws + quotedString + ".*"); - const QRegularExpression lineStepSnapshot = QRegularExpression(regexStepSnapshot); - - const QString functionAddStepName(ws + "nitpick" + ws + "\\." + ws + "addStep"); - const QString regexStep(ws + functionAddStepName + ws + "\\(" + ws + quotedString + ".*"); - const QRegularExpression lineStep = QRegularExpression(regexStep); - - while (!line.isNull()) { - line = stream.readLine(); - if (lineContainingTitle.match(line).hasMatch()) { - QStringList tokens = line.split('"'); - relevantTextFromTest.title = tokens[1]; - } else if (lineStepSnapshot.match(line).hasMatch()) { - QStringList tokens = line.split('"'); - QString nameOfStep = tokens[1]; - - Step *step = new Step(); - step->text = nameOfStep; - step->takeSnapshot = true; - relevantTextFromTest.stepList.emplace_back(step); - - } else if (lineStep.match(line).hasMatch()) { - QStringList tokens = line.split('"'); - QString nameOfStep = tokens[1]; - - Step *step = new Step(); - step->text = nameOfStep; - step->takeSnapshot = false; - relevantTextFromTest.stepList.emplace_back(step); - } - } - - inputFile.close(); - - return relevantTextFromTest; -} - -bool Test::createFileSetup() { - // Folder selection - QString previousSelection = _testDirectory; - QString parent = previousSelection.left(previousSelection.lastIndexOf('/')); - if (!parent.isNull() && parent.right(1) != "/") { - parent += "/"; - } - - _testDirectory = QFileDialog::getExistingDirectory(nullptr, "Please select folder containing the test", parent, - QFileDialog::ShowDirsOnly); - - // If user canceled then restore previous selection and return - if (_testDirectory == "") { - _testDirectory = previousSelection; - return false; - } - - return true; -} - -bool Test::createAllFilesSetup() { - // Select folder to start recursing from - QString previousSelection = _testsRootDirectory; - QString parent = previousSelection.left(previousSelection.lastIndexOf('/')); - if (!parent.isNull() && parent.right(1) != "/") { - parent += "/"; - } - - _testsRootDirectory = QFileDialog::getExistingDirectory(nullptr, "Please select the tests root folder", parent, - QFileDialog::ShowDirsOnly); - - // If user cancelled then restore previous selection and return - if (_testsRootDirectory == "") { - _testsRootDirectory = previousSelection; - return false; - } - - return true; -} - -// Create an MD file for a user-selected test. -// The folder selected must contain a script named "test.js", the file produced is named "test.md" -void Test::createMDFile() { - if (!createFileSetup()) { - return; - } - - if (createMDFile(_testDirectory)) { - QMessageBox::information(0, "Success", "MD file has been created"); - } -} - -void Test::createAllMDFiles() { - if (!createAllFilesSetup()) { - return; - } - - // First test if top-level folder has a test.js file - const QString testPathname{ _testsRootDirectory + "/" + TEST_FILENAME }; - QFileInfo fileInfo(testPathname); - if (fileInfo.exists()) { - createMDFile(_testsRootDirectory); - } - - QDirIterator it(_testsRootDirectory, QDirIterator::Subdirectories); - while (it.hasNext()) { - QString directory = it.next(); - - // Only process directories - QDir dir; - if (!isAValidDirectory(directory)) { - continue; - } - - const QString testPathname{ directory + "/" + TEST_FILENAME }; - QFileInfo fileInfo(testPathname); - if (fileInfo.exists()) { - createMDFile(directory); - } - } - - QMessageBox::information(0, "Success", "MD files have been created"); -} - -bool Test::createMDFile(const QString& directory) { - // Verify folder contains test.js file - QString testFileName(directory + "/" + TEST_FILENAME); - QFileInfo testFileInfo(testFileName); - if (!testFileInfo.exists()) { - QMessageBox::critical(0, "Error", "Could not find file: " + TEST_FILENAME); - return false; - } - - ExtractedText testScriptLines = getTestScriptLines(testFileName); - - QString mdFilename(directory + "/" + "test.md"); - QFile mdFile(mdFilename); - if (!mdFile.open(QIODevice::WriteOnly)) { - QMessageBox::critical(0, "Internal error: " + QString(__FILE__) + ":" + QString::number(__LINE__), "Failed to create file " + mdFilename); - exit(-1); - } - - QTextStream stream(&mdFile); - - //Test title - QString testName = testScriptLines.title; - stream << "# " << testName << "\n"; - - stream << "## Run this script URL: [Manual](./test.js?raw=true) [Auto](./testAuto.js?raw=true)(from menu/Edit/Open and Run scripts from URL...)." << "\n\n"; - - stream << "## Preconditions" << "\n"; - stream << "- In an empty region of a domain with editing rights." << "\n\n"; - - stream << "## Steps\n"; - stream << "Press '" + ADVANCE_KEY + "' key to advance step by step\n\n"; // note apostrophes surrounding 'ADVANCE_KEY' - - int snapShotIndex { 0 }; - for (size_t i = 0; i < testScriptLines.stepList.size(); ++i) { - stream << "### Step " << QString::number(i + 1) << "\n"; - stream << "- " << testScriptLines.stepList[i]->text << "\n"; - if ((i + 1 < testScriptLines.stepList.size()) && testScriptLines.stepList[i]->takeSnapshot) { - stream << "- ![](./ExpectedImage_" << QString::number(snapShotIndex).rightJustified(5, '0') << ".png)\n"; - ++snapShotIndex; - } - } - - mdFile.close(); - - foreach (auto test, testScriptLines.stepList) { - delete test; - } - testScriptLines.stepList.clear(); - - return true; -} - -void Test::createTestAutoScript() { - if (!createFileSetup()) { - return; - } - - if (createTestAutoScript(_testDirectory)) { - QMessageBox::information(0, "Success", "'testAuto.js` script has been created"); - } -} - -void Test::createAllTestAutoScripts() { - if (!createAllFilesSetup()) { - return; - } - - // First test if top-level folder has a test.js file - const QString testPathname{ _testsRootDirectory + "/" + TEST_FILENAME }; - QFileInfo fileInfo(testPathname); - if (fileInfo.exists()) { - createTestAutoScript(_testsRootDirectory); - } - - QDirIterator it(_testsRootDirectory, QDirIterator::Subdirectories); - while (it.hasNext()) { - QString directory = it.next(); - - // Only process directories - QDir dir; - if (!isAValidDirectory(directory)) { - continue; - } - - const QString testPathname{ directory + "/" + TEST_FILENAME }; - QFileInfo fileInfo(testPathname); - if (fileInfo.exists()) { - createTestAutoScript(directory); - } - } - - QMessageBox::information(0, "Success", "All 'testAuto.js' scripts have been created"); -} - -bool Test::createTestAutoScript(const QString& directory) { - // Verify folder contains test.js file - QString testFileName(directory + "/" + TEST_FILENAME); - QFileInfo testFileInfo(testFileName); - if (!testFileInfo.exists()) { - QMessageBox::critical(0, "Error", "Could not find file: " + TEST_FILENAME); - return false; - } - - QString testAutoScriptFilename(directory + "/" + "testAuto.js"); - QFile testAutoScriptFile(testAutoScriptFilename); - if (!testAutoScriptFile.open(QIODevice::WriteOnly)) { - QMessageBox::critical(0, "Internal error: " + QString(__FILE__) + ":" + QString::number(__LINE__), - "Failed to create file " + testAutoScriptFilename); - exit(-1); - } - - QTextStream stream(&testAutoScriptFile); - - stream << "if (typeof PATH_TO_THE_REPO_PATH_UTILS_FILE === 'undefined') PATH_TO_THE_REPO_PATH_UTILS_FILE = 'https://raw.githubusercontent.com/highfidelity/hifi_tests/master/tests/utils/branchUtils.js';\n"; - stream << "Script.include(PATH_TO_THE_REPO_PATH_UTILS_FILE);\n"; - stream << "var nitpick = createNitpick(Script.resolvePath('.'));\n\n"; - stream << "nitpick.enableAuto();\n\n"; - stream << "Script.include('./test.js?raw=true');\n"; - - testAutoScriptFile.close(); - return true; -} - -// Creates a single script in a user-selected folder. -// This script will run all text.js scripts in every applicable sub-folder -void Test::createRecursiveScript() { - if (!createFileSetup()) { - return; - } - - createRecursiveScript(_testDirectory, true); - QMessageBox::information(0, "Success", "'testRecursive.js` script has been created"); -} - -// This method creates a `testRecursive.js` script in every sub-folder. -void Test::createAllRecursiveScripts() { - if (!createAllFilesSetup()) { - return; - } - - createAllRecursiveScripts(_testsRootDirectory); - createRecursiveScript(_testsRootDirectory, false); - QMessageBox::information(0, "Success", "Scripts have been created"); -} - -void Test::createAllRecursiveScripts(const QString& directory) { - QDirIterator it(directory, QDirIterator::Subdirectories); - - while (it.hasNext()) { - QString nextDirectory = it.next(); - if (isAValidDirectory(nextDirectory)) { - createAllRecursiveScripts(nextDirectory); - createRecursiveScript(nextDirectory, false); - } - } -} - -void Test::createRecursiveScript(const QString& directory, bool interactiveMode) { - // If folder contains a test, then we are at a leaf - const QString testPathname{ directory + "/" + TEST_FILENAME }; - if (QFileInfo(testPathname).exists()) { - return; - } - - // Directories are included in reverse order. The nitpick scripts use a stack mechanism, - // so this ensures that the tests run in alphabetical order (a convenience when debugging) - QStringList directories; - QDirIterator it(directory); - while (it.hasNext()) { - QString subDirectory = it.next(); - - // Only process directories - if (!isAValidDirectory(subDirectory)) { - continue; - } - - const QString testPathname{ subDirectory + "/" + TEST_FILENAME }; - if (QFileInfo(testPathname).exists()) { - // Current folder contains a test script - directories.push_front(testPathname); - } - - const QString testRecursivePathname{ subDirectory + "/" + TEST_RECURSIVE_FILENAME }; - if (QFileInfo(testRecursivePathname).exists()) { - // Current folder contains a recursive script - directories.push_front(testRecursivePathname); - } - } - - // If 'directories' is empty, this means that this recursive script has no tests to call, so it is redundant - if (directories.length() == 0) { - return; - } - - // Open the recursive script file - const QString recursiveTestsFilename(directory + "/" + TEST_RECURSIVE_FILENAME); - QFile recursiveTestsFile(recursiveTestsFilename); - if (!recursiveTestsFile.open(QIODevice::WriteOnly | QIODevice::Text)) { - QMessageBox::critical(0, "Internal error: " + QString(__FILE__) + ":" + QString::number(__LINE__), - "Failed to create \"" + TEST_RECURSIVE_FILENAME + "\" in directory \"" + directory + "\""); - - exit(-1); - } - - QTextStream textStream(&recursiveTestsFile); - - textStream << "// This is an automatically generated file, created by nitpick" << endl; - - // Include 'nitpick.js' - QString branch = nitpick->getSelectedBranch(); - QString user = nitpick->getSelectedUser(); - - textStream << "PATH_TO_THE_REPO_PATH_UTILS_FILE = \"https://raw.githubusercontent.com/" + user + "/hifi_tests/" + branch + - "/tests/utils/branchUtils.js\";" - << endl; - textStream << "Script.include(PATH_TO_THE_REPO_PATH_UTILS_FILE);" << endl << endl; - - // The 'depth' variable is used to signal when to start running the recursive scripts - textStream << "if (typeof depth === 'undefined') {" << endl; - textStream << " depth = 0;" << endl; - textStream << " nitpick = createNitpick(Script.resolvePath(\".\"));" << endl; - textStream << " testsRootPath = nitpick.getTestsRootPath();" << endl << endl; - textStream << " nitpick.enableRecursive();" << endl; - textStream << " nitpick.enableAuto();" << endl; - textStream << "} else {" << endl; - textStream << " depth++" << endl; - textStream << "}" << endl << endl; - - // Now include the test scripts - for (int i = 0; i < directories.length(); ++i) { - includeTest(textStream, directories.at(i)); - } - - textStream << endl; - textStream << "if (depth > 0) {" << endl; - textStream << " depth--;" << endl; - textStream << "} else {" << endl; - textStream << " nitpick.runRecursive();" << endl; - textStream << "}" << endl << endl; - - recursiveTestsFile.close(); -} - -void Test::createTestsOutline() { - QString previousSelection = _testDirectory; - QString parent = previousSelection.left(previousSelection.lastIndexOf('/')); - if (!parent.isNull() && parent.right(1) != "/") { - parent += "/"; - } - - _testDirectory = - QFileDialog::getExistingDirectory(nullptr, "Please select the tests root folder", parent, QFileDialog::ShowDirsOnly); - - // If user canceled then restore previous selection and return - if (_testDirectory == "") { - _testDirectory = previousSelection; - return; - } - - const QString testsOutlineFilename { "testsOutline.md" }; - QString mdFilename(_testDirectory + "/" + testsOutlineFilename); - QFile mdFile(mdFilename); - if (!mdFile.open(QIODevice::WriteOnly)) { - QMessageBox::critical(0, "Internal error: " + QString(__FILE__) + ":" + QString::number(__LINE__), "Failed to create file " + mdFilename); - exit(-1); - } - - QTextStream stream(&mdFile); - - //Test title - stream << "# Outline of all tests\n"; - stream << "Directories with an appended (*) have an automatic test\n\n"; - - // We need to know our current depth, as this isn't given by QDirIterator - int rootDepth { _testDirectory.count('/') }; - - // Each test is shown as the folder name linking to the matching GitHub URL, and the path to the associated test.md file - QDirIterator it(_testDirectory, QDirIterator::Subdirectories); - while (it.hasNext()) { - QString directory = it.next(); - - // Only process directories - if (!isAValidDirectory(directory)) { - continue; - } - - // Ignore the utils directory - if (directory.right(5) == "utils") { - continue; - } - - // The prefix is the MarkDown prefix needed for correct indentation - // It consists of 2 spaces for each level of indentation, folled by a dash sign - int currentDepth = directory.count('/') - rootDepth; - QString prefix = QString(" ").repeated(2 * currentDepth - 1) + " - "; - - // The directory name appears after the last slash (we are assured there is at least 1). - QString directoryName = directory.right(directory.length() - directory.lastIndexOf("/") - 1); - - // nitpick is run on a clone of the repository. We use relative paths, so we can use both local disk and GitHub - // For a test in "D:/GitHub/hifi_tests/tests/content/entity/zone/ambientLightInheritance" the - // GitHub URL is "./content/entity/zone/ambientLightInheritance?raw=true" - QString partialPath = directory.right(directory.length() - (directory.lastIndexOf("/tests/") + QString("/tests").length() + 1)); - QString url = "./" + partialPath; - - stream << prefix << "[" << directoryName << "](" << url << "?raw=true" << ")"; - - // note that md files may be named 'test.md' or 'testStory.md' - QFileInfo fileInfo1(directory + "/test.md"); - if (fileInfo1.exists()) { - stream << " [(test description)](" << url << "/test.md)"; - } - - QFileInfo fileInfo2(directory + "/testStory.md"); - if (fileInfo2.exists()) { - stream << " [(test description)](" << url << "/testStory.md)"; - } - - QFileInfo fileInfo3(directory + "/" + TEST_FILENAME); - if (fileInfo3.exists()) { - stream << " (*)"; - } - stream << "\n"; - } - - mdFile.close(); - - QMessageBox::information(0, "Success", "Test outline file " + testsOutlineFilename + " has been created"); -} - -void Test::createTestRailTestCases() { - QString previousSelection = _testDirectory; - QString parent = previousSelection.left(previousSelection.lastIndexOf('/')); - if (!parent.isNull() && parent.right(1) != "/") { - parent += "/"; - } - - _testDirectory = - QFileDialog::getExistingDirectory(nullptr, "Please select the tests root folder", parent, QFileDialog::ShowDirsOnly); - - // If user cancelled then restore previous selection and return - if (_testDirectory.isNull()) { - _testDirectory = previousSelection; - return; - } - - QString outputDirectory = QFileDialog::getExistingDirectory(nullptr, "Please select a folder to store generated files in", - nullptr, QFileDialog::ShowDirsOnly); - - // If user cancelled then return - if (outputDirectory.isNull()) { - return; - } - - if (!_testRailInterface) { - _testRailInterface = new TestRailInterface; - } - - if (_testRailCreateMode == PYTHON) { - _testRailInterface->createTestSuitePython(_testDirectory, outputDirectory, nitpick->getSelectedUser(), - nitpick->getSelectedBranch()); - } else { - _testRailInterface->createTestSuiteXML(_testDirectory, outputDirectory, nitpick->getSelectedUser(), - nitpick->getSelectedBranch()); - } -} - -void Test::createTestRailRun() { - QString outputDirectory = QFileDialog::getExistingDirectory(nullptr, "Please select a folder to store generated files in", - nullptr, QFileDialog::ShowDirsOnly); - - if (outputDirectory.isNull()) { - return; - } - - - if (!_testRailInterface) { - _testRailInterface = new TestRailInterface; - } - - _testRailInterface->createTestRailRun(outputDirectory); -} - -void Test::updateTestRailRunResult() { - QString testResults = QFileDialog::getOpenFileName(nullptr, "Please select the zipped test results to update from", nullptr, - "Zipped Test Results (*.zip)"); - if (testResults.isNull()) { - return; - } - - QString tempDirectory = QFileDialog::getExistingDirectory(nullptr, "Please select a folder to store temporary files in", - nullptr, QFileDialog::ShowDirsOnly); - if (tempDirectory.isNull()) { - return; - } - - - if (!_testRailInterface) { - _testRailInterface = new TestRailInterface; - } - - _testRailInterface->updateTestRailRunResults(testResults, tempDirectory); -} - -QStringList Test::createListOfAll_imagesInDirectory(const QString& imageFormat, const QString& pathToImageDirectory) { - _imageDirectory = QDir(pathToImageDirectory); - QStringList nameFilters; - nameFilters << "*." + imageFormat; - - return _imageDirectory.entryList(nameFilters, QDir::Files, QDir::Name); -} - -// Snapshots are files in the following format: -// Filename (i.e. without extension) contains tests (this is based on all test scripts being within the tests folder) -// Last 5 characters in filename are digits (after removing the extension) -// Extension is 'imageFormat' -bool Test::isInSnapshotFilenameFormat(const QString& imageFormat, const QString& filename) { - bool contains_tests = filename.contains("tests" + PATH_SEPARATOR); - - QString filenameWithoutExtension = filename.left(filename.lastIndexOf('.')); - bool last5CharactersAreDigits; - filenameWithoutExtension.right(5).toInt(&last5CharactersAreDigits, 10); - - bool extensionIsIMAGE_FORMAT = (filename.right(imageFormat.length()) == imageFormat); - - return (contains_tests && last5CharactersAreDigits && extensionIsIMAGE_FORMAT); -} - -// For a file named "D_GitHub_hifi-tests_tests_content_entity_zone_create_0.jpg", the test directory is -// D:/GitHub/hifi-tests/tests/content/entity/zone/create -// This method assumes the filename is in the correct format -QString Test::getExpectedImageDestinationDirectory(const QString& filename) { - QString filenameWithoutExtension = filename.left(filename.length() - 4); - QStringList filenameParts = filenameWithoutExtension.split(PATH_SEPARATOR); - - QString result = filenameParts[0] + ":"; - - for (int i = 1; i < filenameParts.length() - 1; ++i) { - result += "/" + filenameParts[i]; - } - - return result; -} - -// For a file named "D_GitHub_hifi-tests_tests_content_entity_zone_create_0.jpg", the source directory on GitHub -// is ...tests/content/entity/zone/create -// This is used to create the full URL -// This method assumes the filename is in the correct format -QString Test::getExpectedImagePartialSourceDirectory(const QString& filename) { - QString filenameWithoutExtension = filename.left(filename.length() - 4); - QStringList filenameParts = filenameWithoutExtension.split(PATH_SEPARATOR); - - // Note that the bottom-most "tests" folder is assumed to be the root - // This is required because the tests folder is named hifi_tests - int i { filenameParts.length() - 1 }; - while (i >= 0 && filenameParts[i] != "tests") { - --i; - } - - if (i < 0) { - QMessageBox::critical(0, "Internal error: " + QString(__FILE__) + ":" + QString::number(__LINE__), "Bad filename"); - exit(-1); - } - - QString result = filenameParts[i]; - - for (int j = i + 1; j < filenameParts.length() - 1; ++j) { - result += "/" + filenameParts[j]; - } - - return result; -} - -void Test::setTestRailCreateMode(TestRailCreateMode testRailCreateMode) { - _testRailCreateMode = testRailCreateMode; -} - -void Test::createWebPage(QCheckBox* updateAWSCheckBox, QLineEdit* urlLineEdit) { - QString testResults = QFileDialog::getOpenFileName(nullptr, "Please select the zipped test results to update from", nullptr, - "Zipped Test Results (TestResults--*.zip)"); - if (testResults.isNull()) { - return; - } - - QString workingDirectory = QFileDialog::getExistingDirectory(nullptr, "Please select a folder to store temporary files in", - nullptr, QFileDialog::ShowDirsOnly); - if (workingDirectory.isNull()) { - return; - } - - if (!_awsInterface) { - _awsInterface = new AWSInterface; - } - - _awsInterface->createWebPageFromResults(testResults, workingDirectory, updateAWSCheckBox, urlLineEdit); -} \ No newline at end of file diff --git a/tools/nitpick/src/Test.h b/tools/nitpick/src/Test.h deleted file mode 100644 index 23011d0c31..0000000000 --- a/tools/nitpick/src/Test.h +++ /dev/null @@ -1,168 +0,0 @@ -// -// Test.h -// -// Created by Nissim Hadar on 2 Nov 2017. -// Copyright 2013 High Fidelity, Inc. -// -// Distributed under the Apache License, Version 2.0. -// See the accompanying file LICENSE or http://www.apache.org/licenses/LICENSE-2.0.html -// - -#ifndef hifi_test_h -#define hifi_test_h - -#include -#include -#include -#include - -#include "AWSInterface.h" -#include "ImageComparer.h" -#include "MismatchWindow.h" -#include "TestRailInterface.h" - -class Step { -public: - QString text; - bool takeSnapshot; -}; - -using StepList = std::vector; - -class ExtractedText { -public: - QString title; - StepList stepList; -}; - -enum TestRailCreateMode { - PYTHON, - XML -}; - -class Test { -public: - Test(QProgressBar* progressBar, QCheckBox* checkBoxInteractiveMode); - - void startTestsEvaluation(const bool isRunningFromCommandLine, - const bool isRunningInAutomaticTestRun, - const QString& snapshotDirectory = QString(), - const QString& branchFromCommandLine = QString(), - const QString& userFromCommandLine = QString()); - - void finishTestsEvaluation(); - - void createTests(const QString& clientProfile); - - void createTestsOutline(); - - bool createFileSetup(); - bool createAllFilesSetup(); - - void createMDFile(); - void createAllMDFiles(); - bool createMDFile(const QString& directory); - - void createTestAutoScript(); - void createAllTestAutoScripts(); - bool createTestAutoScript(const QString& directory); - - void createTestRailTestCases(); - void createTestRailRun(); - - void updateTestRailRunResult(); - - void createAllRecursiveScripts(); - void createAllRecursiveScripts(const QString& directory); - - void createRecursiveScript(); - void createRecursiveScript(const QString& directory, bool interactiveMode); - - int compareImageLists(); - int checkTextResults(); - - QStringList createListOfAll_imagesInDirectory(const QString& imageFormat, const QString& pathToImageDirectory); - - bool isInSnapshotFilenameFormat(const QString& imageFormat, const QString& filename); - - void includeTest(QTextStream& textStream, const QString& testPathname); - - void appendTestResultsToFile(TestResult testResult, QPixmap comparisonImage, bool hasFailed); - void appendTestResultsToFile(QString testResultFilename, bool hasFailed); - - bool createTestResultsFolderPath(const QString& directory); - QString zipAndDeleteTestResultsFolder(); - - static bool isAValidDirectory(const QString& pathname); - QString extractPathFromTestsDown(const QString& fullPath); - QString getExpectedImageDestinationDirectory(const QString& filename); - QString getExpectedImagePartialSourceDirectory(const QString& filename); - - ExtractedText getTestScriptLines(QString testFileName); - - void setTestRailCreateMode(TestRailCreateMode testRailCreateMode); - - void createWebPage(QCheckBox* updateAWSCheckBox, QLineEdit* urlLineEdit); - -private: - QProgressBar* _progressBar; - QCheckBox* _checkBoxInteractiveMode; - - bool _isRunningFromCommandLine{ false }; - bool _isRunningInAutomaticTestRun{ false }; - - const QString TEST_FILENAME{ "test.js" }; - const QString TEST_RECURSIVE_FILENAME{ "testRecursive.js" }; - const QString TEST_RESULTS_FOLDER { "TestResults" }; - const QString TEST_RESULTS_FILENAME { "TestResults.txt" }; - - const double THRESHOLD{ 0.935 }; - - QDir _imageDirectory; - - MismatchWindow _mismatchWindow; - - ImageComparer _imageComparer; - - QString _testResultsFolderPath; - int _failureIndex{ 1 }; - int _successIndex{ 1 }; - - // Expected images are in the format ExpectedImage_dddd.jpg (d == decimal digit) - const int NUM_DIGITS { 5 }; - const QString EXPECTED_IMAGE_PREFIX { "ExpectedImage_" }; - - // We have two directories to work with. - // The first is the directory containing the test we are working with - // The second is the root directory of all tests - // The third contains the snapshots taken for test runs that need to be evaluated - QString _testDirectory; - QString _testsRootDirectory; - QString _snapshotDirectory; - - QStringList _expectedImagesFilenames; - QStringList _expectedImagesFullFilenames; - QStringList _resultImagesFullFilenames; - - // Used for accessing GitHub - const QString GIT_HUB_DEFAULT_USER{ "highfidelity" }; - const QString GIT_HUB_DEFAULT_BRANCH{ "master" }; - const QString GIT_HUB_REPOSITORY{ "hifi_tests" }; - - const QString DATETIME_FORMAT{ "yyyy-MM-dd_hh-mm-ss" }; - - // NOTE: these need to match the appropriate var's in nitpick.js - // var advanceKey = "n"; - // var pathSeparator = "."; - const QString ADVANCE_KEY{ "n" }; - const QString PATH_SEPARATOR{ "." }; - - bool _exitWhenComplete{ false }; - - TestRailInterface* _testRailInterface; - TestRailCreateMode _testRailCreateMode { PYTHON }; - - AWSInterface* _awsInterface; -}; - -#endif // hifi_test_h \ No newline at end of file diff --git a/tools/nitpick/src/TestRailInterface.cpp b/tools/nitpick/src/TestRailInterface.cpp index 6ed13a72b6..8b8803153f 100644 --- a/tools/nitpick/src/TestRailInterface.cpp +++ b/tools/nitpick/src/TestRailInterface.cpp @@ -9,7 +9,7 @@ // #include "TestRailInterface.h" -#include "Test.h" +#include "TestCreator.h" #include #include @@ -258,7 +258,7 @@ bool TestRailInterface::requestTestRailResultsDataFromUser() { } bool TestRailInterface::isAValidTestDirectory(const QString& directory) { - if (Test::isAValidDirectory(directory)) { + if (TestCreator::isAValidDirectory(directory)) { // Ignore the utils and preformance directories if (directory.right(QString("utils").length()) == "utils" || directory.right(QString("performance").length()) == "performance") { diff --git a/tools/nitpick/src/TestRunner.cpp b/tools/nitpick/src/TestRunner.cpp index 54246de80b..c4e991e5ee 100644 --- a/tools/nitpick/src/TestRunner.cpp +++ b/tools/nitpick/src/TestRunner.cpp @@ -14,6 +14,26 @@ #include "Nitpick.h" extern Nitpick* nitpick; +TestRunner::TestRunner( + QLabel* workingFolderLabel, + QLabel* statusLabel, + QCheckBox* usePreviousInstallationCheckBox, + QCheckBox* runLatest, + QLineEdit* url, + QCheckBox* runFullSuite, + QLineEdit* scriptURL +) { + _workingFolderLabel = workingFolderLabel; + _statusLabel = statusLabel; + _usePreviousInstallationCheckBox = usePreviousInstallationCheckBox; + _runLatest = runLatest; + _url = url; + _runFullSuite = runFullSuite; + _scriptURL = scriptURL; + + _downloader = new Downloader(); +} + void TestRunner::setWorkingFolder(QLabel* workingFolderLabel) { // Everything will be written to this folder QString previousSelection = _workingFolder; @@ -49,7 +69,7 @@ void TestRunner::downloadBuildXml(void* caller) { urls << DEV_BUILD_XML_URL; filenames << DEV_BUILD_XML_FILENAME; - nitpick->downloadFiles(urls, _workingFolder, filenames, caller); + _downloader->downloadFiles(urls, _workingFolder, filenames, caller); } void TestRunner::parseBuildInformation() { diff --git a/tools/nitpick/src/TestRunner.h b/tools/nitpick/src/TestRunner.h index d2468ec2fa..6d36f246f7 100644 --- a/tools/nitpick/src/TestRunner.h +++ b/tools/nitpick/src/TestRunner.h @@ -11,6 +11,8 @@ #ifndef hifi_testRunner_h #define hifi_testRunner_h +#include "Downloader.h" + #include #include #include @@ -28,7 +30,18 @@ public: class TestRunner { public: + TestRunner( + QLabel* workingFolderLabel, + QLabel* statusLabel, + QCheckBox* usePreviousInstallationOnMobileCheckBox, + QCheckBox* runLatest, + QLineEdit* url, + QCheckBox* runFullSuite, + QLineEdit* scriptURL + ); + void setWorkingFolder(QLabel* workingFolderLabel); + void downloadBuildXml(void* caller); void parseBuildInformation(); QString getInstallerNameFromURL(const QString& url); @@ -36,10 +49,15 @@ public: void appendLog(const QString& message); protected: + Downloader* _downloader; + QLabel* _workingFolderLabel; QLabel* _statusLabel; - QLineEdit* _url; + QCheckBox* _usePreviousInstallationCheckBox; QCheckBox* _runLatest; + QLineEdit* _url; + QCheckBox* _runFullSuite; + QLineEdit* _scriptURL; QString _workingFolder; diff --git a/tools/nitpick/src/TestRunnerDesktop.cpp b/tools/nitpick/src/TestRunnerDesktop.cpp index e45d895886..b9caaa0ecb 100644 --- a/tools/nitpick/src/TestRunnerDesktop.cpp +++ b/tools/nitpick/src/TestRunnerDesktop.cpp @@ -27,23 +27,22 @@ TestRunnerDesktop::TestRunnerDesktop( std::vector timeEdits, QLabel* workingFolderLabel, QCheckBox* runServerless, + QCheckBox* usePreviousInstallationOnMobileCheckBox, QCheckBox* runLatest, QLineEdit* url, + QCheckBox* runFullSuite, + QLineEdit* scriptURL, QPushButton* runNow, QLabel* statusLabel, QObject* parent -) : QObject(parent) +) : QObject(parent), TestRunner(workingFolderLabel, statusLabel, usePreviousInstallationOnMobileCheckBox, runLatest, url, runFullSuite, scriptURL) { _dayCheckboxes = dayCheckboxes; _timeEditCheckboxes = timeEditCheckboxes; _timeEdits = timeEdits; - _workingFolderLabel = workingFolderLabel; _runServerless = runServerless; - _runLatest = runLatest; - _url = url; _runNow = runNow; - _statusLabel = statusLabel; _installerThread = new QThread(); _installerWorker = new InstallerWorker(); @@ -179,10 +178,14 @@ void TestRunnerDesktop::run() { // This will be restored at the end of the tests saveExistingHighFidelityAppDataFolder(); - _statusLabel->setText("Downloading Build XML"); - downloadBuildXml((void*)this); + if (_usePreviousInstallationCheckBox->isChecked()) { + installationComplete(); + } else { + _statusLabel->setText("Downloading Build XML"); + downloadBuildXml((void*)this); - // `downloadComplete` will run after download has completed + downloadComplete(); + } } void TestRunnerDesktop::downloadComplete() { @@ -209,9 +212,9 @@ void TestRunnerDesktop::downloadComplete() { _statusLabel->setText("Downloading installer"); - nitpick->downloadFiles(urls, _workingFolder, filenames, (void*)this); + _downloader->downloadFiles(urls, _workingFolder, filenames, (void*)this); - // `downloadComplete` will run again after download has completed + downloadComplete(); } else { // Download of Installer has completed @@ -292,15 +295,19 @@ void TestRunnerDesktop::installationComplete() { void TestRunnerDesktop::verifyInstallationSucceeded() { // Exit if the executables are missing. - // On Windows, the reason is probably that UAC has blocked the installation. This is treated as a critical error #ifdef Q_OS_WIN QFileInfo interfaceExe(QDir::toNativeSeparators(_installationFolder) + "\\interface.exe"); QFileInfo assignmentClientExe(QDir::toNativeSeparators(_installationFolder) + "\\assignment-client.exe"); QFileInfo domainServerExe(QDir::toNativeSeparators(_installationFolder) + "\\domain-server.exe"); if (!interfaceExe.exists() || !assignmentClientExe.exists() || !domainServerExe.exists()) { - QMessageBox::critical(0, "Installation of High Fidelity has failed", "Please verify that UAC has been disabled"); - exit(-1); + if (_runLatest->isChecked()) { + // On Windows, the reason is probably that UAC has blocked the installation. This is treated as a critical error + QMessageBox::critical(0, "Installation of High Fidelity has failed", "Please verify that UAC has been disabled"); + exit(-1); + } else { + QMessageBox::critical(0, "Installation of High Fidelity not found", "Please verify that working folder contains a proper installation"); + } } #endif } @@ -457,8 +464,9 @@ void TestRunnerDesktop::runInterfaceWithTestScript() { QString deleteScript = QString("https://raw.githubusercontent.com/") + _user + "/hifi_tests/" + _branch + "/tests/utils/deleteNearbyEntities.js"; - QString testScript = - QString("https://raw.githubusercontent.com/") + _user + "/hifi_tests/" + _branch + "/tests/testRecursive.js"; + QString testScript = (_runFullSuite->isChecked()) + ? QString("https://raw.githubusercontent.com/") + _user + "/hifi_tests/" + _branch + "/tests/testRecursive.js" + : _scriptURL->text(); QString commandLine; #ifdef Q_OS_WIN @@ -537,15 +545,16 @@ void TestRunnerDesktop::runInterfaceWithTestScript() { } void TestRunnerDesktop::interfaceExecutionComplete() { + QThread::msleep(500); QFileInfo testCompleted(QDir::toNativeSeparators(_snapshotFolder) +"/tests_completed.txt"); if (!testCompleted.exists()) { QMessageBox::critical(0, "Tests not completed", "Interface seems to have crashed before completion of the test scripts\nExisting images will be evaluated"); } + killProcesses(); + evaluateResults(); - killProcesses(); - // The High Fidelity AppData folder will be restored after evaluation has completed } @@ -591,7 +600,6 @@ void TestRunnerDesktop::addBuildNumberToResults(const QString& zippedFolderName) if (!QFile::rename(zippedFolderName, augmentedFilename)) { QMessageBox::critical(0, "Internal error: " + QString(__FILE__) + ":" + QString::number(__LINE__), "Could not rename '" + zippedFolderName + "' to '" + augmentedFilename); exit(-1); - } } @@ -667,6 +675,13 @@ void TestRunnerDesktop::checkTime() { QString TestRunnerDesktop::getPRNumberFromURL(const QString& url) { try { QStringList urlParts = url.split("/"); + if (urlParts.size() <= 2) { +#ifdef Q_OS_WIN + throw "URL not in expected format, should look like `https://deployment.highfidelity.com/jobs/pr-build/label%3Dwindows/13023/HighFidelity-Beta-Interface-PR14006-be76c43.exe`"; +#elif defined Q_OS_MAC + throw "URL not in expected format, should look like `https://deployment.highfidelity.com/jobs/pr-build/label%3Dwindows/13023/HighFidelity-Beta-Interface-PR14006-be76c43.dmg`"; +#endif + } QStringList filenameParts = urlParts[urlParts.size() - 1].split("-"); if (filenameParts.size() <= 3) { #ifdef Q_OS_WIN diff --git a/tools/nitpick/src/TestRunnerDesktop.h b/tools/nitpick/src/TestRunnerDesktop.h index 140a81f465..dce2dce2ba 100644 --- a/tools/nitpick/src/TestRunnerDesktop.h +++ b/tools/nitpick/src/TestRunnerDesktop.h @@ -12,7 +12,6 @@ #define hifi_testRunnerDesktop_h #include -#include #include #include #include @@ -32,8 +31,11 @@ public: std::vector timeEdits, QLabel* workingFolderLabel, QCheckBox* runServerless, + QCheckBox* usePreviousInstallationOnMobileCheckBox, QCheckBox* runLatest, QLineEdit* url, + QCheckBox* runFullSuite, + QLineEdit* scriptURL, QPushButton* runNow, QLabel* statusLabel, @@ -99,7 +101,6 @@ private: std::vector _dayCheckboxes; std::vector _timeEditCheckboxes; std::vector _timeEdits; - QLabel* _workingFolderLabel; QCheckBox* _runServerless; QPushButton* _runNow; QTimer* _timer; diff --git a/tools/nitpick/src/TestRunnerMobile.cpp b/tools/nitpick/src/TestRunnerMobile.cpp index ab276f3337..4d0d18ef3d 100644 --- a/tools/nitpick/src/TestRunnerMobile.cpp +++ b/tools/nitpick/src/TestRunnerMobile.cpp @@ -25,14 +25,16 @@ TestRunnerMobile::TestRunnerMobile( QPushButton* downloadAPKPushbutton, QPushButton* installAPKPushbutton, QPushButton* runInterfacePushbutton, + QCheckBox* usePreviousInstallationOnMobileCheckBox, QCheckBox* runLatest, QLineEdit* url, + QCheckBox* runFullSuite, + QLineEdit* scriptURL, QLabel* statusLabel, QObject* parent -) : QObject(parent), _adbInterface(NULL) +) : QObject(parent), TestRunner(workingFolderLabel, statusLabel, usePreviousInstallationOnMobileCheckBox, runLatest, url, runFullSuite, scriptURL) { - _workingFolderLabel = workingFolderLabel; _connectDeviceButton = connectDeviceButton; _pullFolderButton = pullFolderButton; _detectedDeviceLabel = detectedDeviceLabel; @@ -40,13 +42,15 @@ TestRunnerMobile::TestRunnerMobile( _downloadAPKPushbutton = downloadAPKPushbutton; _installAPKPushbutton = installAPKPushbutton; _runInterfacePushbutton = runInterfacePushbutton; - _runLatest = runLatest; - _url = url; - _statusLabel = statusLabel; folderLineEdit->setText("/sdcard/DCIM/TEST"); modelNames["SM_G955U1"] = "Samsung S8+ unlocked"; + modelNames["SM_N960U1"] = "Samsung Note 9 unlocked"; + modelNames["SM_T380"] = "Samsung Tab A"; + modelNames["Quest"] = "Quest"; + + _adbInterface = NULL; } TestRunnerMobile::~TestRunnerMobile() { @@ -66,6 +70,7 @@ void TestRunnerMobile::connectDevice() { QString devicesFullFilename{ _workingFolder + "/devices.txt" }; QString command = _adbInterface->getAdbCommand() + " devices -l > " + devicesFullFilename; + appendLog(command); system(command.toStdString().c_str()); if (!QFile::exists(devicesFullFilename)) { @@ -93,7 +98,7 @@ void TestRunnerMobile::connectDevice() { QString deviceID = tokens[0]; QString modelID = tokens[3].split(':')[1]; - QString modelName = "UKNOWN"; + QString modelName = "UNKNOWN"; if (modelNames.count(modelID) == 1) { modelName = modelNames[modelID]; } @@ -102,6 +107,8 @@ void TestRunnerMobile::connectDevice() { _pullFolderButton->setEnabled(true); _folderLineEdit->setEnabled(true); _downloadAPKPushbutton->setEnabled(true); + _installAPKPushbutton->setEnabled(true); + _runInterfacePushbutton->setEnabled(true); } } #endif @@ -109,6 +116,8 @@ void TestRunnerMobile::connectDevice() { void TestRunnerMobile::downloadAPK() { downloadBuildXml((void*)this); + + downloadComplete(); } @@ -141,11 +150,12 @@ void TestRunnerMobile::downloadComplete() { _statusLabel->setText("Downloading installer"); - nitpick->downloadFiles(urls, _workingFolder, filenames, (void*)this); + _downloader->downloadFiles(urls, _workingFolder, filenames, (void*)this); } else { _statusLabel->setText("Installer download complete"); - _installAPKPushbutton->setEnabled(true); } + + _installAPKPushbutton->setEnabled(true); } void TestRunnerMobile::installAPK() { @@ -154,11 +164,25 @@ void TestRunnerMobile::installAPK() { _adbInterface = new AdbInterface(); } + if (_installerFilename.isNull()) { + QString installerPathname = QFileDialog::getOpenFileName(nullptr, "Please select the APK", _workingFolder, + "Available APKs (*.apk)" + ); + + if (installerPathname.isNull()) { + return; + } + + // Remove the path + QStringList parts = installerPathname.split('/'); + _installerFilename = parts[parts.length() - 1]; + } + _statusLabel->setText("Installing"); QString command = _adbInterface->getAdbCommand() + " install -r -d " + _workingFolder + "/" + _installerFilename + " >" + _workingFolder + "/installOutput.txt"; + appendLog(command); system(command.toStdString().c_str()); _statusLabel->setText("Installation complete"); - _runInterfacePushbutton->setEnabled(true); #endif } @@ -169,7 +193,22 @@ void TestRunnerMobile::runInterface() { } _statusLabel->setText("Starting Interface"); - QString command = _adbInterface->getAdbCommand() + " shell monkey -p io.highfidelity.hifiinterface -v 1"; + + QString testScript = (_runFullSuite->isChecked()) + ? QString("https://raw.githubusercontent.com/") + nitpick->getSelectedUser() + "/hifi_tests/" + nitpick->getSelectedBranch() + "/tests/testRecursive.js" + : _scriptURL->text(); + + QString command = _adbInterface->getAdbCommand() + + " shell am start -n io.highfidelity.hifiinterface/.PermissionChecker" + + " --es args \\\"" + + " --url file:///~/serverless/tutorial.json" + + " --no-updater" + + " --no-login-suggestion" + + " --testScript " + testScript + " quitWhenFinished" + + " --testResultsLocation /sdcard/snapshots" + + "\\\""; + + appendLog(command); system(command.toStdString().c_str()); _statusLabel->setText("Interface started"); #endif @@ -182,7 +221,8 @@ void TestRunnerMobile::pullFolder() { } _statusLabel->setText("Pulling folder"); - QString command = _adbInterface->getAdbCommand() + " pull " + _folderLineEdit->text() + " " + _workingFolder + _installerFilename; + QString command = _adbInterface->getAdbCommand() + " pull " + _folderLineEdit->text() + " " + _workingFolder; + appendLog(command); system(command.toStdString().c_str()); _statusLabel->setText("Pull complete"); #endif diff --git a/tools/nitpick/src/TestRunnerMobile.h b/tools/nitpick/src/TestRunnerMobile.h index 52c2ba096d..f7b16da6f8 100644 --- a/tools/nitpick/src/TestRunnerMobile.h +++ b/tools/nitpick/src/TestRunnerMobile.h @@ -12,7 +12,6 @@ #define hifi_testRunnerMobile_h #include -#include #include #include @@ -31,8 +30,11 @@ public: QPushButton* downloadAPKPushbutton, QPushButton* installAPKPushbutton, QPushButton* runInterfacePushbutton, + QCheckBox* usePreviousInstallationOnMobileCheckBox, QCheckBox* runLatest, QLineEdit* url, + QCheckBox* runFullSuite, + QLineEdit* scriptURL, QLabel* statusLabel, QObject* parent = 0 diff --git a/tools/nitpick/ui/Nitpick.ui b/tools/nitpick/ui/Nitpick.ui index 47471522db..4a5a18f8d4 100644 --- a/tools/nitpick/ui/Nitpick.ui +++ b/tools/nitpick/ui/Nitpick.ui @@ -34,6 +34,9 @@ + + true + 45 @@ -495,7 +498,7 @@ - 20 + 240 70 120 20 @@ -549,13 +552,80 @@ - 170 + 175 100 - 451 + 445 21 + + + + 128 + 125 + 40 + 31 + + + + Script + + + + + false + + + + 175 + 130 + 445 + 21 + + + + + + + 20 + 130 + 120 + 20 + + + + <html><head/><body><p>If unchecked, will not show results during evaluation</p></body></html> + + + Run Full Suite + + + true + + + + + true + + + + 20 + 70 + 171 + 20 + + + + <html><head/><body><p>If unchecked, will not show results during evaluation</p></body></html> + + + usePreviousInstallation + + + false + + @@ -568,7 +638,7 @@ 10 - 90 + 150 160 30 @@ -581,7 +651,7 @@ 190 - 96 + 156 320 30 @@ -623,7 +693,7 @@ 460 - 410 + 440 160 30 @@ -639,7 +709,7 @@ 10 - 410 + 440 440 30 @@ -651,9 +721,9 @@ - 170 - 170 - 451 + 175 + 245 + 445 21 @@ -662,7 +732,7 @@ 20 - 170 + 245 120 20 @@ -684,7 +754,7 @@ 10 - 210 + 100 160 30 @@ -696,7 +766,7 @@ - 300 + 20 60 41 31 @@ -709,7 +779,7 @@ - 350 + 70 60 271 31 @@ -726,7 +796,7 @@ 10 - 250 + 325 160 30 @@ -742,7 +812,7 @@ 10 - 300 + 375 160 30 @@ -751,6 +821,86 @@ Run Interface + + + + 140 + 240 + 31 + 31 + + + + URL + + + + + false + + + + 175 + 275 + 445 + 21 + + + + + + + 140 + 270 + 40 + 31 + + + + Script + + + + + + 20 + 275 + 120 + 20 + + + + <html><head/><body><p>If unchecked, will not show results during evaluation</p></body></html> + + + Run Full Suite + + + true + + + + + true + + + + 20 + 210 + 171 + 20 + + + + <html><head/><body><p>If unchecked, will not show results during evaluation</p></body></html> + + + usePreviousInstallation + + + false + + @@ -921,6 +1071,9 @@ 21 + + true + groupBox