diff --git a/tools/nitpick/README.md b/tools/nitpick/README.md index c7b9050070..b4bcff81c8 100644 --- a/tools/nitpick/README.md +++ b/tools/nitpick/README.md @@ -47,24 +47,30 @@ These steps assume the hifi repository has been cloned to `~/hifi`. ### Windows 1. (First time) download and install Python 3 from https://hifi-qa.s3.amazonaws.com/nitpick/Windows/python-3.7.0-amd64.exe (also located at https://www.python.org/downloads/) 1. Click the "add python to path" checkbox on the python installer - 1. After installation - add the path to python.exe to the Windows PATH environment variable. + 1. After installation: + 1. Open a new terminal + 1. Enter `python` and hit enter + 1. Verify that python is available (the prompt will change to `>>>`) + 1. Type `exit()` and hit enter to close python + 1. Install requests (a python library to download files from URLs) + `pip3 install requests` 1. (First time) download and install AWS CLI from https://hifi-qa.s3.amazonaws.com/nitpick/Windows/AWSCLI64PY3.msi (also available at https://aws.amazon.com/cli/ 1. Open a new command prompt and run `aws configure` 1. Enter the AWS account number 1. Enter the secret key 1. Leave region name and ouput format as default [None] - 1. Install the latest release of Boto3 via pip: + 1. Install the latest release of Boto3 via pip (from a terminal): `pip install boto3` 1. (First time) Download adb (Android Debug Bridge) from *https://dl.google.com/android/repository/platform-tools-latest-windows.zip* 1. Copy the downloaded file to (for example) **C:\adb** and extract in place. Verify you see *adb.exe* in **C:\adb\platform-tools\\**. 1. After installation - add the path to adb.exe to the Windows PATH environment variable (note that it is in *adb\platform-tools*). -1. `nitpick` is included in the High Fidelity installer but can also be downloaded from: +1. `nitpick` is included in the High Fidelity installer but can also be downloaded from (change X.X.X to correct version): [here]().* ### Mac -1. (first time) Install brew +1. (First time) Install brew In a terminal: `/usr/bin/ruby -e "$(curl -fsSL https://raw.githubusercontent.com/Homebrew/install/master/install)"` Note that you will need to press RETURN again, and will then be asked for your password. @@ -76,11 +82,13 @@ These steps assume the hifi repository has been cloned to `~/hifi`. `open "/Applications/Python 3.7/Install Certificates.command"`. This is needed because the Mac Python supplied no longer links with the deprecated Apple-supplied system OpenSSL libraries but rather supplies a private copy of OpenSSL 1.0.2 which does not automatically access the system default root certificates. 1. Verify that `/usr/local/bin/python3` exists. -1. (First time - AWS interface) Install pip with the script provided by the Python Packaging Authority: In a terminal: `curl -O https://bootstrap.pypa.io/get-pip.py` In a terminal: `python3 get-pip.py --user` + 1. Install requests (a python library to download files from URLs) + `pip3 install requests` +1. (First time - AWS interface) Install pip with the script provided by the Python Packaging Authority: 1. Use pip to install the AWS CLI. `pip3 install awscli --upgrade --user` This will install aws in your user. For user XXX, aws will be located in ~/Library/Python/3.7/bin @@ -92,6 +100,16 @@ This is needed because the Mac Python supplied no longer links with the deprecat 1. Install the latest release of Boto3 via pip: pip3 install boto3 1. (First time)Install adb (the Android Debug Bridge) - in a terminal: `brew cask install android-platform-tools` +1. (First time) Set terminal privileges + 1. Click on Apple icon (top left) + 1. Select System Preferences... + 1. Select Security & Privacy + 1. Select Accessibility + 1. Click on "Click the lock to make changes" and enter passsword if requested + 1. Set Checkbox near *Terminal* to checked. + 1. Click on "Click the lock to prevent furthur changes" + 1. Close window + 1. `nitpick` is included in the High Fidelity installer but can also be downloaded from: [here]().* # Usage diff --git a/tools/nitpick/src/AWSInterface.cpp b/tools/nitpick/src/AWSInterface.cpp index 6dcc255286..04b7e7980e 100644 --- a/tools/nitpick/src/AWSInterface.cpp +++ b/tools/nitpick/src/AWSInterface.cpp @@ -480,7 +480,7 @@ void AWSInterface::updateAWS() { if (!file.open(QIODevice::WriteOnly | QIODevice::Text)) { QMessageBox::critical(0, "Internal error: " + QString(__FILE__) + ":" + QString::number(__LINE__), - "Could not create 'addTestCases.py'"); + "Could not create 'updateAWS.py'"); exit(-1); } diff --git a/tools/nitpick/src/AdbInterface.cpp b/tools/nitpick/src/AdbInterface.cpp index 82ef1446e3..41eb947efa 100644 --- a/tools/nitpick/src/AdbInterface.cpp +++ b/tools/nitpick/src/AdbInterface.cpp @@ -16,12 +16,13 @@ QString AdbInterface::getAdbCommand() { #ifdef Q_OS_WIN if (_adbCommand.isNull()) { - QString adbPath = PathUtils::getPathToExecutable("adb.exe"); + QString adbExe{ "adb.exe" }; + QString adbPath = PathUtils::getPathToExecutable(adbExe); if (!adbPath.isNull()) { - _adbCommand = adbPath + _adbExe; + _adbCommand = adbExe; } else { - QMessageBox::critical(0, "python.exe not found", - "Please verify that pyton.exe is in the PATH"); + QMessageBox::critical(0, "adb.exe not found", + "Please verify that adb.exe is in the PATH"); exit(-1); } } diff --git a/tools/nitpick/src/AdbInterface.h b/tools/nitpick/src/AdbInterface.h index c1ce84c019..a2aa2be8ea 100644 --- a/tools/nitpick/src/AdbInterface.h +++ b/tools/nitpick/src/AdbInterface.h @@ -17,12 +17,6 @@ public: QString getAdbCommand(); private: -#ifdef Q_OS_WIN - const QString _adbExe{ "adb.exe" }; -#else - // Both Mac and Linux use "python" - const QString _adbExe{ "adb" }; -#endif QString _adbCommand; }; diff --git a/tools/nitpick/src/Downloader.cpp b/tools/nitpick/src/Downloader.cpp index 3256e79601..26e2140dbd 100644 --- a/tools/nitpick/src/Downloader.cpp +++ b/tools/nitpick/src/Downloader.cpp @@ -8,32 +8,65 @@ // See the accompanying file LICENSE or http://www.apache.org/licenses/LICENSE-2.0.html // #include "Downloader.h" +#include "PythonInterface.h" -#include +#include +#include +#include +#include +#include -Downloader::Downloader(QUrl fileURL, QObject *parent) : QObject(parent) { - _networkAccessManager.get(QNetworkRequest(fileURL)); - - connect( - &_networkAccessManager, SIGNAL (finished(QNetworkReply*)), - this, SLOT (fileDownloaded(QNetworkReply*)) - ); +Downloader::Downloader() { + PythonInterface* pythonInterface = new PythonInterface(); + _pythonCommand = pythonInterface->getPythonCommand(); } -void Downloader::fileDownloaded(QNetworkReply* reply) { - QNetworkReply::NetworkError error = reply->error(); - if (error != QNetworkReply::NetworkError::NoError) { - QMessageBox::information(0, "Test Aborted", "Failed to download file: " + reply->errorString()); +void Downloader::downloadFiles(const QStringList& URLs, const QString& directoryName, const QStringList& filenames, void *caller) { + if (URLs.size() <= 0) { return; } - _downloadedData = reply->readAll(); + QString filename = directoryName + "/downloadFiles.py"; + if (QFile::exists(filename)) { + QFile::remove(filename); + } + QFile file(filename); - //emit a signal - reply->deleteLater(); - emit downloaded(); -} + if (!file.open(QIODevice::WriteOnly | QIODevice::Text)) { + QMessageBox::critical(0, "Internal error: " + QString(__FILE__) + ":" + QString::number(__LINE__), + "Could not create 'downloadFiles.py'"); + exit(-1); + } -QByteArray Downloader::downloadedData() const { - return _downloadedData; + QTextStream stream(&file); + + stream << "import requests\n"; + + for (int i = 0; i < URLs.size(); ++i) { + stream << "\nurl = '" + URLs[i] + "'\n"; + stream << "r = requests.get(url)\n"; + stream << "open('" + directoryName + '/' + filenames [i] + "', 'wb').write(r.content)\n"; + } + + file.close(); + +#ifdef Q_OS_WIN + QProcess* process = new QProcess(); + connect(process, &QProcess::started, this, [=]() { _busyWindow.exec(); }); + connect(process, SIGNAL(finished(int)), process, SLOT(deleteLater())); + connect(process, static_cast(&QProcess::finished), this, + [=](int exitCode, QProcess::ExitStatus exitStatus) { _busyWindow.hide(); }); + + QStringList parameters = QStringList() << filename; + process->start(_pythonCommand, parameters); +#elif defined Q_OS_MAC + QProcess* process = new QProcess(); + QStringList parameters = QStringList() << "-c" << _pythonCommand + " " + filename; + process->start("sh", parameters); + + // Wait for the last file to download + while (!QFile::exists(directoryName + '/' + filenames[filenames.length() - 1])) { + QThread::msleep(200); + } +#endif } diff --git a/tools/nitpick/src/Downloader.h b/tools/nitpick/src/Downloader.h index 742a88b890..e48c195999 100644 --- a/tools/nitpick/src/Downloader.h +++ b/tools/nitpick/src/Downloader.h @@ -11,38 +11,19 @@ #ifndef hifi_downloader_h #define hifi_downloader_h -#include -#include -#include -#include -#include -#include -#include -#include -#include +#include "BusyWindow.h" #include -#include -#include -#include -#include - class Downloader : public QObject { Q_OBJECT public: - explicit Downloader(QUrl fileURL, QObject *parent = 0); + Downloader(); - QByteArray downloadedData() const; - -signals: - void downloaded(); - -private slots: - void fileDownloaded(QNetworkReply* pReply); + void downloadFiles(const QStringList& URLs, const QString& directoryName, const QStringList& filenames, void *caller); private: - QNetworkAccessManager _networkAccessManager; - QByteArray _downloadedData; + QString _pythonCommand; + BusyWindow _busyWindow; }; #endif // hifi_downloader_h \ No newline at end of file diff --git a/tools/nitpick/src/Nitpick.cpp b/tools/nitpick/src/Nitpick.cpp index c07a76fc58..d8a1ff486a 100644 --- a/tools/nitpick/src/Nitpick.cpp +++ b/tools/nitpick/src/Nitpick.cpp @@ -24,8 +24,6 @@ Nitpick::Nitpick(QWidget* parent) : QMainWindow(parent) { _ui.progressBar->setVisible(false); _ui.tabWidget->setCurrentIndex(0); - _signalMapper = new QSignalMapper(); - connect(_ui.actionClose, &QAction::triggered, this, &Nitpick::on_closePushbutton_clicked); connect(_ui.actionAbout, &QAction::triggered, this, &Nitpick::about); connect(_ui.actionContent, &QAction::triggered, this, &Nitpick::content); @@ -40,7 +38,7 @@ Nitpick::Nitpick(QWidget* parent) : QMainWindow(parent) { _ui.plainTextEdit->setReadOnly(true); - setWindowTitle("Nitpick - v3.0.1"); + setWindowTitle("Nitpick - v3.1.1"); clientProfiles << "VR-High" << "Desktop-High" << "Desktop-Low" << "Mobile-Touch" << "VR-Standalone"; _ui.clientProfileComboBox->insertItems(0, clientProfiles); @@ -48,10 +46,8 @@ Nitpick::Nitpick(QWidget* parent) : QMainWindow(parent) { } Nitpick::~Nitpick() { - delete _signalMapper; - - if (_test) { - delete _test; + if (_testCreator) { + delete _testCreator; } if (_testRunnerDesktop) { @@ -64,10 +60,10 @@ Nitpick::~Nitpick() { } void Nitpick::setup() { - if (_test) { - delete _test; + if (_testCreator) { + delete _testCreator; } - _test = new Test(_ui.progressBar, _ui.checkBoxInteractiveMode); + _testCreator = new TestCreator(_ui.progressBar, _ui.checkBoxInteractiveMode); std::vector dayCheckboxes; dayCheckboxes.emplace_back(_ui.mondayCheckBox); @@ -99,9 +95,12 @@ void Nitpick::setup() { timeEditCheckboxes, timeEdits, _ui.workingFolderRunOnDesktopLabel, - _ui.checkBoxServerless, + _ui.checkBoxServerless, + _ui.usePreviousInstallationOnDesktopCheckBox, _ui.runLatestOnDesktopCheckBox, _ui.urlOnDesktopLineEdit, + _ui.runFullSuiteOnDesktopCheckBox, + _ui.scriptURLOnDesktopLineEdit, _ui.runNowPushbutton, _ui.statusLabelOnDesktop ); @@ -118,8 +117,11 @@ void Nitpick::setup() { _ui.downloadAPKPushbutton, _ui.installAPKPushbutton, _ui.runInterfacePushbutton, + _ui.usePreviousInstallationOnMobileCheckBox, _ui.runLatestOnMobileCheckBox, _ui.urlOnMobileLineEdit, + _ui.runFullSuiteOnMobileCheckBox, + _ui.scriptURLOnMobileLineEdit, _ui.statusLabelOnMobile ); } @@ -130,7 +132,7 @@ void Nitpick::startTestsEvaluation(const bool isRunningFromCommandLine, const QString& branch, const QString& user ) { - _test->startTestsEvaluation(isRunningFromCommandLine, isRunningInAutomaticTestRun, snapshotDirectory, branch, user); + _testCreator->startTestsEvaluation(isRunningFromCommandLine, isRunningInAutomaticTestRun, snapshotDirectory, branch, user); } void Nitpick::on_tabWidget_currentChanged(int index) { @@ -149,43 +151,43 @@ void Nitpick::on_tabWidget_currentChanged(int index) { } void Nitpick::on_createRecursiveScriptPushbutton_clicked() { - _test->createRecursiveScript(); + _testCreator->createRecursiveScript(); } void Nitpick::on_createAllRecursiveScriptsPushbutton_clicked() { - _test->createAllRecursiveScripts(); + _testCreator->createAllRecursiveScripts(); } void Nitpick::on_createTestsPushbutton_clicked() { - _test->createTests(_ui.clientProfileComboBox->currentText()); + _testCreator->createTests(_ui.clientProfileComboBox->currentText()); } void Nitpick::on_createMDFilePushbutton_clicked() { - _test->createMDFile(); + _testCreator->createMDFile(); } void Nitpick::on_createAllMDFilesPushbutton_clicked() { - _test->createAllMDFiles(); + _testCreator->createAllMDFiles(); } void Nitpick::on_createTestAutoScriptPushbutton_clicked() { - _test->createTestAutoScript(); + _testCreator->createTestAutoScript(); } void Nitpick::on_createAllTestAutoScriptsPushbutton_clicked() { - _test->createAllTestAutoScripts(); + _testCreator->createAllTestAutoScripts(); } void Nitpick::on_createTestsOutlinePushbutton_clicked() { - _test->createTestsOutline(); + _testCreator->createTestsOutline(); } void Nitpick::on_createTestRailTestCasesPushbutton_clicked() { - _test->createTestRailTestCases(); + _testCreator->createTestRailTestCases(); } void Nitpick::on_createTestRailRunButton_clicked() { - _test->createTestRailRun(); + _testCreator->createTestRailRun(); } void Nitpick::on_setWorkingFolderRunOnDesktopPushbutton_clicked() { @@ -202,16 +204,25 @@ void Nitpick::on_runNowPushbutton_clicked() { _testRunnerDesktop->run(); } +void Nitpick::on_usePreviousInstallationOnDesktopCheckBox_clicked() { + _ui.runLatestOnDesktopCheckBox->setEnabled(!_ui.usePreviousInstallationOnDesktopCheckBox->isChecked()); + _ui.urlOnDesktopLineEdit->setEnabled(!_ui.usePreviousInstallationOnDesktopCheckBox->isChecked() && !_ui.runLatestOnDesktopCheckBox->isChecked()); +} + void Nitpick::on_runLatestOnDesktopCheckBox_clicked() { _ui.urlOnDesktopLineEdit->setEnabled(!_ui.runLatestOnDesktopCheckBox->isChecked()); } +void Nitpick::on_runFullSuiteOnDesktopCheckBox_clicked() { + _ui.scriptURLOnDesktopLineEdit->setEnabled(!_ui.runFullSuiteOnDesktopCheckBox->isChecked()); +} + void Nitpick::automaticTestRunEvaluationComplete(QString zippedFolderName, int numberOfFailures) { _testRunnerDesktop->automaticTestRunEvaluationComplete(zippedFolderName, numberOfFailures); } void Nitpick::on_updateTestRailRunResultsPushbutton_clicked() { - _test->updateTestRailRunResult(); + _testCreator->updateTestRailRunResult(); } // To toggle between show and hide @@ -239,7 +250,7 @@ void Nitpick::on_showTaskbarPushbutton_clicked() { } void Nitpick::on_evaluateTestsPushbutton_clicked() { - _test->startTestsEvaluation(false, false); + _testCreator->startTestsEvaluation(false, false); } void Nitpick::on_closePushbutton_clicked() { @@ -247,80 +258,15 @@ void Nitpick::on_closePushbutton_clicked() { } void Nitpick::on_createPythonScriptRadioButton_clicked() { - _test->setTestRailCreateMode(PYTHON); + _testCreator->setTestRailCreateMode(PYTHON); } void Nitpick::on_createXMLScriptRadioButton_clicked() { - _test->setTestRailCreateMode(XML); + _testCreator->setTestRailCreateMode(XML); } void Nitpick::on_createWebPagePushbutton_clicked() { - _test->createWebPage(_ui.updateAWSCheckBox, _ui.diffImageRadioButton, _ui.ssimImageRadioButton, _ui.awsURLLineEdit); -} - -void Nitpick::downloadFile(const QUrl& url) { - _downloaders.emplace_back(new Downloader(url, this)); - connect(_downloaders[_index], SIGNAL(downloaded()), _signalMapper, SLOT(map())); - - _signalMapper->setMapping(_downloaders[_index], _index); - - ++_index; -} - -void Nitpick::downloadFiles(const QStringList& URLs, const QString& directoryName, const QStringList& filenames, void *caller) { - connect(_signalMapper, SIGNAL(mapped(int)), this, SLOT(saveFile(int))); - - _directoryName = directoryName; - _filenames = filenames; - _caller = caller; - - _numberOfFilesToDownload = URLs.size(); - _numberOfFilesDownloaded = 0; - _index = 0; - - _ui.progressBar->setMinimum(0); - _ui.progressBar->setMaximum(_numberOfFilesToDownload - 1); - _ui.progressBar->setValue(0); - _ui.progressBar->setVisible(true); - - foreach (auto downloader, _downloaders) { - delete downloader; - } - - _downloaders.clear(); - for (int i = 0; i < _numberOfFilesToDownload; ++i) { - downloadFile(URLs[i]); - } -} - -void Nitpick::saveFile(int index) { - try { - QFile file(_directoryName + "/" + _filenames[index]); - file.open(QIODevice::WriteOnly); - file.write(_downloaders[index]->downloadedData()); - file.close(); - } catch (...) { - QMessageBox::information(0, "Test Aborted", "Failed to save file: " + _filenames[index]); - _ui.progressBar->setVisible(false); - return; - } - - ++_numberOfFilesDownloaded; - - if (_numberOfFilesDownloaded == _numberOfFilesToDownload) { - disconnect(_signalMapper, SIGNAL(mapped(int)), this, SLOT(saveFile(int))); - if (_caller == _test) { - _test->finishTestsEvaluation(); - } else if (_caller == _testRunnerDesktop) { - _testRunnerDesktop->downloadComplete(); - } else if (_caller == _testRunnerMobile) { - _testRunnerMobile->downloadComplete(); - } - - _ui.progressBar->setVisible(false); - } else { - _ui.progressBar->setValue(_numberOfFilesDownloaded); - } + _testCreator->createWebPage(_ui.updateAWSCheckBox, _ui.diffImageRadioButton, _ui.ssimImageRadioButton, _ui.awsURLLineEdit); } void Nitpick::about() { @@ -360,10 +306,19 @@ void Nitpick::on_connectDevicePushbutton_clicked() { _testRunnerMobile->connectDevice(); } +void Nitpick::on_usePreviousInstallationOnMobileCheckBox_clicked() { + _ui.runLatestOnMobileCheckBox->setEnabled(!_ui.usePreviousInstallationOnMobileCheckBox->isChecked()); + _ui.urlOnMobileLineEdit->setEnabled(!_ui.usePreviousInstallationOnMobileCheckBox->isChecked() && !_ui.runLatestOnMobileCheckBox->isChecked()); +} + void Nitpick::on_runLatestOnMobileCheckBox_clicked() { _ui.urlOnMobileLineEdit->setEnabled(!_ui.runLatestOnMobileCheckBox->isChecked()); } +void Nitpick::on_runFullSuiteOnMobileCheckBox_clicked() { + _ui.scriptURLOnMobileLineEdit->setEnabled(!_ui.runFullSuiteOnMobileCheckBox->isChecked()); +} + void Nitpick::on_downloadAPKPushbutton_clicked() { _testRunnerMobile->downloadAPK(); } diff --git a/tools/nitpick/src/Nitpick.h b/tools/nitpick/src/Nitpick.h index 3095a14c05..42f55ee8b2 100644 --- a/tools/nitpick/src/Nitpick.h +++ b/tools/nitpick/src/Nitpick.h @@ -11,12 +11,10 @@ #define hifi_Nitpick_h #include -#include #include #include "ui_Nitpick.h" -#include "Downloader.h" -#include "Test.h" +#include "TestCreator.h" #include "TestRunnerDesktop.h" #include "TestRunnerMobile.h" @@ -38,9 +36,6 @@ public: void automaticTestRunEvaluationComplete(QString zippedFolderName, int numberOfFailures); - void downloadFile(const QUrl& url); - void downloadFiles(const QStringList& URLs, const QString& directoryName, const QStringList& filenames, void* caller); - void setUserText(const QString& user); QString getSelectedUser(); @@ -74,7 +69,9 @@ private slots: void on_setWorkingFolderRunOnDesktopPushbutton_clicked(); void on_runNowPushbutton_clicked(); + void on_usePreviousInstallationOnDesktopCheckBox_clicked(); void on_runLatestOnDesktopCheckBox_clicked(); + void on_runFullSuiteOnDesktopCheckBox_clicked(); void on_updateTestRailRunResultsPushbutton_clicked(); @@ -88,15 +85,16 @@ private slots: void on_createWebPagePushbutton_clicked(); - void saveFile(int index); - void about(); void content(); // Run on Mobile controls void on_setWorkingFolderRunOnMobilePushbutton_clicked(); void on_connectDevicePushbutton_clicked(); + + void on_usePreviousInstallationOnMobileCheckBox_clicked(); void on_runLatestOnMobileCheckBox_clicked(); + void on_runFullSuiteOnMobileCheckBox_clicked(); void on_downloadAPKPushbutton_clicked(); void on_installAPKPushbutton_clicked(); @@ -106,28 +104,13 @@ private slots: private: Ui::NitpickClass _ui; - Test* _test{ nullptr }; + TestCreator* _testCreator{ nullptr }; TestRunnerDesktop* _testRunnerDesktop{ nullptr }; TestRunnerMobile* _testRunnerMobile{ nullptr }; - std::vector _downloaders; - - // local storage for parameters - folder to store downloaded files in, and a list of their names - QString _directoryName; - QStringList _filenames; - - // Used to enable passing a parameter to slots - QSignalMapper* _signalMapper; - - int _numberOfFilesToDownload{ 0 }; - int _numberOfFilesDownloaded{ 0 }; - int _index{ 0 }; - bool _isRunningFromCommandline{ false }; - void* _caller; - QStringList clientProfiles; }; diff --git a/tools/nitpick/src/Test.cpp b/tools/nitpick/src/TestCreator.cpp similarity index 92% rename from tools/nitpick/src/Test.cpp rename to tools/nitpick/src/TestCreator.cpp index 7269fb3f02..c548a63a83 100644 --- a/tools/nitpick/src/Test.cpp +++ b/tools/nitpick/src/TestCreator.cpp @@ -1,5 +1,5 @@ // -// Test.cpp +// TestCreator.cpp // // Created by Nissim Hadar on 2 Nov 2017. // Copyright 2013 High Fidelity, Inc. @@ -7,7 +7,7 @@ // Distributed under the Apache License, Version 2.0. // See the accompanying file LICENSE or http://www.apache.org/licenses/LICENSE-2.0.html // -#include "Test.h" +#include "TestCreator.h" #include #include @@ -24,7 +24,9 @@ extern Nitpick* nitpick; #include -Test::Test(QProgressBar* progressBar, QCheckBox* checkBoxInteractiveMode) : _awsInterface(NULL) { +TestCreator::TestCreator(QProgressBar* progressBar, QCheckBox* checkBoxInteractiveMode) : _awsInterface(NULL) { + _downloader = new Downloader(); + _progressBar = progressBar; _checkBoxInteractiveMode = checkBoxInteractiveMode; @@ -36,7 +38,7 @@ Test::Test(QProgressBar* progressBar, QCheckBox* checkBoxInteractiveMode) : _aws } } -bool Test::createTestResultsFolderPath(const QString& directory) { +bool TestCreator::createTestResultsFolderPath(const QString& directory) { QDateTime now = QDateTime::currentDateTime(); _testResultsFolderPath = directory + "/" + TEST_RESULTS_FOLDER + "--" + now.toString(DATETIME_FORMAT) + "(local)[" + QHostInfo::localHostName() + "]"; QDir testResultsFolder(_testResultsFolderPath); @@ -45,7 +47,7 @@ bool Test::createTestResultsFolderPath(const QString& directory) { return QDir().mkdir(_testResultsFolderPath); } -QString Test::zipAndDeleteTestResultsFolder() { +QString TestCreator::zipAndDeleteTestResultsFolder() { QString zippedResultsFileName { _testResultsFolderPath + ".zip" }; QFileInfo fileInfo(zippedResultsFileName); if (fileInfo.exists()) { @@ -65,7 +67,7 @@ QString Test::zipAndDeleteTestResultsFolder() { return zippedResultsFileName; } -int Test::compareImageLists() { +int TestCreator::compareImageLists() { _progressBar->setMinimum(0); _progressBar->setMaximum(_expectedImagesFullFilenames.length() - 1); _progressBar->setValue(0); @@ -136,7 +138,7 @@ int Test::compareImageLists() { return numberOfFailures; } -int Test::checkTextResults() { +int TestCreator::checkTextResults() { // Create lists of failed and passed tests QStringList nameFilterFailed; nameFilterFailed << "*.failed.txt"; @@ -146,7 +148,7 @@ int Test::checkTextResults() { nameFilterPassed << "*.passed.txt"; QStringList testsPassed = QDir(_snapshotDirectory).entryList(nameFilterPassed, QDir::Files, QDir::Name); - // Add results to Test Results folder + // Add results to TestCreator Results folder foreach(QString currentFilename, testsFailed) { appendTestResultsToFile(currentFilename, true); } @@ -158,7 +160,7 @@ int Test::checkTextResults() { return testsFailed.length(); } -void Test::appendTestResultsToFile(const TestResult& testResult, const QPixmap& comparisonImage, const QPixmap& ssimResultsImage, bool hasFailed) { +void TestCreator::appendTestResultsToFile(const TestResult& testResult, const QPixmap& comparisonImage, const QPixmap& ssimResultsImage, bool hasFailed) { // Critical error if Test Results folder does not exist if (!QDir().exists(_testResultsFolderPath)) { QMessageBox::critical(0, "Internal error: " + QString(__FILE__) + ":" + QString::number(__LINE__), "Folder " + _testResultsFolderPath + " not found"); @@ -193,7 +195,7 @@ void Test::appendTestResultsToFile(const TestResult& testResult, const QPixmap& // Create text file describing the failure QTextStream stream(&descriptionFile); - stream << "Test in folder " << testResult._pathname.left(testResult._pathname.length() - 1) << endl; // remove trailing '/' + stream << "TestCreator in folder " << testResult._pathname.left(testResult._pathname.length() - 1) << endl; // remove trailing '/' stream << "Expected image was " << testResult._expectedImageFilename << endl; stream << "Actual image was " << testResult._actualImageFilename << endl; stream << "Similarity index was " << testResult._error << endl; @@ -224,7 +226,7 @@ void Test::appendTestResultsToFile(const TestResult& testResult, const QPixmap& ssimResultsImage.save(resultFolderPath + "/" + "SSIM Image.png"); } -void::Test::appendTestResultsToFile(QString testResultFilename, bool hasFailed) { +void::TestCreator::appendTestResultsToFile(QString testResultFilename, bool hasFailed) { // The test name includes everything until the penultimate period QString testNameTemp = testResultFilename.left(testResultFilename.lastIndexOf('.')); QString testName = testResultFilename.left(testNameTemp.lastIndexOf('.')); @@ -251,7 +253,7 @@ void::Test::appendTestResultsToFile(QString testResultFilename, bool hasFailed) } } -void Test::startTestsEvaluation(const bool isRunningFromCommandLine, +void TestCreator::startTestsEvaluation(const bool isRunningFromCommandLine, const bool isRunningInAutomaticTestRun, const QString& snapshotDirectory, const QString& branchFromCommandLine, @@ -324,10 +326,11 @@ void Test::startTestsEvaluation(const bool isRunningFromCommandLine, } } - nitpick->downloadFiles(expectedImagesURLs, _snapshotDirectory, _expectedImagesFilenames, (void *)this); + _downloader->downloadFiles(expectedImagesURLs, _snapshotDirectory, _expectedImagesFilenames, (void *)this); + finishTestsEvaluation(); } -void Test::finishTestsEvaluation() { +void TestCreator::finishTestsEvaluation() { // First - compare the pairs of images int numberOfFailures = compareImageLists(); @@ -353,7 +356,7 @@ void Test::finishTestsEvaluation() { } } -bool Test::isAValidDirectory(const QString& pathname) { +bool TestCreator::isAValidDirectory(const QString& pathname) { // Only process directories QDir dir(pathname); if (!dir.exists()) { @@ -368,7 +371,7 @@ bool Test::isAValidDirectory(const QString& pathname) { return true; } -QString Test::extractPathFromTestsDown(const QString& fullPath) { +QString TestCreator::extractPathFromTestsDown(const QString& fullPath) { // `fullPath` includes the full path to the test. We need the portion below (and including) `tests` QStringList pathParts = fullPath.split('/'); int i{ 0 }; @@ -389,14 +392,14 @@ QString Test::extractPathFromTestsDown(const QString& fullPath) { return partialPath; } -void Test::includeTest(QTextStream& textStream, const QString& testPathname) { +void TestCreator::includeTest(QTextStream& textStream, const QString& testPathname) { QString partialPath = extractPathFromTestsDown(testPathname); QString partialPathWithoutTests = partialPath.right(partialPath.length() - 7); textStream << "Script.include(testsRootPath + \"" << partialPathWithoutTests + "\");" << endl; } -void Test::createTests(const QString& clientProfile) { +void TestCreator::createTests(const QString& clientProfile) { // Rename files sequentially, as ExpectedResult_00000.png, ExpectedResult_00001.png and so on // Any existing expected result images will be deleted QString previousSelection = _snapshotDirectory; @@ -474,7 +477,7 @@ void Test::createTests(const QString& clientProfile) { QMessageBox::information(0, "Success", "Test images have been created"); } -ExtractedText Test::getTestScriptLines(QString testFileName) { +ExtractedText TestCreator::getTestScriptLines(QString testFileName) { ExtractedText relevantTextFromTest; QFile inputFile(testFileName); @@ -539,7 +542,7 @@ ExtractedText Test::getTestScriptLines(QString testFileName) { return relevantTextFromTest; } -bool Test::createFileSetup() { +bool TestCreator::createFileSetup() { // Folder selection QString previousSelection = _testDirectory; QString parent = previousSelection.left(previousSelection.lastIndexOf('/')); @@ -559,7 +562,7 @@ bool Test::createFileSetup() { return true; } -bool Test::createAllFilesSetup() { +bool TestCreator::createAllFilesSetup() { // Select folder to start recursing from QString previousSelection = _testsRootDirectory; QString parent = previousSelection.left(previousSelection.lastIndexOf('/')); @@ -581,7 +584,7 @@ bool Test::createAllFilesSetup() { // Create an MD file for a user-selected test. // The folder selected must contain a script named "test.js", the file produced is named "test.md" -void Test::createMDFile() { +void TestCreator::createMDFile() { if (!createFileSetup()) { return; } @@ -591,7 +594,7 @@ void Test::createMDFile() { } } -void Test::createAllMDFiles() { +void TestCreator::createAllMDFiles() { if (!createAllFilesSetup()) { return; } @@ -623,7 +626,7 @@ void Test::createAllMDFiles() { QMessageBox::information(0, "Success", "MD files have been created"); } -bool Test::createMDFile(const QString& directory) { +bool TestCreator::createMDFile(const QString& directory) { // Verify folder contains test.js file QString testFileName(directory + "/" + TEST_FILENAME); QFileInfo testFileInfo(testFileName); @@ -643,7 +646,7 @@ bool Test::createMDFile(const QString& directory) { QTextStream stream(&mdFile); - //Test title + //TestCreator title QString testName = testScriptLines.title; stream << "# " << testName << "\n"; @@ -675,7 +678,7 @@ bool Test::createMDFile(const QString& directory) { return true; } -void Test::createTestAutoScript() { +void TestCreator::createTestAutoScript() { if (!createFileSetup()) { return; } @@ -685,7 +688,7 @@ void Test::createTestAutoScript() { } } -void Test::createAllTestAutoScripts() { +void TestCreator::createAllTestAutoScripts() { if (!createAllFilesSetup()) { return; } @@ -717,7 +720,7 @@ void Test::createAllTestAutoScripts() { QMessageBox::information(0, "Success", "All 'testAuto.js' scripts have been created"); } -bool Test::createTestAutoScript(const QString& directory) { +bool TestCreator::createTestAutoScript(const QString& directory) { // Verify folder contains test.js file QString testFileName(directory + "/" + TEST_FILENAME); QFileInfo testFileInfo(testFileName); @@ -748,7 +751,7 @@ bool Test::createTestAutoScript(const QString& directory) { // Creates a single script in a user-selected folder. // This script will run all text.js scripts in every applicable sub-folder -void Test::createRecursiveScript() { +void TestCreator::createRecursiveScript() { if (!createFileSetup()) { return; } @@ -758,7 +761,7 @@ void Test::createRecursiveScript() { } // This method creates a `testRecursive.js` script in every sub-folder. -void Test::createAllRecursiveScripts() { +void TestCreator::createAllRecursiveScripts() { if (!createAllFilesSetup()) { return; } @@ -768,7 +771,7 @@ void Test::createAllRecursiveScripts() { QMessageBox::information(0, "Success", "Scripts have been created"); } -void Test::createAllRecursiveScripts(const QString& directory) { +void TestCreator::createAllRecursiveScripts(const QString& directory) { QDirIterator it(directory, QDirIterator::Subdirectories); while (it.hasNext()) { @@ -780,7 +783,7 @@ void Test::createAllRecursiveScripts(const QString& directory) { } } -void Test::createRecursiveScript(const QString& directory, bool interactiveMode) { +void TestCreator::createRecursiveScript(const QString& directory, bool interactiveMode) { // If folder contains a test, then we are at a leaf const QString testPathname{ directory + "/" + TEST_FILENAME }; if (QFileInfo(testPathname).exists()) { @@ -846,7 +849,10 @@ void Test::createRecursiveScript(const QString& directory, bool interactiveMode) textStream << " nitpick = createNitpick(Script.resolvePath(\".\"));" << endl; textStream << " testsRootPath = nitpick.getTestsRootPath();" << endl << endl; textStream << " nitpick.enableRecursive();" << endl; - textStream << " nitpick.enableAuto();" << endl; + textStream << " nitpick.enableAuto();" << endl << endl; + textStream << " if (typeof Test !== 'undefined') {" << endl; + textStream << " Test.wait(10000);" << endl; + textStream << " }" << endl; textStream << "} else {" << endl; textStream << " depth++" << endl; textStream << "}" << endl << endl; @@ -866,7 +872,7 @@ void Test::createRecursiveScript(const QString& directory, bool interactiveMode) recursiveTestsFile.close(); } -void Test::createTestsOutline() { +void TestCreator::createTestsOutline() { QString previousSelection = _testDirectory; QString parent = previousSelection.left(previousSelection.lastIndexOf('/')); if (!parent.isNull() && parent.right(1) != "/") { @@ -892,7 +898,7 @@ void Test::createTestsOutline() { QTextStream stream(&mdFile); - //Test title + //TestCreator title stream << "# Outline of all tests\n"; stream << "Directories with an appended (*) have an automatic test\n\n"; @@ -950,10 +956,10 @@ void Test::createTestsOutline() { mdFile.close(); - QMessageBox::information(0, "Success", "Test outline file " + testsOutlineFilename + " has been created"); + QMessageBox::information(0, "Success", "TestCreator outline file " + testsOutlineFilename + " has been created"); } -void Test::createTestRailTestCases() { +void TestCreator::createTestRailTestCases() { QString previousSelection = _testDirectory; QString parent = previousSelection.left(previousSelection.lastIndexOf('/')); if (!parent.isNull() && parent.right(1) != "/") { @@ -990,7 +996,7 @@ void Test::createTestRailTestCases() { } } -void Test::createTestRailRun() { +void TestCreator::createTestRailRun() { QString outputDirectory = QFileDialog::getExistingDirectory(nullptr, "Please select a folder to store generated files in", nullptr, QFileDialog::ShowDirsOnly); @@ -1006,9 +1012,9 @@ void Test::createTestRailRun() { _testRailInterface->createTestRailRun(outputDirectory); } -void Test::updateTestRailRunResult() { +void TestCreator::updateTestRailRunResult() { QString testResults = QFileDialog::getOpenFileName(nullptr, "Please select the zipped test results to update from", nullptr, - "Zipped Test Results (*.zip)"); + "Zipped TestCreator Results (*.zip)"); if (testResults.isNull()) { return; } @@ -1027,7 +1033,7 @@ void Test::updateTestRailRunResult() { _testRailInterface->updateTestRailRunResults(testResults, tempDirectory); } -QStringList Test::createListOfAll_imagesInDirectory(const QString& imageFormat, const QString& pathToImageDirectory) { +QStringList TestCreator::createListOfAll_imagesInDirectory(const QString& imageFormat, const QString& pathToImageDirectory) { _imageDirectory = QDir(pathToImageDirectory); QStringList nameFilters; nameFilters << "*." + imageFormat; @@ -1039,7 +1045,7 @@ QStringList Test::createListOfAll_imagesInDirectory(const QString& imageFormat, // Filename (i.e. without extension) contains tests (this is based on all test scripts being within the tests folder) // Last 5 characters in filename are digits (after removing the extension) // Extension is 'imageFormat' -bool Test::isInSnapshotFilenameFormat(const QString& imageFormat, const QString& filename) { +bool TestCreator::isInSnapshotFilenameFormat(const QString& imageFormat, const QString& filename) { bool contains_tests = filename.contains("tests" + PATH_SEPARATOR); QString filenameWithoutExtension = filename.left(filename.lastIndexOf('.')); @@ -1054,7 +1060,7 @@ bool Test::isInSnapshotFilenameFormat(const QString& imageFormat, const QString& // For a file named "D_GitHub_hifi-tests_tests_content_entity_zone_create_0.jpg", the test directory is // D:/GitHub/hifi-tests/tests/content/entity/zone/create // This method assumes the filename is in the correct format -QString Test::getExpectedImageDestinationDirectory(const QString& filename) { +QString TestCreator::getExpectedImageDestinationDirectory(const QString& filename) { QString filenameWithoutExtension = filename.left(filename.length() - 4); QStringList filenameParts = filenameWithoutExtension.split(PATH_SEPARATOR); @@ -1071,7 +1077,7 @@ QString Test::getExpectedImageDestinationDirectory(const QString& filename) { // is ...tests/content/entity/zone/create // This is used to create the full URL // This method assumes the filename is in the correct format -QString Test::getExpectedImagePartialSourceDirectory(const QString& filename) { +QString TestCreator::getExpectedImagePartialSourceDirectory(const QString& filename) { QString filenameWithoutExtension = filename.left(filename.length() - 4); QStringList filenameParts = filenameWithoutExtension.split(PATH_SEPARATOR); @@ -1096,18 +1102,18 @@ QString Test::getExpectedImagePartialSourceDirectory(const QString& filename) { return result; } -void Test::setTestRailCreateMode(TestRailCreateMode testRailCreateMode) { +void TestCreator::setTestRailCreateMode(TestRailCreateMode testRailCreateMode) { _testRailCreateMode = testRailCreateMode; } -void Test::createWebPage( +void TestCreator::createWebPage( QCheckBox* updateAWSCheckBox, QRadioButton* diffImageRadioButton, QRadioButton* ssimImageRadionButton, QLineEdit* urlLineEdit ) { QString testResults = QFileDialog::getOpenFileName(nullptr, "Please select the zipped test results to update from", nullptr, - "Zipped Test Results (TestResults--*.zip)"); + "Zipped TestCreator Results (TestResults--*.zip)"); if (testResults.isNull()) { return; } diff --git a/tools/nitpick/src/Test.h b/tools/nitpick/src/TestCreator.h similarity index 95% rename from tools/nitpick/src/Test.h rename to tools/nitpick/src/TestCreator.h index 8753b9fcda..cc32499967 100644 --- a/tools/nitpick/src/Test.h +++ b/tools/nitpick/src/TestCreator.h @@ -1,5 +1,5 @@ // -// Test.h +// TestCreator.h // // Created by Nissim Hadar on 2 Nov 2017. // Copyright 2013 High Fidelity, Inc. @@ -8,8 +8,8 @@ // See the accompanying file LICENSE or http://www.apache.org/licenses/LICENSE-2.0.html // -#ifndef hifi_test_h -#define hifi_test_h +#ifndef hifi_testCreator_h +#define hifi_testCreator_h #include #include @@ -18,6 +18,7 @@ #include "AWSInterface.h" #include "ImageComparer.h" +#include "Downloader.h" #include "MismatchWindow.h" #include "TestRailInterface.h" @@ -40,9 +41,9 @@ enum TestRailCreateMode { XML }; -class Test { +class TestCreator { public: - Test(QProgressBar* progressBar, QCheckBox* checkBoxInteractiveMode); + TestCreator(QProgressBar* progressBar, QCheckBox* checkBoxInteractiveMode); void startTestsEvaluation(const bool isRunningFromCommandLine, const bool isRunningInAutomaticTestRun, @@ -167,6 +168,7 @@ private: TestRailCreateMode _testRailCreateMode { PYTHON }; AWSInterface* _awsInterface; + Downloader* _downloader; }; -#endif // hifi_test_h \ No newline at end of file +#endif \ No newline at end of file diff --git a/tools/nitpick/src/TestRailInterface.cpp b/tools/nitpick/src/TestRailInterface.cpp index 6ed13a72b6..8b8803153f 100644 --- a/tools/nitpick/src/TestRailInterface.cpp +++ b/tools/nitpick/src/TestRailInterface.cpp @@ -9,7 +9,7 @@ // #include "TestRailInterface.h" -#include "Test.h" +#include "TestCreator.h" #include #include @@ -258,7 +258,7 @@ bool TestRailInterface::requestTestRailResultsDataFromUser() { } bool TestRailInterface::isAValidTestDirectory(const QString& directory) { - if (Test::isAValidDirectory(directory)) { + if (TestCreator::isAValidDirectory(directory)) { // Ignore the utils and preformance directories if (directory.right(QString("utils").length()) == "utils" || directory.right(QString("performance").length()) == "performance") { diff --git a/tools/nitpick/src/TestRunner.cpp b/tools/nitpick/src/TestRunner.cpp index 54246de80b..c4e991e5ee 100644 --- a/tools/nitpick/src/TestRunner.cpp +++ b/tools/nitpick/src/TestRunner.cpp @@ -14,6 +14,26 @@ #include "Nitpick.h" extern Nitpick* nitpick; +TestRunner::TestRunner( + QLabel* workingFolderLabel, + QLabel* statusLabel, + QCheckBox* usePreviousInstallationCheckBox, + QCheckBox* runLatest, + QLineEdit* url, + QCheckBox* runFullSuite, + QLineEdit* scriptURL +) { + _workingFolderLabel = workingFolderLabel; + _statusLabel = statusLabel; + _usePreviousInstallationCheckBox = usePreviousInstallationCheckBox; + _runLatest = runLatest; + _url = url; + _runFullSuite = runFullSuite; + _scriptURL = scriptURL; + + _downloader = new Downloader(); +} + void TestRunner::setWorkingFolder(QLabel* workingFolderLabel) { // Everything will be written to this folder QString previousSelection = _workingFolder; @@ -49,7 +69,7 @@ void TestRunner::downloadBuildXml(void* caller) { urls << DEV_BUILD_XML_URL; filenames << DEV_BUILD_XML_FILENAME; - nitpick->downloadFiles(urls, _workingFolder, filenames, caller); + _downloader->downloadFiles(urls, _workingFolder, filenames, caller); } void TestRunner::parseBuildInformation() { diff --git a/tools/nitpick/src/TestRunner.h b/tools/nitpick/src/TestRunner.h index d2468ec2fa..6d36f246f7 100644 --- a/tools/nitpick/src/TestRunner.h +++ b/tools/nitpick/src/TestRunner.h @@ -11,6 +11,8 @@ #ifndef hifi_testRunner_h #define hifi_testRunner_h +#include "Downloader.h" + #include #include #include @@ -28,7 +30,18 @@ public: class TestRunner { public: + TestRunner( + QLabel* workingFolderLabel, + QLabel* statusLabel, + QCheckBox* usePreviousInstallationOnMobileCheckBox, + QCheckBox* runLatest, + QLineEdit* url, + QCheckBox* runFullSuite, + QLineEdit* scriptURL + ); + void setWorkingFolder(QLabel* workingFolderLabel); + void downloadBuildXml(void* caller); void parseBuildInformation(); QString getInstallerNameFromURL(const QString& url); @@ -36,10 +49,15 @@ public: void appendLog(const QString& message); protected: + Downloader* _downloader; + QLabel* _workingFolderLabel; QLabel* _statusLabel; - QLineEdit* _url; + QCheckBox* _usePreviousInstallationCheckBox; QCheckBox* _runLatest; + QLineEdit* _url; + QCheckBox* _runFullSuite; + QLineEdit* _scriptURL; QString _workingFolder; diff --git a/tools/nitpick/src/TestRunnerDesktop.cpp b/tools/nitpick/src/TestRunnerDesktop.cpp index e45d895886..b9caaa0ecb 100644 --- a/tools/nitpick/src/TestRunnerDesktop.cpp +++ b/tools/nitpick/src/TestRunnerDesktop.cpp @@ -27,23 +27,22 @@ TestRunnerDesktop::TestRunnerDesktop( std::vector timeEdits, QLabel* workingFolderLabel, QCheckBox* runServerless, + QCheckBox* usePreviousInstallationOnMobileCheckBox, QCheckBox* runLatest, QLineEdit* url, + QCheckBox* runFullSuite, + QLineEdit* scriptURL, QPushButton* runNow, QLabel* statusLabel, QObject* parent -) : QObject(parent) +) : QObject(parent), TestRunner(workingFolderLabel, statusLabel, usePreviousInstallationOnMobileCheckBox, runLatest, url, runFullSuite, scriptURL) { _dayCheckboxes = dayCheckboxes; _timeEditCheckboxes = timeEditCheckboxes; _timeEdits = timeEdits; - _workingFolderLabel = workingFolderLabel; _runServerless = runServerless; - _runLatest = runLatest; - _url = url; _runNow = runNow; - _statusLabel = statusLabel; _installerThread = new QThread(); _installerWorker = new InstallerWorker(); @@ -179,10 +178,14 @@ void TestRunnerDesktop::run() { // This will be restored at the end of the tests saveExistingHighFidelityAppDataFolder(); - _statusLabel->setText("Downloading Build XML"); - downloadBuildXml((void*)this); + if (_usePreviousInstallationCheckBox->isChecked()) { + installationComplete(); + } else { + _statusLabel->setText("Downloading Build XML"); + downloadBuildXml((void*)this); - // `downloadComplete` will run after download has completed + downloadComplete(); + } } void TestRunnerDesktop::downloadComplete() { @@ -209,9 +212,9 @@ void TestRunnerDesktop::downloadComplete() { _statusLabel->setText("Downloading installer"); - nitpick->downloadFiles(urls, _workingFolder, filenames, (void*)this); + _downloader->downloadFiles(urls, _workingFolder, filenames, (void*)this); - // `downloadComplete` will run again after download has completed + downloadComplete(); } else { // Download of Installer has completed @@ -292,15 +295,19 @@ void TestRunnerDesktop::installationComplete() { void TestRunnerDesktop::verifyInstallationSucceeded() { // Exit if the executables are missing. - // On Windows, the reason is probably that UAC has blocked the installation. This is treated as a critical error #ifdef Q_OS_WIN QFileInfo interfaceExe(QDir::toNativeSeparators(_installationFolder) + "\\interface.exe"); QFileInfo assignmentClientExe(QDir::toNativeSeparators(_installationFolder) + "\\assignment-client.exe"); QFileInfo domainServerExe(QDir::toNativeSeparators(_installationFolder) + "\\domain-server.exe"); if (!interfaceExe.exists() || !assignmentClientExe.exists() || !domainServerExe.exists()) { - QMessageBox::critical(0, "Installation of High Fidelity has failed", "Please verify that UAC has been disabled"); - exit(-1); + if (_runLatest->isChecked()) { + // On Windows, the reason is probably that UAC has blocked the installation. This is treated as a critical error + QMessageBox::critical(0, "Installation of High Fidelity has failed", "Please verify that UAC has been disabled"); + exit(-1); + } else { + QMessageBox::critical(0, "Installation of High Fidelity not found", "Please verify that working folder contains a proper installation"); + } } #endif } @@ -457,8 +464,9 @@ void TestRunnerDesktop::runInterfaceWithTestScript() { QString deleteScript = QString("https://raw.githubusercontent.com/") + _user + "/hifi_tests/" + _branch + "/tests/utils/deleteNearbyEntities.js"; - QString testScript = - QString("https://raw.githubusercontent.com/") + _user + "/hifi_tests/" + _branch + "/tests/testRecursive.js"; + QString testScript = (_runFullSuite->isChecked()) + ? QString("https://raw.githubusercontent.com/") + _user + "/hifi_tests/" + _branch + "/tests/testRecursive.js" + : _scriptURL->text(); QString commandLine; #ifdef Q_OS_WIN @@ -537,15 +545,16 @@ void TestRunnerDesktop::runInterfaceWithTestScript() { } void TestRunnerDesktop::interfaceExecutionComplete() { + QThread::msleep(500); QFileInfo testCompleted(QDir::toNativeSeparators(_snapshotFolder) +"/tests_completed.txt"); if (!testCompleted.exists()) { QMessageBox::critical(0, "Tests not completed", "Interface seems to have crashed before completion of the test scripts\nExisting images will be evaluated"); } + killProcesses(); + evaluateResults(); - killProcesses(); - // The High Fidelity AppData folder will be restored after evaluation has completed } @@ -591,7 +600,6 @@ void TestRunnerDesktop::addBuildNumberToResults(const QString& zippedFolderName) if (!QFile::rename(zippedFolderName, augmentedFilename)) { QMessageBox::critical(0, "Internal error: " + QString(__FILE__) + ":" + QString::number(__LINE__), "Could not rename '" + zippedFolderName + "' to '" + augmentedFilename); exit(-1); - } } @@ -667,6 +675,13 @@ void TestRunnerDesktop::checkTime() { QString TestRunnerDesktop::getPRNumberFromURL(const QString& url) { try { QStringList urlParts = url.split("/"); + if (urlParts.size() <= 2) { +#ifdef Q_OS_WIN + throw "URL not in expected format, should look like `https://deployment.highfidelity.com/jobs/pr-build/label%3Dwindows/13023/HighFidelity-Beta-Interface-PR14006-be76c43.exe`"; +#elif defined Q_OS_MAC + throw "URL not in expected format, should look like `https://deployment.highfidelity.com/jobs/pr-build/label%3Dwindows/13023/HighFidelity-Beta-Interface-PR14006-be76c43.dmg`"; +#endif + } QStringList filenameParts = urlParts[urlParts.size() - 1].split("-"); if (filenameParts.size() <= 3) { #ifdef Q_OS_WIN diff --git a/tools/nitpick/src/TestRunnerDesktop.h b/tools/nitpick/src/TestRunnerDesktop.h index 140a81f465..dce2dce2ba 100644 --- a/tools/nitpick/src/TestRunnerDesktop.h +++ b/tools/nitpick/src/TestRunnerDesktop.h @@ -12,7 +12,6 @@ #define hifi_testRunnerDesktop_h #include -#include #include #include #include @@ -32,8 +31,11 @@ public: std::vector timeEdits, QLabel* workingFolderLabel, QCheckBox* runServerless, + QCheckBox* usePreviousInstallationOnMobileCheckBox, QCheckBox* runLatest, QLineEdit* url, + QCheckBox* runFullSuite, + QLineEdit* scriptURL, QPushButton* runNow, QLabel* statusLabel, @@ -99,7 +101,6 @@ private: std::vector _dayCheckboxes; std::vector _timeEditCheckboxes; std::vector _timeEdits; - QLabel* _workingFolderLabel; QCheckBox* _runServerless; QPushButton* _runNow; QTimer* _timer; diff --git a/tools/nitpick/src/TestRunnerMobile.cpp b/tools/nitpick/src/TestRunnerMobile.cpp index ab276f3337..4d0d18ef3d 100644 --- a/tools/nitpick/src/TestRunnerMobile.cpp +++ b/tools/nitpick/src/TestRunnerMobile.cpp @@ -25,14 +25,16 @@ TestRunnerMobile::TestRunnerMobile( QPushButton* downloadAPKPushbutton, QPushButton* installAPKPushbutton, QPushButton* runInterfacePushbutton, + QCheckBox* usePreviousInstallationOnMobileCheckBox, QCheckBox* runLatest, QLineEdit* url, + QCheckBox* runFullSuite, + QLineEdit* scriptURL, QLabel* statusLabel, QObject* parent -) : QObject(parent), _adbInterface(NULL) +) : QObject(parent), TestRunner(workingFolderLabel, statusLabel, usePreviousInstallationOnMobileCheckBox, runLatest, url, runFullSuite, scriptURL) { - _workingFolderLabel = workingFolderLabel; _connectDeviceButton = connectDeviceButton; _pullFolderButton = pullFolderButton; _detectedDeviceLabel = detectedDeviceLabel; @@ -40,13 +42,15 @@ TestRunnerMobile::TestRunnerMobile( _downloadAPKPushbutton = downloadAPKPushbutton; _installAPKPushbutton = installAPKPushbutton; _runInterfacePushbutton = runInterfacePushbutton; - _runLatest = runLatest; - _url = url; - _statusLabel = statusLabel; folderLineEdit->setText("/sdcard/DCIM/TEST"); modelNames["SM_G955U1"] = "Samsung S8+ unlocked"; + modelNames["SM_N960U1"] = "Samsung Note 9 unlocked"; + modelNames["SM_T380"] = "Samsung Tab A"; + modelNames["Quest"] = "Quest"; + + _adbInterface = NULL; } TestRunnerMobile::~TestRunnerMobile() { @@ -66,6 +70,7 @@ void TestRunnerMobile::connectDevice() { QString devicesFullFilename{ _workingFolder + "/devices.txt" }; QString command = _adbInterface->getAdbCommand() + " devices -l > " + devicesFullFilename; + appendLog(command); system(command.toStdString().c_str()); if (!QFile::exists(devicesFullFilename)) { @@ -93,7 +98,7 @@ void TestRunnerMobile::connectDevice() { QString deviceID = tokens[0]; QString modelID = tokens[3].split(':')[1]; - QString modelName = "UKNOWN"; + QString modelName = "UNKNOWN"; if (modelNames.count(modelID) == 1) { modelName = modelNames[modelID]; } @@ -102,6 +107,8 @@ void TestRunnerMobile::connectDevice() { _pullFolderButton->setEnabled(true); _folderLineEdit->setEnabled(true); _downloadAPKPushbutton->setEnabled(true); + _installAPKPushbutton->setEnabled(true); + _runInterfacePushbutton->setEnabled(true); } } #endif @@ -109,6 +116,8 @@ void TestRunnerMobile::connectDevice() { void TestRunnerMobile::downloadAPK() { downloadBuildXml((void*)this); + + downloadComplete(); } @@ -141,11 +150,12 @@ void TestRunnerMobile::downloadComplete() { _statusLabel->setText("Downloading installer"); - nitpick->downloadFiles(urls, _workingFolder, filenames, (void*)this); + _downloader->downloadFiles(urls, _workingFolder, filenames, (void*)this); } else { _statusLabel->setText("Installer download complete"); - _installAPKPushbutton->setEnabled(true); } + + _installAPKPushbutton->setEnabled(true); } void TestRunnerMobile::installAPK() { @@ -154,11 +164,25 @@ void TestRunnerMobile::installAPK() { _adbInterface = new AdbInterface(); } + if (_installerFilename.isNull()) { + QString installerPathname = QFileDialog::getOpenFileName(nullptr, "Please select the APK", _workingFolder, + "Available APKs (*.apk)" + ); + + if (installerPathname.isNull()) { + return; + } + + // Remove the path + QStringList parts = installerPathname.split('/'); + _installerFilename = parts[parts.length() - 1]; + } + _statusLabel->setText("Installing"); QString command = _adbInterface->getAdbCommand() + " install -r -d " + _workingFolder + "/" + _installerFilename + " >" + _workingFolder + "/installOutput.txt"; + appendLog(command); system(command.toStdString().c_str()); _statusLabel->setText("Installation complete"); - _runInterfacePushbutton->setEnabled(true); #endif } @@ -169,7 +193,22 @@ void TestRunnerMobile::runInterface() { } _statusLabel->setText("Starting Interface"); - QString command = _adbInterface->getAdbCommand() + " shell monkey -p io.highfidelity.hifiinterface -v 1"; + + QString testScript = (_runFullSuite->isChecked()) + ? QString("https://raw.githubusercontent.com/") + nitpick->getSelectedUser() + "/hifi_tests/" + nitpick->getSelectedBranch() + "/tests/testRecursive.js" + : _scriptURL->text(); + + QString command = _adbInterface->getAdbCommand() + + " shell am start -n io.highfidelity.hifiinterface/.PermissionChecker" + + " --es args \\\"" + + " --url file:///~/serverless/tutorial.json" + + " --no-updater" + + " --no-login-suggestion" + + " --testScript " + testScript + " quitWhenFinished" + + " --testResultsLocation /sdcard/snapshots" + + "\\\""; + + appendLog(command); system(command.toStdString().c_str()); _statusLabel->setText("Interface started"); #endif @@ -182,7 +221,8 @@ void TestRunnerMobile::pullFolder() { } _statusLabel->setText("Pulling folder"); - QString command = _adbInterface->getAdbCommand() + " pull " + _folderLineEdit->text() + " " + _workingFolder + _installerFilename; + QString command = _adbInterface->getAdbCommand() + " pull " + _folderLineEdit->text() + " " + _workingFolder; + appendLog(command); system(command.toStdString().c_str()); _statusLabel->setText("Pull complete"); #endif diff --git a/tools/nitpick/src/TestRunnerMobile.h b/tools/nitpick/src/TestRunnerMobile.h index 52c2ba096d..f7b16da6f8 100644 --- a/tools/nitpick/src/TestRunnerMobile.h +++ b/tools/nitpick/src/TestRunnerMobile.h @@ -12,7 +12,6 @@ #define hifi_testRunnerMobile_h #include -#include #include #include @@ -31,8 +30,11 @@ public: QPushButton* downloadAPKPushbutton, QPushButton* installAPKPushbutton, QPushButton* runInterfacePushbutton, + QCheckBox* usePreviousInstallationOnMobileCheckBox, QCheckBox* runLatest, QLineEdit* url, + QCheckBox* runFullSuite, + QLineEdit* scriptURL, QLabel* statusLabel, QObject* parent = 0 diff --git a/tools/nitpick/ui/Nitpick.ui b/tools/nitpick/ui/Nitpick.ui index 1857a2118f..a0f368863d 100644 --- a/tools/nitpick/ui/Nitpick.ui +++ b/tools/nitpick/ui/Nitpick.ui @@ -34,6 +34,9 @@ + + true + 45 @@ -495,7 +498,7 @@ - 20 + 240 70 120 20 @@ -549,13 +552,80 @@ - 170 + 175 100 - 451 + 445 21 + + + + 128 + 125 + 40 + 31 + + + + Script + + + + + false + + + + 175 + 130 + 445 + 21 + + + + + + + 20 + 130 + 120 + 20 + + + + <html><head/><body><p>If unchecked, will not show results during evaluation</p></body></html> + + + Run Full Suite + + + true + + + + + true + + + + 20 + 70 + 171 + 20 + + + + <html><head/><body><p>If unchecked, will not show results during evaluation</p></body></html> + + + usePreviousInstallation + + + false + + @@ -568,7 +638,7 @@ 10 - 90 + 150 160 30 @@ -581,7 +651,7 @@ 190 - 96 + 156 320 30 @@ -623,7 +693,7 @@ 460 - 410 + 440 160 30 @@ -639,7 +709,7 @@ 10 - 410 + 440 440 30 @@ -651,9 +721,9 @@ - 170 - 170 - 451 + 175 + 245 + 445 21 @@ -662,7 +732,7 @@ 20 - 170 + 245 120 20 @@ -684,7 +754,7 @@ 10 - 210 + 100 160 30 @@ -696,7 +766,7 @@ - 300 + 20 60 41 31 @@ -709,7 +779,7 @@ - 350 + 70 60 271 31 @@ -726,7 +796,7 @@ 10 - 250 + 325 160 30 @@ -742,7 +812,7 @@ 10 - 300 + 375 160 30 @@ -751,6 +821,86 @@ Run Interface + + + + 140 + 240 + 31 + 31 + + + + URL + + + + + false + + + + 175 + 275 + 445 + 21 + + + + + + + 140 + 270 + 40 + 31 + + + + Script + + + + + + 20 + 275 + 120 + 20 + + + + <html><head/><body><p>If unchecked, will not show results during evaluation</p></body></html> + + + Run Full Suite + + + true + + + + + true + + + + 20 + 210 + 171 + 20 + + + + <html><head/><body><p>If unchecked, will not show results during evaluation</p></body></html> + + + usePreviousInstallation + + + false + + @@ -921,6 +1071,9 @@ 21 + + true +