Mirrors/overte-HifiExperiments
3
0
Fork 0
mirror of https://github.com/HifiExperiments/overte.git synced 2025-04-07 10:02:24 +02:00
Code Issues Projects Releases Packages Wiki Activity Actions

Use python downloader.

Browse source
This commit is contained in:
NissimHadar 2019-03-01 12:32:55 -08:00
parent 8dadeb197b
commit 6697d9d49f
17 changed files with 424 additions and 1505 deletions
Show stats Download patch file Download diff file Expand all files Collapse all files

14
tools/nitpick/README.md
View file

@ -47,14 +47,20 @@ These steps assume the hifi repository has been cloned to `~/hifi`.
### Windows
1. (First time) download and install Python 3 from https://hifi-qa.s3.amazonaws.com/nitpick/Windows/python-3.7.0-amd64.exe (also located at https://www.python.org/downloads/)
1. Click the "add python to path" checkbox on the python installer
1. After installation - add the path to python.exe to the Windows PATH environment variable.
1. After installation:
1. Open a new terminal
1. Enter `python` and hit enter
1. Verify that python is available (the prompt will change to `>>>`)
1. Type `exit()` and hit enter to close python
1. Install requests (a python library to download files from URLs)
`pip3 install requests`
1. (First time) download and install AWS CLI from https://hifi-qa.s3.amazonaws.com/nitpick/Windows/AWSCLI64PY3.msi (also available at https://aws.amazon.com/cli/
1. Open a new command prompt and run
`aws configure`
1. Enter the AWS account number
1. Enter the secret key
1. Leave region name and ouput format as default [None]
1. Install the latest release of Boto3 via pip:
1. Install the latest release of Boto3 via pip (from a terminal):
`pip install boto3`
1. (First time) Download adb (Android Debug Bridge) from *https://dl.google.com/android/repository/platform-tools-latest-windows.zip*
@ -76,11 +82,13 @@ These steps assume the hifi repository has been cloned to `~/hifi`.
`open "/Applications/Python 3.7/Install Certificates.command"`.
This is needed because the Mac Python supplied no longer links with the deprecated Apple-supplied system OpenSSL libraries but rather supplies a private copy of OpenSSL 1.0.2 which does not automatically access the system default root certificates.
1. Verify that `/usr/local/bin/python3` exists.
1. (First time - AWS interface) Install pip with the script provided by the Python Packaging Authority:
In a terminal:
`curl -O https://bootstrap.pypa.io/get-pip.py`
In a terminal:
`python3 get-pip.py --user`
1. Install requests (a python library to download files from URLs)
`pip3 install requests`
1. (First time - AWS interface) Install pip with the script provided by the Python Packaging Authority:
1. Use pip to install the AWS CLI.
`pip3 install awscli --upgrade --user`
This will install aws in your user. For user XXX, aws will be located in ~/Library/Python/3.7/bin

2
tools/nitpick/src/AWSInterface.cpp
View file

@ -469,7 +469,7 @@ void AWSInterface::updateAWS() {
if (!file.open(QIODevice::WriteOnly | QIODevice::Text)) {
QMessageBox::critical(0, "Internal error: " + QString(__FILE__) + ":" + QString::number(__LINE__),
"Could not create 'addTestCases.py'");
"Could not create 'updateAWS.py'");
exit(-1);
}

6
tools/nitpick/src/AdbInterface.h
View file

@ -17,12 +17,6 @@ public:
QString getAdbCommand();
private:
#ifdef Q_OS_WIN
const QString _adbExe{ "adb.exe" };
#else
// Both Mac and Linux use "python"
const QString _adbExe{ "adb" };
#endif
QString _adbCommand;
};

71
tools/nitpick/src/Downloader.cpp
View file

@ -8,32 +8,65 @@
// See the accompanying file LICENSE or http://www.apache.org/licenses/LICENSE-2.0.html
//
#include "Downloader.h"
#include "PythonInterface.h"
#include <QtWidgets/QMessageBox>
#include <QFile>
#include <QMessageBox>
#include <QProcess>
#include <QThread>
#include <QTextStream>
Downloader::Downloader(QUrl fileURL, QObject *parent) : QObject(parent) {
_networkAccessManager.get(QNetworkRequest(fileURL));
connect(
&_networkAccessManager, SIGNAL (finished(QNetworkReply*)),
this, SLOT (fileDownloaded(QNetworkReply*))
);
Downloader::Downloader() {
PythonInterface* pythonInterface = new PythonInterface();
_pythonCommand = pythonInterface->getPythonCommand();
}
void Downloader::fileDownloaded(QNetworkReply* reply) {
QNetworkReply::NetworkError error = reply->error();
if (error != QNetworkReply::NetworkError::NoError) {
QMessageBox::information(0, "Test Aborted", "Failed to download file: " + reply->errorString());
void Downloader::downloadFiles(const QStringList& URLs, const QString& directoryName, const QStringList& filenames, void *caller) {
if (URLs.size() <= 0) {
return;
}
_downloadedData = reply->readAll();
QString filename = directoryName + "/downloadFiles.py";
if (QFile::exists(filename)) {
QFile::remove(filename);
}
QFile file(filename);
//emit a signal
reply->deleteLater();
emit downloaded();
}
if (!file.open(QIODevice::WriteOnly | QIODevice::Text)) {
QMessageBox::critical(0, "Internal error: " + QString(__FILE__) + ":" + QString::number(__LINE__),
"Could not create 'downloadFiles.py'");
exit(-1);
}
QByteArray Downloader::downloadedData() const {
return _downloadedData;
QTextStream stream(&file);
stream << "import requests\n";
for (int i = 0; i < URLs.size(); ++i) {
stream << "\nurl = '" + URLs[i] + "'\n";
stream << "r = requests.get(url)\n";
stream << "open('" + directoryName + '/' + filenames [i] + "', 'wb').write(r.content)\n";
}
file.close();
#ifdef Q_OS_WIN
QProcess* process = new QProcess();
connect(process, &QProcess::started, this, [=]() { _busyWindow.exec(); });
connect(process, SIGNAL(finished(int)), process, SLOT(deleteLater()));
connect(process, static_cast<void (QProcess::*)(int, QProcess::ExitStatus)>(&QProcess::finished), this,
[=](int exitCode, QProcess::ExitStatus exitStatus) { _busyWindow.hide(); });
QStringList parameters = QStringList() << filename;
process->start(_pythonCommand, parameters);
#elif defined Q_OS_MAC
QProcess* process = new QProcess();
QStringList parameters = QStringList() << "-c" << _pythonCommand + " " + filename;
process->start("sh", parameters);
// Wait for the last file to download
while (!QFile::exists(directoryName + '/' + filenames[filenames.length() - 1])) {
QThread::msleep(200);
}
#endif
}

29
tools/nitpick/src/Downloader.h
View file

@ -11,38 +11,19 @@
#ifndef hifi_downloader_h
#define hifi_downloader_h
#include <QObject>
#include <QNetworkAccessManager>
#include <QNetworkRequest>
#include <QNetworkReply>
#include <QUrl>
#include <QDateTime>
#include <QFile>
#include <QFileInfo>
#include <QDebug>
#include "BusyWindow.h"
#include <QObject>
#include <QByteArray>
#include <QNetworkAccessManager>
#include <QNetworkRequest>
#include <QNetworkReply>
class Downloader : public QObject {
Q_OBJECT
public:
explicit Downloader(QUrl fileURL, QObject *parent = 0);
Downloader();
QByteArray downloadedData() const;
signals:
void downloaded();
private slots:
void fileDownloaded(QNetworkReply* pReply);
void downloadFiles(const QStringList& URLs, const QString& directoryName, const QStringList& filenames, void *caller);
private:
QNetworkAccessManager _networkAccessManager;
QByteArray _downloadedData;
QString _pythonCommand;
BusyWindow _busyWindow;
};
#endif // hifi_downloader_h

139
tools/nitpick/src/Nitpick.cpp
View file

@ -24,8 +24,6 @@ Nitpick::Nitpick(QWidget* parent) : QMainWindow(parent) {
_ui.progressBar->setVisible(false);
_ui.tabWidget->setCurrentIndex(0);
_signalMapper = new QSignalMapper();
connect(_ui.actionClose, &QAction::triggered, this, &Nitpick::on_closePushbutton_clicked);
connect(_ui.actionAbout, &QAction::triggered, this, &Nitpick::about);
connect(_ui.actionContent, &QAction::triggered, this, &Nitpick::content);
@ -40,7 +38,7 @@ Nitpick::Nitpick(QWidget* parent) : QMainWindow(parent) {
_ui.plainTextEdit->setReadOnly(true);
setWindowTitle("Nitpick - v3.0.0");
setWindowTitle("Nitpick - v3.1.0");
clientProfiles << "VR-High" << "Desktop-High" << "Desktop-Low" << "Mobile-Touch" << "VR-Standalone";
_ui.clientProfileComboBox->insertItems(0, clientProfiles);
@ -48,10 +46,8 @@ Nitpick::Nitpick(QWidget* parent) : QMainWindow(parent) {
}
Nitpick::~Nitpick() {
delete _signalMapper;
if (_test) {
delete _test;
if (_testCreator) {
delete _testCreator;
}
if (_testRunnerDesktop) {
@ -64,10 +60,10 @@ Nitpick::~Nitpick() {
}
void Nitpick::setup() {
if (_test) {
delete _test;
if (_testCreator) {
delete _testCreator;
}
_test = new Test(_ui.progressBar, _ui.checkBoxInteractiveMode);
_testCreator = new TestCreator(_ui.progressBar, _ui.checkBoxInteractiveMode);
std::vector<QCheckBox*> dayCheckboxes;
dayCheckboxes.emplace_back(_ui.mondayCheckBox);
@ -99,9 +95,12 @@ void Nitpick::setup() {
timeEditCheckboxes,
timeEdits,
_ui.workingFolderRunOnDesktopLabel,
_ui.checkBoxServerless,
_ui.checkBoxServerless,
_ui.usePreviousInstallationOnDesktopCheckBox,
_ui.runLatestOnDesktopCheckBox,
_ui.urlOnDesktopLineEdit,
_ui.runFullSuiteOnDesktopCheckBox,
_ui.scriptURLOnDesktopLineEdit,
_ui.runNowPushbutton,
_ui.statusLabelOnDesktop
);
@ -118,8 +117,11 @@ void Nitpick::setup() {
_ui.downloadAPKPushbutton,
_ui.installAPKPushbutton,
_ui.runInterfacePushbutton,
_ui.usePreviousInstallationOnMobileCheckBox,
_ui.runLatestOnMobileCheckBox,
_ui.urlOnMobileLineEdit,
_ui.runFullSuiteOnMobileCheckBox,
_ui.scriptURLOnMobileLineEdit,
_ui.statusLabelOnMobile
);
}
@ -130,7 +132,7 @@ void Nitpick::startTestsEvaluation(const bool isRunningFromCommandLine,
const QString& branch,
const QString& user
) {
_test->startTestsEvaluation(isRunningFromCommandLine, isRunningInAutomaticTestRun, snapshotDirectory, branch, user);
_testCreator->startTestsEvaluation(isRunningFromCommandLine, isRunningInAutomaticTestRun, snapshotDirectory, branch, user);
}
void Nitpick::on_tabWidget_currentChanged(int index) {
@ -149,47 +151,47 @@ void Nitpick::on_tabWidget_currentChanged(int index) {
}
void Nitpick::on_evaluateTestsPushbutton_clicked() {
_test->startTestsEvaluation(false, false);
_testCreator->startTestsEvaluation(false, false);
}
void Nitpick::on_createRecursiveScriptPushbutton_clicked() {
_test->createRecursiveScript();
_testCreator->createRecursiveScript();
}
void Nitpick::on_createAllRecursiveScriptsPushbutton_clicked() {
_test->createAllRecursiveScripts();
_testCreator->createAllRecursiveScripts();
}
void Nitpick::on_createTestsPushbutton_clicked() {
_test->createTests(_ui.clientProfileComboBox->currentText());
_testCreator->createTests(_ui.clientProfileComboBox->currentText());
}
void Nitpick::on_createMDFilePushbutton_clicked() {
_test->createMDFile();
_testCreator->createMDFile();
}
void Nitpick::on_createAllMDFilesPushbutton_clicked() {
_test->createAllMDFiles();
_testCreator->createAllMDFiles();
}
void Nitpick::on_createTestAutoScriptPushbutton_clicked() {
_test->createTestAutoScript();
_testCreator->createTestAutoScript();
}
void Nitpick::on_createAllTestAutoScriptsPushbutton_clicked() {
_test->createAllTestAutoScripts();
_testCreator->createAllTestAutoScripts();
}
void Nitpick::on_createTestsOutlinePushbutton_clicked() {
_test->createTestsOutline();
_testCreator->createTestsOutline();
}
void Nitpick::on_createTestRailTestCasesPushbutton_clicked() {
_test->createTestRailTestCases();
_testCreator->createTestRailTestCases();
}
void Nitpick::on_createTestRailRunButton_clicked() {
_test->createTestRailRun();
_testCreator->createTestRailRun();
}
void Nitpick::on_setWorkingFolderRunOnDesktopPushbutton_clicked() {
@ -206,16 +208,25 @@ void Nitpick::on_runNowPushbutton_clicked() {
_testRunnerDesktop->run();
}
void Nitpick::on_usePreviousInstallationOnDesktopCheckBox_clicked() {
_ui.runLatestOnDesktopCheckBox->setEnabled(!_ui.usePreviousInstallationOnDesktopCheckBox->isChecked());
_ui.urlOnDesktopLineEdit->setEnabled(!_ui.usePreviousInstallationOnDesktopCheckBox->isChecked() && !_ui.runLatestOnDesktopCheckBox->isChecked());
}
void Nitpick::on_runLatestOnDesktopCheckBox_clicked() {
_ui.urlOnDesktopLineEdit->setEnabled(!_ui.runLatestOnDesktopCheckBox->isChecked());
}
void Nitpick::on_runFullSuiteOnDesktopCheckBox_clicked() {
_ui.scriptURLOnDesktopLineEdit->setEnabled(!_ui.runFullSuiteOnDesktopCheckBox->isChecked());
}
void Nitpick::automaticTestRunEvaluationComplete(QString zippedFolderName, int numberOfFailures) {
_testRunnerDesktop->automaticTestRunEvaluationComplete(zippedFolderName, numberOfFailures);
}
void Nitpick::on_updateTestRailRunResultsPushbutton_clicked() {
_test->updateTestRailRunResult();
_testCreator->updateTestRailRunResult();
}
// To toggle between show and hide
@ -247,80 +258,15 @@ void Nitpick::on_closePushbutton_clicked() {
}
void Nitpick::on_createPythonScriptRadioButton_clicked() {
_test->setTestRailCreateMode(PYTHON);
_testCreator->setTestRailCreateMode(PYTHON);
}
void Nitpick::on_createXMLScriptRadioButton_clicked() {
_test->setTestRailCreateMode(XML);
_testCreator->setTestRailCreateMode(XML);
}
void Nitpick::on_createWebPagePushbutton_clicked() {
_test->createWebPage(_ui.updateAWSCheckBox, _ui.awsURLLineEdit);
}
void Nitpick::downloadFile(const QUrl& url) {
_downloaders.emplace_back(new Downloader(url, this));
connect(_downloaders[_index], SIGNAL(downloaded()), _signalMapper, SLOT(map()));
_signalMapper->setMapping(_downloaders[_index], _index);
++_index;
}
void Nitpick::downloadFiles(const QStringList& URLs, const QString& directoryName, const QStringList& filenames, void *caller) {
connect(_signalMapper, SIGNAL(mapped(int)), this, SLOT(saveFile(int)));
_directoryName = directoryName;
_filenames = filenames;
_caller = caller;
_numberOfFilesToDownload = URLs.size();
_numberOfFilesDownloaded = 0;
_index = 0;
_ui.progressBar->setMinimum(0);
_ui.progressBar->setMaximum(_numberOfFilesToDownload - 1);
_ui.progressBar->setValue(0);
_ui.progressBar->setVisible(true);
foreach (auto downloader, _downloaders) {
delete downloader;
}
_downloaders.clear();
for (int i = 0; i < _numberOfFilesToDownload; ++i) {
downloadFile(URLs[i]);
}
}
void Nitpick::saveFile(int index) {
try {
QFile file(_directoryName + "/" + _filenames[index]);
file.open(QIODevice::WriteOnly);
file.write(_downloaders[index]->downloadedData());
file.close();
} catch (...) {
QMessageBox::information(0, "Test Aborted", "Failed to save file: " + _filenames[index]);
_ui.progressBar->setVisible(false);
return;
}
++_numberOfFilesDownloaded;
if (_numberOfFilesDownloaded == _numberOfFilesToDownload) {
disconnect(_signalMapper, SIGNAL(mapped(int)), this, SLOT(saveFile(int)));
if (_caller == _test) {
_test->finishTestsEvaluation();
} else if (_caller == _testRunnerDesktop) {
_testRunnerDesktop->downloadComplete();
} else if (_caller == _testRunnerMobile) {
_testRunnerMobile->downloadComplete();
}
_ui.progressBar->setVisible(false);
} else {
_ui.progressBar->setValue(_numberOfFilesDownloaded);
}
_testCreator->createWebPage(_ui.updateAWSCheckBox, _ui.awsURLLineEdit);
}
void Nitpick::about() {
@ -360,10 +306,19 @@ void Nitpick::on_connectDevicePushbutton_clicked() {
_testRunnerMobile->connectDevice();
}
void Nitpick::on_usePreviousInstallationOnMobileCheckBox_clicked() {
_ui.runLatestOnMobileCheckBox->setEnabled(!_ui.usePreviousInstallationOnMobileCheckBox->isChecked());
_ui.urlOnMobileLineEdit->setEnabled(!_ui.usePreviousInstallationOnMobileCheckBox->isChecked() && !_ui.runLatestOnMobileCheckBox->isChecked());
}
void Nitpick::on_runLatestOnMobileCheckBox_clicked() {
_ui.urlOnMobileLineEdit->setEnabled(!_ui.runLatestOnMobileCheckBox->isChecked());
}
void Nitpick::on_runFullSuiteOnMobileCheckBox_clicked() {
_ui.scriptURLOnMobileLineEdit->setEnabled(!_ui.runFullSuiteOnMobileCheckBox->isChecked());
}
void Nitpick::on_downloadAPKPushbutton_clicked() {
_testRunnerMobile->downloadAPK();
}

31
tools/nitpick/src/Nitpick.h
View file

@ -11,12 +11,10 @@
#define hifi_Nitpick_h
#include <QtWidgets/QMainWindow>
#include <QSignalMapper>
#include <QTextEdit>
#include "ui_Nitpick.h"
#include "Downloader.h"
#include "Test.h"
#include "TestCreator.h"
#include "TestRunnerDesktop.h"
#include "TestRunnerMobile.h"
@ -38,9 +36,6 @@ public:
void automaticTestRunEvaluationComplete(QString zippedFolderName, int numberOfFailures);
void downloadFile(const QUrl& url);
void downloadFiles(const QStringList& URLs, const QString& directoryName, const QStringList& filenames, void* caller);
void setUserText(const QString& user);
QString getSelectedUser();
@ -75,7 +70,9 @@ private slots:
void on_setWorkingFolderRunOnDesktopPushbutton_clicked();
void on_runNowPushbutton_clicked();
void on_usePreviousInstallationOnDesktopCheckBox_clicked();
void on_runLatestOnDesktopCheckBox_clicked();
void on_runFullSuiteOnDesktopCheckBox_clicked();
void on_updateTestRailRunResultsPushbutton_clicked();
@ -87,15 +84,16 @@ private slots:
void on_createWebPagePushbutton_clicked();
void saveFile(int index);
void about();
void content();
// Run on Mobile controls
void on_setWorkingFolderRunOnMobilePushbutton_clicked();
void on_connectDevicePushbutton_clicked();
void on_usePreviousInstallationOnMobileCheckBox_clicked();
void on_runLatestOnMobileCheckBox_clicked();
void on_runFullSuiteOnMobileCheckBox_clicked();
void on_downloadAPKPushbutton_clicked();
void on_installAPKPushbutton_clicked();
@ -105,28 +103,13 @@ private slots:
private:
Ui::NitpickClass _ui;
Test* _test{ nullptr };
TestCreator* _testCreator{ nullptr };
TestRunnerDesktop* _testRunnerDesktop{ nullptr };
TestRunnerMobile* _testRunnerMobile{ nullptr };
std::vector<Downloader*> _downloaders;
// local storage for parameters - folder to store downloaded files in, and a list of their names
QString _directoryName;
QStringList _filenames;
// Used to enable passing a parameter to slots
QSignalMapper* _signalMapper;
int _numberOfFilesToDownload{ 0 };
int _numberOfFilesDownloaded{ 0 };
int _index{ 0 };
bool _isRunningFromCommandline{ false };
void* _caller;
QStringList clientProfiles;
};

1116
tools/nitpick/src/Test.cpp
View file

File diff suppressed because it is too large Load diff

168
tools/nitpick/src/Test.h
View file

@ -1,168 +0,0 @@
//
// Test.h
//
// Created by Nissim Hadar on 2 Nov 2017.
// Copyright 2013 High Fidelity, Inc.
//
// Distributed under the Apache License, Version 2.0.
// See the accompanying file LICENSE or http://www.apache.org/licenses/LICENSE-2.0.html
//
#ifndef hifi_test_h
#define hifi_test_h
#include <QtWidgets/QFileDialog>
#include <QtWidgets/QMessageBox>
#include <QtCore/QRegularExpression>
#include <QProgressBar>
#include "AWSInterface.h"
#include "ImageComparer.h"
#include "MismatchWindow.h"
#include "TestRailInterface.h"
class Step {
public:
QString text;
bool takeSnapshot;
};
using StepList = std::vector<Step*>;
class ExtractedText {
public:
QString title;
StepList stepList;
};
enum TestRailCreateMode {
PYTHON,
XML
};
class Test {
public:
Test(QProgressBar* progressBar, QCheckBox* checkBoxInteractiveMode);
void startTestsEvaluation(const bool isRunningFromCommandLine,
const bool isRunningInAutomaticTestRun,
const QString& snapshotDirectory = QString(),
const QString& branchFromCommandLine = QString(),
const QString& userFromCommandLine = QString());
void finishTestsEvaluation();
void createTests(const QString& clientProfile);
void createTestsOutline();
bool createFileSetup();
bool createAllFilesSetup();
void createMDFile();
void createAllMDFiles();
bool createMDFile(const QString& directory);
void createTestAutoScript();
void createAllTestAutoScripts();
bool createTestAutoScript(const QString& directory);
void createTestRailTestCases();
void createTestRailRun();
void updateTestRailRunResult();
void createAllRecursiveScripts();
void createAllRecursiveScripts(const QString& directory);
void createRecursiveScript();
void createRecursiveScript(const QString& directory, bool interactiveMode);
int compareImageLists();
int checkTextResults();
QStringList createListOfAll_imagesInDirectory(const QString& imageFormat, const QString& pathToImageDirectory);
bool isInSnapshotFilenameFormat(const QString& imageFormat, const QString& filename);
void includeTest(QTextStream& textStream, const QString& testPathname);
void appendTestResultsToFile(TestResult testResult, QPixmap comparisonImage, bool hasFailed);
void appendTestResultsToFile(QString testResultFilename, bool hasFailed);
bool createTestResultsFolderPath(const QString& directory);
QString zipAndDeleteTestResultsFolder();
static bool isAValidDirectory(const QString& pathname);
QString extractPathFromTestsDown(const QString& fullPath);
QString getExpectedImageDestinationDirectory(const QString& filename);
QString getExpectedImagePartialSourceDirectory(const QString& filename);
ExtractedText getTestScriptLines(QString testFileName);
void setTestRailCreateMode(TestRailCreateMode testRailCreateMode);
void createWebPage(QCheckBox* updateAWSCheckBox, QLineEdit* urlLineEdit);
private:
QProgressBar* _progressBar;
QCheckBox* _checkBoxInteractiveMode;
bool _isRunningFromCommandLine{ false };
bool _isRunningInAutomaticTestRun{ false };
const QString TEST_FILENAME{ "test.js" };
const QString TEST_RECURSIVE_FILENAME{ "testRecursive.js" };
const QString TEST_RESULTS_FOLDER { "TestResults" };
const QString TEST_RESULTS_FILENAME { "TestResults.txt" };
const double THRESHOLD{ 0.935 };
QDir _imageDirectory;
MismatchWindow _mismatchWindow;
ImageComparer _imageComparer;
QString _testResultsFolderPath;
int _failureIndex{ 1 };
int _successIndex{ 1 };
// Expected images are in the format ExpectedImage_dddd.jpg (d == decimal digit)
const int NUM_DIGITS { 5 };
const QString EXPECTED_IMAGE_PREFIX { "ExpectedImage_" };
// We have two directories to work with.
// The first is the directory containing the test we are working with
// The second is the root directory of all tests
// The third contains the snapshots taken for test runs that need to be evaluated
QString _testDirectory;
QString _testsRootDirectory;
QString _snapshotDirectory;
QStringList _expectedImagesFilenames;
QStringList _expectedImagesFullFilenames;
QStringList _resultImagesFullFilenames;
// Used for accessing GitHub
const QString GIT_HUB_DEFAULT_USER{ "highfidelity" };
const QString GIT_HUB_DEFAULT_BRANCH{ "master" };
const QString GIT_HUB_REPOSITORY{ "hifi_tests" };
const QString DATETIME_FORMAT{ "yyyy-MM-dd_hh-mm-ss" };
// NOTE: these need to match the appropriate var's in nitpick.js
// var advanceKey = "n";
// var pathSeparator = ".";
const QString ADVANCE_KEY{ "n" };
const QString PATH_SEPARATOR{ "." };
bool _exitWhenComplete{ false };
TestRailInterface* _testRailInterface;
TestRailCreateMode _testRailCreateMode { PYTHON };
AWSInterface* _awsInterface;
};
#endif // hifi_test_h

4
tools/nitpick/src/TestRailInterface.cpp
View file

@ -9,7 +9,7 @@
//
#include "TestRailInterface.h"
#include "Test.h"
#include "TestCreator.h"
#include <quazip5/quazip.h>
#include <quazip5/JlCompress.h>
@ -258,7 +258,7 @@ bool TestRailInterface::requestTestRailResultsDataFromUser() {
}
bool TestRailInterface::isAValidTestDirectory(const QString& directory) {
if (Test::isAValidDirectory(directory)) {
if (TestCreator::isAValidDirectory(directory)) {
// Ignore the utils and preformance directories
if (directory.right(QString("utils").length()) == "utils" ||
directory.right(QString("performance").length()) == "performance") {

22
tools/nitpick/src/TestRunner.cpp
View file

@ -14,6 +14,26 @@
#include "Nitpick.h"
extern Nitpick* nitpick;
TestRunner::TestRunner(
QLabel* workingFolderLabel,
QLabel* statusLabel,
QCheckBox* usePreviousInstallationCheckBox,
QCheckBox* runLatest,
QLineEdit* url,
QCheckBox* runFullSuite,
QLineEdit* scriptURL
) {
_workingFolderLabel = workingFolderLabel;
_statusLabel = statusLabel;
_usePreviousInstallationCheckBox = usePreviousInstallationCheckBox;
_runLatest = runLatest;
_url = url;
_runFullSuite = runFullSuite;
_scriptURL = scriptURL;
_downloader = new Downloader();
}
void TestRunner::setWorkingFolder(QLabel* workingFolderLabel) {
// Everything will be written to this folder
QString previousSelection = _workingFolder;
@ -49,7 +69,7 @@ void TestRunner::downloadBuildXml(void* caller) {
urls << DEV_BUILD_XML_URL;
filenames << DEV_BUILD_XML_FILENAME;
nitpick->downloadFiles(urls, _workingFolder, filenames, caller);
_downloader->downloadFiles(urls, _workingFolder, filenames, caller);
}
void TestRunner::parseBuildInformation() {

20
tools/nitpick/src/TestRunner.h
View file

@ -11,6 +11,8 @@
#ifndef hifi_testRunner_h
#define hifi_testRunner_h
#include "Downloader.h"
#include <QCheckBox>
#include <QDir>
#include <QLabel>
@ -28,7 +30,18 @@ public:
class TestRunner {
public:
TestRunner(
QLabel* workingFolderLabel,
QLabel* statusLabel,
QCheckBox* usePreviousInstallationOnMobileCheckBox,
QCheckBox* runLatest,
QLineEdit* url,
QCheckBox* runFullSuite,
QLineEdit* scriptURL
);
void setWorkingFolder(QLabel* workingFolderLabel);
void downloadBuildXml(void* caller);
void parseBuildInformation();
QString getInstallerNameFromURL(const QString& url);
@ -36,10 +49,15 @@ public:
void appendLog(const QString& message);
protected:
Downloader* _downloader;
QLabel* _workingFolderLabel;
QLabel* _statusLabel;
QLineEdit* _url;
QCheckBox* _usePreviousInstallationCheckBox;
QCheckBox* _runLatest;
QLineEdit* _url;
QCheckBox* _runFullSuite;
QLineEdit* _scriptURL;
QString _workingFolder;

51
tools/nitpick/src/TestRunnerDesktop.cpp
View file

@ -27,23 +27,22 @@ TestRunnerDesktop::TestRunnerDesktop(
std::vector<QTimeEdit*> timeEdits,
QLabel* workingFolderLabel,
QCheckBox* runServerless,
QCheckBox* usePreviousInstallationOnMobileCheckBox,
QCheckBox* runLatest,
QLineEdit* url,
QCheckBox* runFullSuite,
QLineEdit* scriptURL,
QPushButton* runNow,
QLabel* statusLabel,
QObject* parent
) : QObject(parent)
) : QObject(parent), TestRunner(workingFolderLabel, statusLabel, usePreviousInstallationOnMobileCheckBox, runLatest, url, runFullSuite, scriptURL)
{
_dayCheckboxes = dayCheckboxes;
_timeEditCheckboxes = timeEditCheckboxes;
_timeEdits = timeEdits;
_workingFolderLabel = workingFolderLabel;
_runServerless = runServerless;
_runLatest = runLatest;
_url = url;
_runNow = runNow;
_statusLabel = statusLabel;
_installerThread = new QThread();
_installerWorker = new InstallerWorker();
@ -179,10 +178,14 @@ void TestRunnerDesktop::run() {
// This will be restored at the end of the tests
saveExistingHighFidelityAppDataFolder();
_statusLabel->setText("Downloading Build XML");
downloadBuildXml((void*)this);
if (_usePreviousInstallationCheckBox->isChecked()) {
installationComplete();
} else {
_statusLabel->setText("Downloading Build XML");
downloadBuildXml((void*)this);
// `downloadComplete` will run after download has completed
downloadComplete();
}
}
void TestRunnerDesktop::downloadComplete() {
@ -209,9 +212,9 @@ void TestRunnerDesktop::downloadComplete() {
_statusLabel->setText("Downloading installer");
nitpick->downloadFiles(urls, _workingFolder, filenames, (void*)this);
_downloader->downloadFiles(urls, _workingFolder, filenames, (void*)this);
// `downloadComplete` will run again after download has completed
downloadComplete();
} else {
// Download of Installer has completed
@ -292,15 +295,19 @@ void TestRunnerDesktop::installationComplete() {
void TestRunnerDesktop::verifyInstallationSucceeded() {
// Exit if the executables are missing.
// On Windows, the reason is probably that UAC has blocked the installation. This is treated as a critical error
#ifdef Q_OS_WIN
QFileInfo interfaceExe(QDir::toNativeSeparators(_installationFolder) + "\\interface.exe");
QFileInfo assignmentClientExe(QDir::toNativeSeparators(_installationFolder) + "\\assignment-client.exe");
QFileInfo domainServerExe(QDir::toNativeSeparators(_installationFolder) + "\\domain-server.exe");
if (!interfaceExe.exists() || !assignmentClientExe.exists() || !domainServerExe.exists()) {
QMessageBox::critical(0, "Installation of High Fidelity has failed", "Please verify that UAC has been disabled");
exit(-1);
if (_runLatest->isChecked()) {
// On Windows, the reason is probably that UAC has blocked the installation. This is treated as a critical error
QMessageBox::critical(0, "Installation of High Fidelity has failed", "Please verify that UAC has been disabled");
exit(-1);
} else {
QMessageBox::critical(0, "Installation of High Fidelity not found", "Please verify that working folder contains a proper installation");
}
}
#endif
}
@ -457,8 +464,9 @@ void TestRunnerDesktop::runInterfaceWithTestScript() {
QString deleteScript =
QString("https://raw.githubusercontent.com/") + _user + "/hifi_tests/" + _branch + "/tests/utils/deleteNearbyEntities.js";
QString testScript =
QString("https://raw.githubusercontent.com/") + _user + "/hifi_tests/" + _branch + "/tests/testRecursive.js";
QString testScript = (_runFullSuite->isChecked())
? QString("https://raw.githubusercontent.com/") + _user + "/hifi_tests/" + _branch + "/tests/testRecursive.js"
: _scriptURL->text();
QString commandLine;
#ifdef Q_OS_WIN
@ -537,15 +545,16 @@ void TestRunnerDesktop::runInterfaceWithTestScript() {
}
void TestRunnerDesktop::interfaceExecutionComplete() {
QThread::msleep(500);
QFileInfo testCompleted(QDir::toNativeSeparators(_snapshotFolder) +"/tests_completed.txt");
if (!testCompleted.exists()) {
QMessageBox::critical(0, "Tests not completed", "Interface seems to have crashed before completion of the test scripts\nExisting images will be evaluated");
}
killProcesses();
evaluateResults();
killProcesses();
// The High Fidelity AppData folder will be restored after evaluation has completed
}
@ -591,7 +600,6 @@ void TestRunnerDesktop::addBuildNumberToResults(const QString& zippedFolderName)
if (!QFile::rename(zippedFolderName, augmentedFilename)) {
QMessageBox::critical(0, "Internal error: " + QString(__FILE__) + ":" + QString::number(__LINE__), "Could not rename '" + zippedFolderName + "' to '" + augmentedFilename);
exit(-1);
}
}
@ -667,6 +675,13 @@ void TestRunnerDesktop::checkTime() {
QString TestRunnerDesktop::getPRNumberFromURL(const QString& url) {
try {
QStringList urlParts = url.split("/");
if (urlParts.size() <= 2) {
#ifdef Q_OS_WIN
throw "URL not in expected format, should look like `https://deployment.highfidelity.com/jobs/pr-build/label%3Dwindows/13023/HighFidelity-Beta-Interface-PR14006-be76c43.exe`";
#elif defined Q_OS_MAC
throw "URL not in expected format, should look like `https://deployment.highfidelity.com/jobs/pr-build/label%3Dwindows/13023/HighFidelity-Beta-Interface-PR14006-be76c43.dmg`";
#endif
}
QStringList filenameParts = urlParts[urlParts.size() - 1].split("-");
if (filenameParts.size() <= 3) {
#ifdef Q_OS_WIN

5
tools/nitpick/src/TestRunnerDesktop.h
View file

@ -12,7 +12,6 @@
#define hifi_testRunnerDesktop_h
#include <QDir>
#include <QLabel>
#include <QObject>
#include <QPushButton>
#include <QThread>
@ -32,8 +31,11 @@ public:
std::vector<QTimeEdit*> timeEdits,
QLabel* workingFolderLabel,
QCheckBox* runServerless,
QCheckBox* usePreviousInstallationOnMobileCheckBox,
QCheckBox* runLatest,
QLineEdit* url,
QCheckBox* runFullSuite,
QLineEdit* scriptURL,
QPushButton* runNow,
QLabel* statusLabel,
@ -99,7 +101,6 @@ private:
std::vector<QCheckBox*> _dayCheckboxes;
std::vector<QCheckBox*> _timeEditCheckboxes;
std::vector<QTimeEdit*> _timeEdits;
QLabel* _workingFolderLabel;
QCheckBox* _runServerless;
QPushButton* _runNow;
QTimer* _timer;

62
tools/nitpick/src/TestRunnerMobile.cpp
View file

@ -25,14 +25,16 @@ TestRunnerMobile::TestRunnerMobile(
QPushButton* downloadAPKPushbutton,
QPushButton* installAPKPushbutton,
QPushButton* runInterfacePushbutton,
QCheckBox* usePreviousInstallationOnMobileCheckBox,
QCheckBox* runLatest,
QLineEdit* url,
QCheckBox* runFullSuite,
QLineEdit* scriptURL,
QLabel* statusLabel,
QObject* parent
) : QObject(parent), _adbInterface(NULL)
) : QObject(parent), TestRunner(workingFolderLabel, statusLabel, usePreviousInstallationOnMobileCheckBox, runLatest, url, runFullSuite, scriptURL)
{
_workingFolderLabel = workingFolderLabel;
_connectDeviceButton = connectDeviceButton;
_pullFolderButton = pullFolderButton;
_detectedDeviceLabel = detectedDeviceLabel;
@ -40,13 +42,15 @@ TestRunnerMobile::TestRunnerMobile(
_downloadAPKPushbutton = downloadAPKPushbutton;
_installAPKPushbutton = installAPKPushbutton;
_runInterfacePushbutton = runInterfacePushbutton;
_runLatest = runLatest;
_url = url;
_statusLabel = statusLabel;
folderLineEdit->setText("/sdcard/DCIM/TEST");
modelNames["SM_G955U1"] = "Samsung S8+ unlocked";
modelNames["SM_N960U1"] = "Samsung Note 9 unlocked";
modelNames["SM_T380"] = "Samsung Tab A";
modelNames["Quest"] = "Quest";
_adbInterface = NULL;
}
TestRunnerMobile::~TestRunnerMobile() {
@ -66,6 +70,7 @@ void TestRunnerMobile::connectDevice() {
QString devicesFullFilename{ _workingFolder + "/devices.txt" };
QString command = _adbInterface->getAdbCommand() + " devices -l > " + devicesFullFilename;
appendLog(command);
system(command.toStdString().c_str());
if (!QFile::exists(devicesFullFilename)) {
@ -93,7 +98,7 @@ void TestRunnerMobile::connectDevice() {
QString deviceID = tokens[0];
QString modelID = tokens[3].split(':')[1];
QString modelName = "UKNOWN";
QString modelName = "UNKNOWN";
if (modelNames.count(modelID) == 1) {
modelName = modelNames[modelID];
}
@ -102,6 +107,8 @@ void TestRunnerMobile::connectDevice() {
_pullFolderButton->setEnabled(true);
_folderLineEdit->setEnabled(true);
_downloadAPKPushbutton->setEnabled(true);
_installAPKPushbutton->setEnabled(true);
_runInterfacePushbutton->setEnabled(true);
}
}
#endif
@ -109,6 +116,8 @@ void TestRunnerMobile::connectDevice() {
void TestRunnerMobile::downloadAPK() {
downloadBuildXml((void*)this);
downloadComplete();
}
@ -141,11 +150,12 @@ void TestRunnerMobile::downloadComplete() {
_statusLabel->setText("Downloading installer");
nitpick->downloadFiles(urls, _workingFolder, filenames, (void*)this);
_downloader->downloadFiles(urls, _workingFolder, filenames, (void*)this);
} else {
_statusLabel->setText("Installer download complete");
_installAPKPushbutton->setEnabled(true);
}
_installAPKPushbutton->setEnabled(true);
}
void TestRunnerMobile::installAPK() {
@ -154,11 +164,25 @@ void TestRunnerMobile::installAPK() {
_adbInterface = new AdbInterface();
}
if (_installerFilename.isNull()) {
QString installerPathname = QFileDialog::getOpenFileName(nullptr, "Please select the APK", _workingFolder,
"Available APKs (*.apk)"
);
if (installerPathname.isNull()) {
return;
}
// Remove the path
QStringList parts = installerPathname.split('/');
_installerFilename = parts[parts.length() - 1];
}
_statusLabel->setText("Installing");
QString command = _adbInterface->getAdbCommand() + " install -r -d " + _workingFolder + "/" + _installerFilename + " >" + _workingFolder + "/installOutput.txt";
appendLog(command);
system(command.toStdString().c_str());
_statusLabel->setText("Installation complete");
_runInterfacePushbutton->setEnabled(true);
#endif
}
@ -169,7 +193,22 @@ void TestRunnerMobile::runInterface() {
}
_statusLabel->setText("Starting Interface");
QString command = _adbInterface->getAdbCommand() + " shell monkey -p io.highfidelity.hifiinterface -v 1";
QString testScript = (_runFullSuite->isChecked())
? QString("https://raw.githubusercontent.com/") + nitpick->getSelectedUser() + "/hifi_tests/" + nitpick->getSelectedBranch() + "/tests/testRecursive.js"
: _scriptURL->text();
QString command = _adbInterface->getAdbCommand() +
" shell am start -n io.highfidelity.hifiinterface/.PermissionChecker" +
" --es args \\\"" +
" --url file:///~/serverless/tutorial.json" +
" --no-updater" +
" --no-login-suggestion" +
" --testScript " + testScript + " quitWhenFinished" +
" --testResultsLocation /sdcard/snapshots" +
"\\\"";
appendLog(command);
system(command.toStdString().c_str());
_statusLabel->setText("Interface started");
#endif
@ -182,7 +221,8 @@ void TestRunnerMobile::pullFolder() {
}
_statusLabel->setText("Pulling folder");
QString command = _adbInterface->getAdbCommand() + " pull " + _folderLineEdit->text() + " " + _workingFolder + _installerFilename;
QString command = _adbInterface->getAdbCommand() + " pull " + _folderLineEdit->text() + " " + _workingFolder;
appendLog(command);
system(command.toStdString().c_str());
_statusLabel->setText("Pull complete");
#endif

4
tools/nitpick/src/TestRunnerMobile.h
View file

@ -12,7 +12,6 @@
#define hifi_testRunnerMobile_h
#include <QMap>
#include <QLabel>
#include <QObject>
#include <QPushButton>
@ -31,8 +30,11 @@ public:
QPushButton* downloadAPKPushbutton,
QPushButton* installAPKPushbutton,
QPushButton* runInterfacePushbutton,
QCheckBox* usePreviousInstallationOnMobileCheckBox,
QCheckBox* runLatest,
QLineEdit* url,
QCheckBox* runFullSuite,
QLineEdit* scriptURL,
QLabel* statusLabel,
QObject* parent = 0

185
tools/nitpick/ui/Nitpick.ui
View file

@ -34,6 +34,9 @@
</property>
</widget>
<widget class="QTabWidget" name="tabWidget">
<property name="enabled">
<bool>true</bool>
</property>
<property name="geometry">
<rect>
<x>45</x>
@ -495,7 +498,7 @@
<widget class="QCheckBox" name="checkBoxServerless">
<property name="geometry">
<rect>
<x>20</x>
<x>240</x>
<y>70</y>
<width>120</width>
<height>20</height>
@ -549,13 +552,80 @@
</property>
<property name="geometry">
<rect>
<x>170</x>
<x>175</x>
<y>100</y>
<width>451</width>
<width>445</width>
<height>21</height>
</rect>
</property>
</widget>
<widget class="QLabel" name="workingFolderLabel_5">
<property name="geometry">
<rect>
<x>128</x>
<y>125</y>
<width>40</width>
<height>31</height>
</rect>
</property>
<property name="text">
<string>Script</string>
</property>
</widget>
<widget class="QLineEdit" name="scriptURLOnDesktopLineEdit">
<property name="enabled">
<bool>false</bool>
</property>
<property name="geometry">
<rect>
<x>175</x>
<y>130</y>
<width>445</width>
<height>21</height>
</rect>
</property>
</widget>
<widget class="QCheckBox" name="runFullSuiteOnDesktopCheckBox">
<property name="geometry">
<rect>
<x>20</x>
<y>130</y>
<width>120</width>
<height>20</height>
</rect>
</property>
<property name="toolTip">
<string>&lt;html&gt;&lt;head/&gt;&lt;body&gt;&lt;p&gt;If unchecked, will not show results during evaluation&lt;/p&gt;&lt;/body&gt;&lt;/html&gt;</string>
</property>
<property name="text">
<string>Run Full Suite</string>
</property>
<property name="checked">
<bool>true</bool>
</property>
</widget>
<widget class="QCheckBox" name="usePreviousInstallationOnDesktopCheckBox">
<property name="enabled">
<bool>true</bool>
</property>
<property name="geometry">
<rect>
<x>20</x>
<y>70</y>
<width>171</width>
<height>20</height>
</rect>
</property>
<property name="toolTip">
<string>&lt;html&gt;&lt;head/&gt;&lt;body&gt;&lt;p&gt;If unchecked, will not show results during evaluation&lt;/p&gt;&lt;/body&gt;&lt;/html&gt;</string>
</property>
<property name="text">
<string>usePreviousInstallation</string>
</property>
<property name="checked">
<bool>false</bool>
</property>
</widget>
</widget>
<widget class="QWidget" name="tab_5">
<attribute name="title">
@ -568,7 +638,7 @@
<property name="geometry">
<rect>
<x>10</x>
<y>90</y>
<y>150</y>
<width>160</width>
<height>30</height>
</rect>
@ -581,7 +651,7 @@
<property name="geometry">
<rect>
<x>190</x>
<y>96</y>
<y>156</y>
<width>320</width>
<height>30</height>
</rect>
@ -623,7 +693,7 @@
<property name="geometry">
<rect>
<x>460</x>
<y>410</y>
<y>440</y>
<width>160</width>
<height>30</height>
</rect>
@ -639,7 +709,7 @@
<property name="geometry">
<rect>
<x>10</x>
<y>410</y>
<y>440</y>
<width>440</width>
<height>30</height>
</rect>
@ -651,9 +721,9 @@
</property>
<property name="geometry">
<rect>
<x>170</x>
<y>170</y>
<width>451</width>
<x>175</x>
<y>245</y>
<width>445</width>
<height>21</height>
</rect>
</property>
@ -662,7 +732,7 @@
<property name="geometry">
<rect>
<x>20</x>
<y>170</y>
<y>245</y>
<width>120</width>
<height>20</height>
</rect>
@ -684,7 +754,7 @@
<property name="geometry">
<rect>
<x>10</x>
<y>210</y>
<y>100</y>
<width>160</width>
<height>30</height>
</rect>
@ -696,7 +766,7 @@
<widget class="QLabel" name="workingFolderLabel_4">
<property name="geometry">
<rect>
<x>300</x>
<x>20</x>
<y>60</y>
<width>41</width>
<height>31</height>
@ -709,7 +779,7 @@
<widget class="QLabel" name="statusLabelOnMobile">
<property name="geometry">
<rect>
<x>350</x>
<x>70</x>
<y>60</y>
<width>271</width>
<height>31</height>
@ -726,7 +796,7 @@
<property name="geometry">
<rect>
<x>10</x>
<y>250</y>
<y>325</y>
<width>160</width>
<height>30</height>
</rect>
@ -742,7 +812,7 @@
<property name="geometry">
<rect>
<x>10</x>
<y>300</y>
<y>375</y>
<width>160</width>
<height>30</height>
</rect>
@ -751,6 +821,86 @@
<string>Run Interface</string>
</property>
</widget>
<widget class="QLabel" name="workingFolderLabel_6">
<property name="geometry">
<rect>
<x>140</x>
<y>240</y>
<width>31</width>
<height>31</height>
</rect>
</property>
<property name="text">
<string>URL</string>
</property>
</widget>
<widget class="QLineEdit" name="scriptURLOnMobileLineEdit">
<property name="enabled">
<bool>false</bool>
</property>
<property name="geometry">
<rect>
<x>175</x>
<y>275</y>
<width>445</width>
<height>21</height>
</rect>
</property>
</widget>
<widget class="QLabel" name="workingFolderLabel_7">
<property name="geometry">
<rect>
<x>140</x>
<y>270</y>
<width>40</width>
<height>31</height>
</rect>
</property>
<property name="text">
<string>Script</string>
</property>
</widget>
<widget class="QCheckBox" name="runFullSuiteOnMobileCheckBox">
<property name="geometry">
<rect>
<x>20</x>
<y>275</y>
<width>120</width>
<height>20</height>
</rect>
</property>
<property name="toolTip">
<string>&lt;html&gt;&lt;head/&gt;&lt;body&gt;&lt;p&gt;If unchecked, will not show results during evaluation&lt;/p&gt;&lt;/body&gt;&lt;/html&gt;</string>
</property>
<property name="text">
<string>Run Full Suite</string>
</property>
<property name="checked">
<bool>true</bool>
</property>
</widget>
<widget class="QCheckBox" name="usePreviousInstallationOnMobileCheckBox">
<property name="enabled">
<bool>true</bool>
</property>
<property name="geometry">
<rect>
<x>20</x>
<y>210</y>
<width>171</width>
<height>20</height>
</rect>
</property>
<property name="toolTip">
<string>&lt;html&gt;&lt;head/&gt;&lt;body&gt;&lt;p&gt;If unchecked, will not show results during evaluation&lt;/p&gt;&lt;/body&gt;&lt;/html&gt;</string>
</property>
<property name="text">
<string>usePreviousInstallation</string>
</property>
<property name="checked">
<bool>false</bool>
</property>
</widget>
</widget>
<widget class="QWidget" name="tab_2">
<attribute name="title">
@ -921,6 +1071,9 @@
<height>21</height>
</rect>
</property>
<property name="readOnly">
<bool>true</bool>
</property>
</widget>
</widget>
<zorder>groupBox</zorder>