Merge branch 'addDailyTests' of https://github.com/NissimHadar/hifi into addDailyTests

This commit is contained in:
NissimHadar 2018-09-12 22:22:49 -07:00
commit cc9196fc26
5 changed files with 558 additions and 65 deletions

View file

@ -16,17 +16,65 @@
#include "ui/AutoTester.h"
extern AutoTester* autoTester;
TestRunner::TestRunner(QObject* parent) : QObject(parent) {
#ifdef Q_OS_WIN
#include <windows.h>
#include <tlhelp32.h>
#endif
TestRunner::TestRunner(std::vector<QCheckBox*> dayCheckboxes,
std::vector<QCheckBox*> timeEditCheckboxes,
std::vector<QTimeEdit*> timeEdits,
QLabel* workingFolderLabel,
QObject* parent) :
QObject(parent)
{
_dayCheckboxes = dayCheckboxes;
_timeEditCheckboxes = timeEditCheckboxes;
_timeEdits = timeEdits;
_workingFolderLabel = workingFolderLabel;
}
TestRunner::~TestRunner() {
disconnect(_timer, SIGNAL(timeout()), this, SLOT(checkTime()));
}
void TestRunner::setWorkingFolder() {
// Everything will be written to this folder
QString previousSelection = _workingFolder;
QString parent = previousSelection.left(previousSelection.lastIndexOf('/'));
if (!parent.isNull() && parent.right(1) != "/") {
parent += "/";
}
_workingFolder = QFileDialog::getExistingDirectory(nullptr, "Please select a temporary folder for installation", parent,
QFileDialog::ShowDirsOnly);
// If user canceled then restore previous selection and return
if (_workingFolder == "") {
_workingFolder = previousSelection;
return;
}
_installationFolder = _workingFolder + "/High Fidelity";
_logFile.setFileName(_workingFolder + "/log.txt");
autoTester->enableRunTabControls();
_workingFolderLabel->setText(QDir::toNativeSeparators(_workingFolder));
// The time is checked every 30 seconds for automatic test start
_timer = new QTimer(this);
connect(_timer, SIGNAL(timeout()), this, SLOT(checkTime()));
_timer->start(30 * 1000); //time specified in ms
}
void TestRunner::run() {
_testStartDateTime = QDateTime::currentDateTime();
_automatedTestIsRunning = true;
// Initial setup
_branch = autoTester->getSelectedBranch();
_user = autoTester->getSelectedUser();
// Everything will be written to this folder
selectTemporaryFolder();
// This will be restored at the end of the tests
saveExistingHighFidelityAppDataFolder();
@ -37,12 +85,20 @@ void TestRunner::run() {
QStringList filenames;
filenames << INSTALLER_FILENAME << BUILD_XML_FILENAME;
autoTester->downloadFiles(urls, _tempFolder, filenames, (void*)this);
updateStatusLabel("Downloading installer");
autoTester->downloadFiles(urls, _workingFolder, filenames, (void*)this);
// `installerDownloadComplete` will run after download has completed
}
void TestRunner::installerDownloadComplete() {
appendLog(QString("Test started at ") + QString::number(_testStartDateTime.time().hour()) + ":" +
QString("%1").arg(_testStartDateTime.time().minute(), 2, 10, QChar('0')) + ", on " +
_testStartDateTime.date().toString("ddd, MMM d, yyyy"));
updateStatusLabel("Installing");
// Kill any existing processes that would interfere with installation
killProcesses();
@ -50,6 +106,8 @@ void TestRunner::installerDownloadComplete() {
createSnapshotFolder();
updateStatusLabel("Running tests");
startLocalServerProcesses();
runInterfaceWithTestScript();
killProcesses();
@ -64,7 +122,7 @@ void TestRunner::runInstaller() {
// To allow installation, the installer is run using the `system` command
QStringList arguments{ QStringList() << QString("/S") << QString("/D=") + QDir::toNativeSeparators(_installationFolder) };
QString installerFullPath = _tempFolder + "/" + INSTALLER_FILENAME;
QString installerFullPath = _workingFolder + "/" + INSTALLER_FILENAME;
QString commandLine =
QDir::toNativeSeparators(installerFullPath) + " /S /D=" + QDir::toNativeSeparators(_installationFolder);
@ -91,35 +149,8 @@ void TestRunner::saveExistingHighFidelityAppDataFolder() {
copyFolder(QDir::currentPath() + "/AppDataHighFidelity", _appDataFolder.path());
}
void TestRunner::restoreHighFidelityAppDataFolder() {
_appDataFolder.removeRecursively();
if (_savedAppDataFolder != QDir()) {
_appDataFolder.rename(_savedAppDataFolder.path(), _appDataFolder.path());
}
}
void TestRunner::selectTemporaryFolder() {
QString previousSelection = _tempFolder;
QString parent = previousSelection.left(previousSelection.lastIndexOf('/'));
if (!parent.isNull() && parent.right(1) != "/") {
parent += "/";
}
_tempFolder = QFileDialog::getExistingDirectory(nullptr, "Please select a temporary folder for installation", parent,
QFileDialog::ShowDirsOnly);
// If user canceled then restore previous selection and return
if (_tempFolder == "") {
_tempFolder = previousSelection;
return;
}
_installationFolder = _tempFolder + "/High Fidelity";
}
void TestRunner::createSnapshotFolder() {
_snapshotFolder = _tempFolder + "/" + SNAPSHOT_FOLDER_NAME;
_snapshotFolder = _workingFolder + "/" + SNAPSHOT_FOLDER_NAME;
// Just delete all PNGs from the folder if it already exists
if (QDir(_snapshotFolder).exists()) {
@ -138,15 +169,51 @@ void TestRunner::createSnapshotFolder() {
}
void TestRunner::killProcesses() {
killProcessByName("assignment-client.exe");
killProcessByName("domain-server.exe");
killProcessByName("server-console.exe");
}
void TestRunner::killProcessByName(QString processName) {
#ifdef Q_OS_WIN
QString commandLine = "taskkill /im " + processName + " /f >nul";
system(commandLine.toStdString().c_str());
try {
QStringList processesToKill = QStringList() << "interface.exe"
<< "assignment-client.exe"
<< "domain-server.exe"
<< "server-console.exe";
// Loop until all pending processes to kill have actually died
QStringList pendingProcessesToKill;
do {
pendingProcessesToKill.clear();
// Get list of running tasks
HANDLE processSnapHandle = CreateToolhelp32Snapshot(TH32CS_SNAPPROCESS, 0);
if (processSnapHandle == INVALID_HANDLE_VALUE) {
throw("Process snapshot creation failure");
}
PROCESSENTRY32 processEntry32;
processEntry32.dwSize = sizeof(PROCESSENTRY32);
if (!Process32First(processSnapHandle, &processEntry32)) {
CloseHandle(processSnapHandle);
throw("Process32First failed");
}
// Kill any task in the list
do {
foreach (QString process, processesToKill)
if (QString(processEntry32.szExeFile) == process) {
QString commandLine = "taskkill /im " + process + " /f >nul";
system(commandLine.toStdString().c_str());
pendingProcessesToKill << process;
}
} while (Process32Next(processSnapHandle, &processEntry32));
QThread::sleep(2);
} while (!pendingProcessesToKill.isEmpty());
} catch (QString errorMessage) {
QMessageBox::critical(0, "Internal error: " + QString(__FILE__) + ":" + QString::number(__LINE__), errorMessage);
exit(-1);
} catch (...) {
QMessageBox::critical(0, "Internal error: " + QString(__FILE__) + ":" + QString::number(__LINE__), "unknown error");
exit(-1);
}
#endif
}
@ -162,7 +229,7 @@ void TestRunner::startLocalServerProcesses() {
system(commandLine.toStdString().c_str());
#endif
// Give server processes time to stabilize
QThread::sleep(8);
QThread::sleep(12);
}
void TestRunner::runInterfaceWithTestScript() {
@ -177,18 +244,22 @@ void TestRunner::runInterfaceWithTestScript() {
}
void TestRunner::evaluateResults() {
updateStatusLabel("Evaluating results");
autoTester->startTestsEvaluation(false, true, _snapshotFolder, _branch, _user);
}
void TestRunner::automaticTestRunEvaluationComplete(QString zippedFolder) {
addBuildNumberToResults(zippedFolder);
restoreHighFidelityAppDataFolder();
updateStatusLabel("Testing complete");
_automatedTestIsRunning = false;
}
void TestRunner::addBuildNumberToResults(QString zippedFolderName) {
try {
QDomDocument domDocument;
QString filename{ _tempFolder + "/" + BUILD_XML_FILENAME };
QString filename{ _workingFolder + "/" + BUILD_XML_FILENAME };
QFile file(filename);
if (!file.open(QIODevice::ReadOnly) || !domDocument.setContent(&file)) {
throw QString("Could not open " + filename);
@ -239,7 +310,7 @@ void TestRunner::addBuildNumberToResults(QString zippedFolderName) {
// Add the build number to the end of the filename
QString build = element.text();
QStringList filenameParts = zippedFolderName.split(".");
QString augmentedFilename = filenameParts[0] + "_" + build + "." + filenameParts[1];
QString augmentedFilename = filenameParts[0] + "(" + build + ")." + filenameParts[1];
QFile::rename(zippedFolderName, augmentedFilename);
}
@ -252,6 +323,14 @@ void TestRunner::addBuildNumberToResults(QString zippedFolderName) {
}
}
void TestRunner::restoreHighFidelityAppDataFolder() {
_appDataFolder.removeRecursively();
if (_savedAppDataFolder != QDir()) {
_appDataFolder.rename(_savedAppDataFolder.path(), _appDataFolder.path());
}
}
// Copies a folder recursively
void TestRunner::copyFolder(const QString& source, const QString& destination) {
try {
@ -283,3 +362,56 @@ void TestRunner::copyFolder(const QString& source, const QString& destination) {
exit(-1);
}
}
void TestRunner::checkTime() {
// No processing is done if a test is running
if (_automatedTestIsRunning) {
return;
}
QDateTime now = QDateTime::currentDateTime();
// Check day of week
if (!_dayCheckboxes.at(now.date().dayOfWeek() - 1)->isChecked()) {
return;
}
// Check the time
bool timeToRun{ false };
QTime time = now.time();
int h = time.hour();
int m = time.minute();
for (int i = 0; i < std::min(_timeEditCheckboxes.size(), _timeEdits.size()); ++i) {
bool is = _timeEditCheckboxes[i]->isChecked();
int hh = _timeEdits[i]->time().hour();
int mm = _timeEdits[i]->time().minute();
if (_timeEditCheckboxes[i]->isChecked() && (_timeEdits[i]->time().hour() == now.time().hour()) &&
(_timeEdits[i]->time().minute() == now.time().minute())) {
timeToRun = true;
break;
}
}
if (timeToRun) {
run();
}
}
void TestRunner::updateStatusLabel(const QString& message) {
autoTester->updateStatusLabel(message);
}
void TestRunner::appendLog(const QString& message) {
if (!_logFile.open(QIODevice::Append | QIODevice::Text)) {
QMessageBox::critical(0, "Internal error: " + QString(__FILE__) + ":" + QString::number(__LINE__),
"Could not open the log file");
exit(-1);
}
_logFile.write(message.toStdString().c_str());
_logFile.write("\n");
_logFile.close();
autoTester->appendLogWindow(message);
}

View file

@ -11,27 +11,38 @@
#ifndef hifi_testRunner_h
#define hifi_testRunner_h
#include <QObject>
#include <QCheckBox>
#include <QDir>
#include <QLabel>
#include <QObject>
#include <QProcess>
#include <QTimeEdit>
#include <QTimer>
#include "Downloader.h"
class TestRunner : public QObject {
Q_OBJECT
public:
explicit TestRunner(QObject* parent = 0);
explicit TestRunner(std::vector<QCheckBox*> dayCheckboxes,
std::vector<QCheckBox*> timeEditCheckboxes,
std::vector<QTimeEdit*> timeEdits,
QLabel* workingFolderLabel,
QObject* parent = 0);
~TestRunner();
void setWorkingFolder();
void run();
void installerDownloadComplete();
void runInstaller();
void saveExistingHighFidelityAppDataFolder();
void restoreHighFidelityAppDataFolder();
void selectTemporaryFolder();
void createSnapshotFolder();
void killProcesses();
void killProcessByName(QString processName);
void startLocalServerProcesses();
void runInterfaceWithTestScript();
void evaluateResults();
@ -40,11 +51,19 @@ public:
void copyFolder(const QString& source, const QString& destination);
void updateStatusLabel(const QString& message);
void appendLog(const QString& message);
private slots:
void checkTime();
private:
bool _automatedTestIsRunning{ false };
QDir _appDataFolder;
QDir _savedAppDataFolder;
QString _tempFolder;
QString _workingFolder;
QString _snapshotFolder;
QString _installationFolder;
@ -62,6 +81,17 @@ private:
QString _branch;
QString _user;
std::vector<QCheckBox*> _dayCheckboxes;
std::vector<QCheckBox*> _timeEditCheckboxes;
std::vector<QTimeEdit*> _timeEdits;
QLabel* _workingFolderLabel;
QTimer* _timer;
QFile _logFile;
QDateTime _testStartDateTime;
};
#endif // hifi_testRunner_h

View file

@ -32,20 +32,47 @@ AutoTester::AutoTester(QWidget* parent) : QMainWindow(parent) {
#ifndef Q_OS_WIN
_ui.tabWidget->removeTab(1);
#endif
//// Coming soon...
_ui.statusLabel->setText("");
_ui.plainTextEdit->setReadOnly(true);
// Coming soon to an auto-tester near you...
//// _helpWindow.textBrowser->setText()
}
void AutoTester::setup() {
_test = new Test(_ui.progressBar, _ui.checkBoxInteractiveMode);
_testRunner = new TestRunner();
std::vector<QCheckBox*> dayCheckboxes;
dayCheckboxes.emplace_back(_ui.mondayCheckBox);
dayCheckboxes.emplace_back(_ui.tuesdayCheckBox);
dayCheckboxes.emplace_back(_ui.wednesdayCheckBox);
dayCheckboxes.emplace_back(_ui.thursdayCheckBox);
dayCheckboxes.emplace_back(_ui.fridayCheckBox);
dayCheckboxes.emplace_back(_ui.saturdayCheckBox);
dayCheckboxes.emplace_back(_ui.sundayCheckBox);
std::vector<QCheckBox*> timeEditCheckboxes;
timeEditCheckboxes.emplace_back(_ui.timeEdit1checkBox);
timeEditCheckboxes.emplace_back(_ui.timeEdit2checkBox);
timeEditCheckboxes.emplace_back(_ui.timeEdit3checkBox);
timeEditCheckboxes.emplace_back(_ui.timeEdit4checkBox);
std::vector<QTimeEdit*> timeEdits;
timeEdits.emplace_back(_ui.timeEdit1);
timeEdits.emplace_back(_ui.timeEdit2);
timeEdits.emplace_back(_ui.timeEdit3);
timeEdits.emplace_back(_ui.timeEdit4);
_testRunner = new TestRunner(dayCheckboxes, timeEditCheckboxes, timeEdits, _ui.workingFolderLabel);
}
void AutoTester::startTestsEvaluation(const bool isRunningFromCommandLine,
const bool isRunningInAutomaticTestRun,
const QString& snapshotDirectory,
const QString& branch,
const QString& user) {
const QString& user
) {
_test->startTestsEvaluation(isRunningFromCommandLine, isRunningInAutomaticTestRun, snapshotDirectory, branch, user);
}
@ -103,6 +130,16 @@ void AutoTester::on_createTestRailRunButton_clicked() {
_test->createTestRailRun();
}
void AutoTester::on_setWorkingFolderButton_clicked() {
_testRunner->setWorkingFolder();
}
void AutoTester::enableRunTabControls() {
_ui.runNowButton->setEnabled(true);
_ui.daysGroupBox->setEnabled(true);
_ui.timesGroupBox->setEnabled(true);
}
void AutoTester::on_runNowButton_clicked() {
_testRunner->run();
}
@ -231,3 +268,11 @@ void AutoTester::setBranchText(const QString& branch) {
QString AutoTester::getSelectedBranch() {
return _ui.branchTextEdit->toPlainText();
}
void AutoTester::updateStatusLabel(const QString& status) {
_ui.statusLabel->setText(status);
}
void AutoTester::appendLogWindow(const QString& message) {
_ui.plainTextEdit->appendPlainText(message);
}

View file

@ -47,6 +47,11 @@ public:
void setBranchText(const QString& branch);
QString getSelectedBranch();
void enableRunTabControls();
void updateStatusLabel(const QString& status);
void appendLogWindow(const QString& message);
private slots:
void on_tabWidget_currentChanged(int index);
@ -66,6 +71,7 @@ private slots:
void on_createTestRailTestCasesButton_clicked();
void on_createTestRailRunButton_clicked();
void on_setWorkingFolderButton_clicked();
void on_runNowButton_clicked();
void on_updateTestRailRunResultsButton_clicked();

View file

@ -6,8 +6,8 @@
<rect>
<x>0</x>
<y>0</y>
<width>582</width>
<height>734</height>
<width>707</width>
<height>796</height>
</rect>
</property>
<property name="sizePolicy">
@ -23,8 +23,8 @@
<widget class="QPushButton" name="closeButton">
<property name="geometry">
<rect>
<x>430</x>
<y>620</y>
<x>470</x>
<y>660</y>
<width>100</width>
<height>40</height>
</rect>
@ -38,12 +38,12 @@
<rect>
<x>30</x>
<y>140</y>
<width>521</width>
<height>461</height>
<width>631</width>
<height>501</height>
</rect>
</property>
<property name="currentIndex">
<number>1</number>
<number>2</number>
</property>
<widget class="QWidget" name="tab_1">
<attribute name="title">
@ -190,11 +190,14 @@
<string>Run</string>
</attribute>
<widget class="QPushButton" name="runNowButton">
<property name="enabled">
<bool>false</bool>
</property>
<property name="geometry">
<rect>
<x>200</x>
<y>200</y>
<width>93</width>
<x>10</x>
<y>70</y>
<width>161</width>
<height>28</height>
</rect>
</property>
@ -202,6 +205,283 @@
<string>Run now</string>
</property>
</widget>
<widget class="QGroupBox" name="daysGroupBox">
<property name="enabled">
<bool>false</bool>
</property>
<property name="geometry">
<rect>
<x>20</x>
<y>150</y>
<width>91</width>
<height>241</height>
</rect>
</property>
<property name="title">
<string>Days</string>
</property>
<widget class="QCheckBox" name="sundayCheckBox">
<property name="geometry">
<rect>
<x>10</x>
<y>210</y>
<width>80</width>
<height>17</height>
</rect>
</property>
<property name="text">
<string>Sunday</string>
</property>
</widget>
<widget class="QCheckBox" name="wednesdayCheckBox">
<property name="geometry">
<rect>
<x>10</x>
<y>90</y>
<width>80</width>
<height>17</height>
</rect>
</property>
<property name="text">
<string>Wednesday</string>
</property>
</widget>
<widget class="QCheckBox" name="tuesdayCheckBox">
<property name="geometry">
<rect>
<x>10</x>
<y>60</y>
<width>80</width>
<height>17</height>
</rect>
</property>
<property name="text">
<string>Tuesday</string>
</property>
</widget>
<widget class="QCheckBox" name="thursdayCheckBox">
<property name="geometry">
<rect>
<x>10</x>
<y>120</y>
<width>80</width>
<height>17</height>
</rect>
</property>
<property name="text">
<string>Thursday</string>
</property>
</widget>
<widget class="QCheckBox" name="fridayCheckBox">
<property name="geometry">
<rect>
<x>10</x>
<y>150</y>
<width>80</width>
<height>17</height>
</rect>
</property>
<property name="text">
<string>Friday</string>
</property>
</widget>
<widget class="QCheckBox" name="saturdayCheckBox">
<property name="geometry">
<rect>
<x>10</x>
<y>180</y>
<width>80</width>
<height>17</height>
</rect>
</property>
<property name="text">
<string>Saturday</string>
</property>
</widget>
<widget class="QCheckBox" name="mondayCheckBox">
<property name="geometry">
<rect>
<x>10</x>
<y>30</y>
<width>80</width>
<height>17</height>
</rect>
</property>
<property name="text">
<string>Monday</string>
</property>
</widget>
</widget>
<widget class="QGroupBox" name="timesGroupBox">
<property name="enabled">
<bool>false</bool>
</property>
<property name="geometry">
<rect>
<x>130</x>
<y>150</y>
<width>161</width>
<height>191</height>
</rect>
</property>
<property name="title">
<string>Times</string>
</property>
<widget class="QTimeEdit" name="timeEdit1">
<property name="geometry">
<rect>
<x>30</x>
<y>20</y>
<width>118</width>
<height>22</height>
</rect>
</property>
</widget>
<widget class="QTimeEdit" name="timeEdit2">
<property name="geometry">
<rect>
<x>30</x>
<y>60</y>
<width>118</width>
<height>22</height>
</rect>
</property>
</widget>
<widget class="QTimeEdit" name="timeEdit3">
<property name="geometry">
<rect>
<x>30</x>
<y>100</y>
<width>118</width>
<height>22</height>
</rect>
</property>
</widget>
<widget class="QTimeEdit" name="timeEdit4">
<property name="geometry">
<rect>
<x>30</x>
<y>140</y>
<width>118</width>
<height>22</height>
</rect>
</property>
</widget>
<widget class="QCheckBox" name="timeEdit1checkBox">
<property name="geometry">
<rect>
<x>10</x>
<y>23</y>
<width>21</width>
<height>17</height>
</rect>
</property>
<property name="text">
<string/>
</property>
</widget>
<widget class="QCheckBox" name="timeEdit2checkBox">
<property name="geometry">
<rect>
<x>10</x>
<y>63</y>
<width>21</width>
<height>17</height>
</rect>
</property>
<property name="text">
<string/>
</property>
</widget>
<widget class="QCheckBox" name="timeEdit3checkBox">
<property name="geometry">
<rect>
<x>10</x>
<y>103</y>
<width>21</width>
<height>17</height>
</rect>
</property>
<property name="text">
<string/>
</property>
</widget>
<widget class="QCheckBox" name="timeEdit4checkBox">
<property name="geometry">
<rect>
<x>10</x>
<y>143</y>
<width>21</width>
<height>17</height>
</rect>
</property>
<property name="text">
<string/>
</property>
</widget>
</widget>
<widget class="QPushButton" name="setWorkingFolderButton">
<property name="geometry">
<rect>
<x>10</x>
<y>20</y>
<width>161</width>
<height>28</height>
</rect>
</property>
<property name="text">
<string>Set Working Folder</string>
</property>
</widget>
<widget class="QLabel" name="workingFolderLabel">
<property name="geometry">
<rect>
<x>190</x>
<y>20</y>
<width>321</width>
<height>31</height>
</rect>
</property>
<property name="text">
<string>#######</string>
</property>
</widget>
<widget class="QPlainTextEdit" name="plainTextEdit">
<property name="geometry">
<rect>
<x>300</x>
<y>120</y>
<width>311</width>
<height>331</height>
</rect>
</property>
</widget>
<widget class="QLabel" name="workingFolderLabel_2">
<property name="geometry">
<rect>
<x>300</x>
<y>80</y>
<width>41</width>
<height>31</height>
</rect>
</property>
<property name="text">
<string>Status:</string>
</property>
</widget>
<widget class="QLabel" name="statusLabel">
<property name="geometry">
<rect>
<x>350</x>
<y>80</y>
<width>271</width>
<height>31</height>
</rect>
</property>
<property name="text">
<string>#######</string>
</property>
</widget>
</widget>
<widget class="QWidget" name="tab_2">
<attribute name="title">
@ -371,7 +651,7 @@
<property name="geometry">
<rect>
<x>80</x>
<y>630</y>
<y>670</y>
<width>255</width>
<height>23</height>
</rect>
@ -386,7 +666,7 @@
<rect>
<x>0</x>
<y>0</y>
<width>582</width>
<width>707</width>
<height>21</height>
</rect>
</property>