diff --git a/tools/auto-tester/src/TestRunner.cpp b/tools/auto-tester/src/TestRunner.cpp index 5c7bb97055..dddc30e8a8 100644 --- a/tools/auto-tester/src/TestRunner.cpp +++ b/tools/auto-tester/src/TestRunner.cpp @@ -16,17 +16,65 @@ #include "ui/AutoTester.h" extern AutoTester* autoTester; -TestRunner::TestRunner(QObject* parent) : QObject(parent) { +#ifdef Q_OS_WIN +#include +#include +#endif + +TestRunner::TestRunner(std::vector dayCheckboxes, + std::vector timeEditCheckboxes, + std::vector timeEdits, + QLabel* workingFolderLabel, + QObject* parent) : + QObject(parent) +{ + _dayCheckboxes = dayCheckboxes; + _timeEditCheckboxes = timeEditCheckboxes; + _timeEdits = timeEdits; + _workingFolderLabel = workingFolderLabel; +} + +TestRunner::~TestRunner() { + disconnect(_timer, SIGNAL(timeout()), this, SLOT(checkTime())); +} + +void TestRunner::setWorkingFolder() { + // Everything will be written to this folder + QString previousSelection = _workingFolder; + QString parent = previousSelection.left(previousSelection.lastIndexOf('/')); + if (!parent.isNull() && parent.right(1) != "/") { + parent += "/"; + } + + _workingFolder = QFileDialog::getExistingDirectory(nullptr, "Please select a temporary folder for installation", parent, + QFileDialog::ShowDirsOnly); + + // If user canceled then restore previous selection and return + if (_workingFolder == "") { + _workingFolder = previousSelection; + return; + } + + _installationFolder = _workingFolder + "/High Fidelity"; + _logFile.setFileName(_workingFolder + "/log.txt"); + + autoTester->enableRunTabControls(); + _workingFolderLabel->setText(QDir::toNativeSeparators(_workingFolder)); + + // The time is checked every 30 seconds for automatic test start + _timer = new QTimer(this); + connect(_timer, SIGNAL(timeout()), this, SLOT(checkTime())); + _timer->start(30 * 1000); //time specified in ms } void TestRunner::run() { + _testStartDateTime = QDateTime::currentDateTime(); + _automatedTestIsRunning = true; + // Initial setup _branch = autoTester->getSelectedBranch(); _user = autoTester->getSelectedUser(); - // Everything will be written to this folder - selectTemporaryFolder(); - // This will be restored at the end of the tests saveExistingHighFidelityAppDataFolder(); @@ -37,12 +85,20 @@ void TestRunner::run() { QStringList filenames; filenames << INSTALLER_FILENAME << BUILD_XML_FILENAME; - autoTester->downloadFiles(urls, _tempFolder, filenames, (void*)this); + updateStatusLabel("Downloading installer"); + + autoTester->downloadFiles(urls, _workingFolder, filenames, (void*)this); // `installerDownloadComplete` will run after download has completed } void TestRunner::installerDownloadComplete() { + appendLog(QString("Test started at ") + QString::number(_testStartDateTime.time().hour()) + ":" + + QString("%1").arg(_testStartDateTime.time().minute(), 2, 10, QChar('0')) + ", on " + + _testStartDateTime.date().toString("ddd, MMM d, yyyy")); + + updateStatusLabel("Installing"); + // Kill any existing processes that would interfere with installation killProcesses(); @@ -50,6 +106,8 @@ void TestRunner::installerDownloadComplete() { createSnapshotFolder(); + updateStatusLabel("Running tests"); + startLocalServerProcesses(); runInterfaceWithTestScript(); killProcesses(); @@ -64,7 +122,7 @@ void TestRunner::runInstaller() { // To allow installation, the installer is run using the `system` command QStringList arguments{ QStringList() << QString("/S") << QString("/D=") + QDir::toNativeSeparators(_installationFolder) }; - QString installerFullPath = _tempFolder + "/" + INSTALLER_FILENAME; + QString installerFullPath = _workingFolder + "/" + INSTALLER_FILENAME; QString commandLine = QDir::toNativeSeparators(installerFullPath) + " /S /D=" + QDir::toNativeSeparators(_installationFolder); @@ -91,35 +149,8 @@ void TestRunner::saveExistingHighFidelityAppDataFolder() { copyFolder(QDir::currentPath() + "/AppDataHighFidelity", _appDataFolder.path()); } -void TestRunner::restoreHighFidelityAppDataFolder() { - _appDataFolder.removeRecursively(); - - if (_savedAppDataFolder != QDir()) { - _appDataFolder.rename(_savedAppDataFolder.path(), _appDataFolder.path()); - } -} - -void TestRunner::selectTemporaryFolder() { - QString previousSelection = _tempFolder; - QString parent = previousSelection.left(previousSelection.lastIndexOf('/')); - if (!parent.isNull() && parent.right(1) != "/") { - parent += "/"; - } - - _tempFolder = QFileDialog::getExistingDirectory(nullptr, "Please select a temporary folder for installation", parent, - QFileDialog::ShowDirsOnly); - - // If user canceled then restore previous selection and return - if (_tempFolder == "") { - _tempFolder = previousSelection; - return; - } - - _installationFolder = _tempFolder + "/High Fidelity"; -} - void TestRunner::createSnapshotFolder() { - _snapshotFolder = _tempFolder + "/" + SNAPSHOT_FOLDER_NAME; + _snapshotFolder = _workingFolder + "/" + SNAPSHOT_FOLDER_NAME; // Just delete all PNGs from the folder if it already exists if (QDir(_snapshotFolder).exists()) { @@ -138,15 +169,51 @@ void TestRunner::createSnapshotFolder() { } void TestRunner::killProcesses() { - killProcessByName("assignment-client.exe"); - killProcessByName("domain-server.exe"); - killProcessByName("server-console.exe"); -} - -void TestRunner::killProcessByName(QString processName) { #ifdef Q_OS_WIN - QString commandLine = "taskkill /im " + processName + " /f >nul"; - system(commandLine.toStdString().c_str()); + try { + QStringList processesToKill = QStringList() << "interface.exe" + << "assignment-client.exe" + << "domain-server.exe" + << "server-console.exe"; + + // Loop until all pending processes to kill have actually died + QStringList pendingProcessesToKill; + do { + pendingProcessesToKill.clear(); + + // Get list of running tasks + HANDLE processSnapHandle = CreateToolhelp32Snapshot(TH32CS_SNAPPROCESS, 0); + if (processSnapHandle == INVALID_HANDLE_VALUE) { + throw("Process snapshot creation failure"); + } + + PROCESSENTRY32 processEntry32; + processEntry32.dwSize = sizeof(PROCESSENTRY32); + if (!Process32First(processSnapHandle, &processEntry32)) { + CloseHandle(processSnapHandle); + throw("Process32First failed"); + } + + // Kill any task in the list + do { + foreach (QString process, processesToKill) + if (QString(processEntry32.szExeFile) == process) { + QString commandLine = "taskkill /im " + process + " /f >nul"; + system(commandLine.toStdString().c_str()); + pendingProcessesToKill << process; + } + } while (Process32Next(processSnapHandle, &processEntry32)); + + QThread::sleep(2); + } while (!pendingProcessesToKill.isEmpty()); + + } catch (QString errorMessage) { + QMessageBox::critical(0, "Internal error: " + QString(__FILE__) + ":" + QString::number(__LINE__), errorMessage); + exit(-1); + } catch (...) { + QMessageBox::critical(0, "Internal error: " + QString(__FILE__) + ":" + QString::number(__LINE__), "unknown error"); + exit(-1); + } #endif } @@ -162,7 +229,7 @@ void TestRunner::startLocalServerProcesses() { system(commandLine.toStdString().c_str()); #endif // Give server processes time to stabilize - QThread::sleep(8); + QThread::sleep(12); } void TestRunner::runInterfaceWithTestScript() { @@ -177,18 +244,22 @@ void TestRunner::runInterfaceWithTestScript() { } void TestRunner::evaluateResults() { + updateStatusLabel("Evaluating results"); autoTester->startTestsEvaluation(false, true, _snapshotFolder, _branch, _user); } void TestRunner::automaticTestRunEvaluationComplete(QString zippedFolder) { addBuildNumberToResults(zippedFolder); restoreHighFidelityAppDataFolder(); + + updateStatusLabel("Testing complete"); + _automatedTestIsRunning = false; } void TestRunner::addBuildNumberToResults(QString zippedFolderName) { try { QDomDocument domDocument; - QString filename{ _tempFolder + "/" + BUILD_XML_FILENAME }; + QString filename{ _workingFolder + "/" + BUILD_XML_FILENAME }; QFile file(filename); if (!file.open(QIODevice::ReadOnly) || !domDocument.setContent(&file)) { throw QString("Could not open " + filename); @@ -239,7 +310,7 @@ void TestRunner::addBuildNumberToResults(QString zippedFolderName) { // Add the build number to the end of the filename QString build = element.text(); QStringList filenameParts = zippedFolderName.split("."); - QString augmentedFilename = filenameParts[0] + "_" + build + "." + filenameParts[1]; + QString augmentedFilename = filenameParts[0] + "(" + build + ")." + filenameParts[1]; QFile::rename(zippedFolderName, augmentedFilename); } @@ -252,6 +323,14 @@ void TestRunner::addBuildNumberToResults(QString zippedFolderName) { } } +void TestRunner::restoreHighFidelityAppDataFolder() { + _appDataFolder.removeRecursively(); + + if (_savedAppDataFolder != QDir()) { + _appDataFolder.rename(_savedAppDataFolder.path(), _appDataFolder.path()); + } +} + // Copies a folder recursively void TestRunner::copyFolder(const QString& source, const QString& destination) { try { @@ -283,3 +362,56 @@ void TestRunner::copyFolder(const QString& source, const QString& destination) { exit(-1); } } + +void TestRunner::checkTime() { + // No processing is done if a test is running + if (_automatedTestIsRunning) { + return; + } + + QDateTime now = QDateTime::currentDateTime(); + + // Check day of week + if (!_dayCheckboxes.at(now.date().dayOfWeek() - 1)->isChecked()) { + return; + } + + // Check the time + bool timeToRun{ false }; + QTime time = now.time(); + int h = time.hour(); + int m = time.minute(); + + for (int i = 0; i < std::min(_timeEditCheckboxes.size(), _timeEdits.size()); ++i) { + bool is = _timeEditCheckboxes[i]->isChecked(); + int hh = _timeEdits[i]->time().hour(); + int mm = _timeEdits[i]->time().minute(); + if (_timeEditCheckboxes[i]->isChecked() && (_timeEdits[i]->time().hour() == now.time().hour()) && + (_timeEdits[i]->time().minute() == now.time().minute())) { + timeToRun = true; + break; + } + } + + if (timeToRun) { + run(); + } +} + +void TestRunner::updateStatusLabel(const QString& message) { + autoTester->updateStatusLabel(message); +} + +void TestRunner::appendLog(const QString& message) { + if (!_logFile.open(QIODevice::Append | QIODevice::Text)) { + QMessageBox::critical(0, "Internal error: " + QString(__FILE__) + ":" + QString::number(__LINE__), + "Could not open the log file"); + exit(-1); + } + + _logFile.write(message.toStdString().c_str()); + _logFile.write("\n"); + _logFile.close(); + + autoTester->appendLogWindow(message); +} \ No newline at end of file diff --git a/tools/auto-tester/src/TestRunner.h b/tools/auto-tester/src/TestRunner.h index 5a5cbf1cf3..e13b0be070 100644 --- a/tools/auto-tester/src/TestRunner.h +++ b/tools/auto-tester/src/TestRunner.h @@ -11,27 +11,38 @@ #ifndef hifi_testRunner_h #define hifi_testRunner_h -#include +#include #include +#include +#include #include +#include +#include #include "Downloader.h" class TestRunner : public QObject { Q_OBJECT public: - explicit TestRunner(QObject* parent = 0); + explicit TestRunner(std::vector dayCheckboxes, + std::vector timeEditCheckboxes, + std::vector timeEdits, + QLabel* workingFolderLabel, + QObject* parent = 0); + ~TestRunner(); + + void setWorkingFolder(); void run(); + void installerDownloadComplete(); void runInstaller(); void saveExistingHighFidelityAppDataFolder(); void restoreHighFidelityAppDataFolder(); - void selectTemporaryFolder(); + void createSnapshotFolder(); void killProcesses(); - void killProcessByName(QString processName); void startLocalServerProcesses(); void runInterfaceWithTestScript(); void evaluateResults(); @@ -40,11 +51,19 @@ public: void copyFolder(const QString& source, const QString& destination); + void updateStatusLabel(const QString& message); + void appendLog(const QString& message); + +private slots: + void checkTime(); + private: + bool _automatedTestIsRunning{ false }; + QDir _appDataFolder; QDir _savedAppDataFolder; - QString _tempFolder; + QString _workingFolder; QString _snapshotFolder; QString _installationFolder; @@ -62,6 +81,17 @@ private: QString _branch; QString _user; + + std::vector _dayCheckboxes; + std::vector _timeEditCheckboxes; + std::vector _timeEdits; + QLabel* _workingFolderLabel; + + QTimer* _timer; + + QFile _logFile; + + QDateTime _testStartDateTime; }; #endif // hifi_testRunner_h \ No newline at end of file diff --git a/tools/auto-tester/src/ui/AutoTester.cpp b/tools/auto-tester/src/ui/AutoTester.cpp index 3c8be24f69..05eee2957a 100644 --- a/tools/auto-tester/src/ui/AutoTester.cpp +++ b/tools/auto-tester/src/ui/AutoTester.cpp @@ -32,20 +32,47 @@ AutoTester::AutoTester(QWidget* parent) : QMainWindow(parent) { #ifndef Q_OS_WIN _ui.tabWidget->removeTab(1); #endif - //// Coming soon... + + _ui.statusLabel->setText(""); + _ui.plainTextEdit->setReadOnly(true); + + // Coming soon to an auto-tester near you... //// _helpWindow.textBrowser->setText() } void AutoTester::setup() { _test = new Test(_ui.progressBar, _ui.checkBoxInteractiveMode); - _testRunner = new TestRunner(); + + std::vector dayCheckboxes; + dayCheckboxes.emplace_back(_ui.mondayCheckBox); + dayCheckboxes.emplace_back(_ui.tuesdayCheckBox); + dayCheckboxes.emplace_back(_ui.wednesdayCheckBox); + dayCheckboxes.emplace_back(_ui.thursdayCheckBox); + dayCheckboxes.emplace_back(_ui.fridayCheckBox); + dayCheckboxes.emplace_back(_ui.saturdayCheckBox); + dayCheckboxes.emplace_back(_ui.sundayCheckBox); + + std::vector timeEditCheckboxes; + timeEditCheckboxes.emplace_back(_ui.timeEdit1checkBox); + timeEditCheckboxes.emplace_back(_ui.timeEdit2checkBox); + timeEditCheckboxes.emplace_back(_ui.timeEdit3checkBox); + timeEditCheckboxes.emplace_back(_ui.timeEdit4checkBox); + + std::vector timeEdits; + timeEdits.emplace_back(_ui.timeEdit1); + timeEdits.emplace_back(_ui.timeEdit2); + timeEdits.emplace_back(_ui.timeEdit3); + timeEdits.emplace_back(_ui.timeEdit4); + + _testRunner = new TestRunner(dayCheckboxes, timeEditCheckboxes, timeEdits, _ui.workingFolderLabel); } void AutoTester::startTestsEvaluation(const bool isRunningFromCommandLine, const bool isRunningInAutomaticTestRun, const QString& snapshotDirectory, const QString& branch, - const QString& user) { + const QString& user +) { _test->startTestsEvaluation(isRunningFromCommandLine, isRunningInAutomaticTestRun, snapshotDirectory, branch, user); } @@ -103,6 +130,16 @@ void AutoTester::on_createTestRailRunButton_clicked() { _test->createTestRailRun(); } +void AutoTester::on_setWorkingFolderButton_clicked() { + _testRunner->setWorkingFolder(); +} + +void AutoTester::enableRunTabControls() { + _ui.runNowButton->setEnabled(true); + _ui.daysGroupBox->setEnabled(true); + _ui.timesGroupBox->setEnabled(true); +} + void AutoTester::on_runNowButton_clicked() { _testRunner->run(); } @@ -231,3 +268,11 @@ void AutoTester::setBranchText(const QString& branch) { QString AutoTester::getSelectedBranch() { return _ui.branchTextEdit->toPlainText(); } + +void AutoTester::updateStatusLabel(const QString& status) { + _ui.statusLabel->setText(status); +} + +void AutoTester::appendLogWindow(const QString& message) { + _ui.plainTextEdit->appendPlainText(message); +} \ No newline at end of file diff --git a/tools/auto-tester/src/ui/AutoTester.h b/tools/auto-tester/src/ui/AutoTester.h index 6c8641fa49..de0622c670 100644 --- a/tools/auto-tester/src/ui/AutoTester.h +++ b/tools/auto-tester/src/ui/AutoTester.h @@ -47,6 +47,11 @@ public: void setBranchText(const QString& branch); QString getSelectedBranch(); + void enableRunTabControls(); + + void updateStatusLabel(const QString& status); + void appendLogWindow(const QString& message); + private slots: void on_tabWidget_currentChanged(int index); @@ -66,6 +71,7 @@ private slots: void on_createTestRailTestCasesButton_clicked(); void on_createTestRailRunButton_clicked(); + void on_setWorkingFolderButton_clicked(); void on_runNowButton_clicked(); void on_updateTestRailRunResultsButton_clicked(); diff --git a/tools/auto-tester/src/ui/AutoTester.ui b/tools/auto-tester/src/ui/AutoTester.ui index 8c95bba7d1..fa9569a768 100644 --- a/tools/auto-tester/src/ui/AutoTester.ui +++ b/tools/auto-tester/src/ui/AutoTester.ui @@ -6,8 +6,8 @@ 0 0 - 582 - 734 + 707 + 796 @@ -23,8 +23,8 @@ - 430 - 620 + 470 + 660 100 40 @@ -38,12 +38,12 @@ 30 140 - 521 - 461 + 631 + 501 - 1 + 2 @@ -190,11 +190,14 @@ Run + + false + - 200 - 200 - 93 + 10 + 70 + 161 28 @@ -202,6 +205,283 @@ Run now + + + false + + + + 20 + 150 + 91 + 241 + + + + Days + + + + + 10 + 210 + 80 + 17 + + + + Sunday + + + + + + 10 + 90 + 80 + 17 + + + + Wednesday + + + + + + 10 + 60 + 80 + 17 + + + + Tuesday + + + + + + 10 + 120 + 80 + 17 + + + + Thursday + + + + + + 10 + 150 + 80 + 17 + + + + Friday + + + + + + 10 + 180 + 80 + 17 + + + + Saturday + + + + + + 10 + 30 + 80 + 17 + + + + Monday + + + + + + false + + + + 130 + 150 + 161 + 191 + + + + Times + + + + + 30 + 20 + 118 + 22 + + + + + + + 30 + 60 + 118 + 22 + + + + + + + 30 + 100 + 118 + 22 + + + + + + + 30 + 140 + 118 + 22 + + + + + + + 10 + 23 + 21 + 17 + + + + + + + + + + 10 + 63 + 21 + 17 + + + + + + + + + + 10 + 103 + 21 + 17 + + + + + + + + + + 10 + 143 + 21 + 17 + + + + + + + + + + + 10 + 20 + 161 + 28 + + + + Set Working Folder + + + + + + 190 + 20 + 321 + 31 + + + + ####### + + + + + + 300 + 120 + 311 + 331 + + + + + + + 300 + 80 + 41 + 31 + + + + Status: + + + + + + 350 + 80 + 271 + 31 + + + + ####### + + @@ -371,7 +651,7 @@ 80 - 630 + 670 255 23 @@ -386,7 +666,7 @@ 0 0 - 582 + 707 21