From f046d3b87dda1c42a9d5ac7ee38b7eb9bccf38d0 Mon Sep 17 00:00:00 2001 From: NissimHadar Date: Mon, 17 Sep 2018 13:21:01 -0700 Subject: [PATCH] Delete allocated memory where needed. Added option to run server-less Added option to select build. --- tools/auto-tester/src/Test.cpp | 6 + tools/auto-tester/src/TestRailInterface.cpp | 9 +- tools/auto-tester/src/TestRunner.cpp | 147 +++++++++++++----- tools/auto-tester/src/TestRunner.h | 34 +++- tools/auto-tester/src/ui/AutoTester.cpp | 28 +++- tools/auto-tester/src/ui/AutoTester.h | 24 +-- tools/auto-tester/src/ui/AutoTester.ui | 92 +++++++++-- .../src/ui/TestRailRunSelectorWindow.cpp | 1 - 8 files changed, 271 insertions(+), 70 deletions(-) diff --git a/tools/auto-tester/src/Test.cpp b/tools/auto-tester/src/Test.cpp index d3aa310d7d..aedf0be3d3 100644 --- a/tools/auto-tester/src/Test.cpp +++ b/tools/auto-tester/src/Test.cpp @@ -593,6 +593,12 @@ bool Test::createMDFile(const QString& directory) { } mdFile.close(); + + foreach (auto test, testScriptLines.stepList) { + delete test; + } + testScriptLines.stepList.clear(); + return true; } diff --git a/tools/auto-tester/src/TestRailInterface.cpp b/tools/auto-tester/src/TestRailInterface.cpp index e6f12100df..b2132cf85e 100644 --- a/tools/auto-tester/src/TestRailInterface.cpp +++ b/tools/auto-tester/src/TestRailInterface.cpp @@ -367,6 +367,7 @@ void TestRailInterface::createAddTestCasesPythonScript(const QString& testDirect QProcess* process = new QProcess(); connect(process, &QProcess::started, this, [=]() { _busyWindow.exec(); }); + connect(process, SIGNAL(finished(int)), process, SLOT(deleteLater())); connect(process, static_cast(&QProcess::finished), this, [=](int exitCode, QProcess::ExitStatus exitStatus) { _busyWindow.hide(); }); @@ -491,7 +492,7 @@ void TestRailInterface::addRun() { ) { QProcess* process = new QProcess(); connect(process, &QProcess::started, this, [=]() { _busyWindow.exec(); }); - + connect(process, SIGNAL(finished(int)), process, SLOT(deleteLater())); connect(process, static_cast(&QProcess::finished), this, [=](int exitCode, QProcess::ExitStatus exitStatus) { _busyWindow.hide(); }); @@ -499,6 +500,7 @@ void TestRailInterface::addRun() { process->start(_pythonCommand, parameters); } } + void TestRailInterface::updateRunWithResults() { QString filename = _outputDirectory + "/updateRunWithResults.py"; if (QFile::exists(filename)) { @@ -578,7 +580,7 @@ void TestRailInterface::updateRunWithResults() { ) { QProcess* process = new QProcess(); connect(process, &QProcess::started, this, [=]() { _busyWindow.exec(); }); - + connect(process, SIGNAL(finished(int)), process, SLOT(deleteLater())); connect(process, static_cast(&QProcess::finished), this, [=](int exitCode, QProcess::ExitStatus exitStatus) { _busyWindow.hide(); }); @@ -753,6 +755,7 @@ void TestRailInterface::getReleasesFromTestRail() { QProcess* process = new QProcess(); connect(process, static_cast(&QProcess::finished), this, [=](int exitCode, QProcess::ExitStatus exitStatus) { updateReleasesComboData(exitCode, exitStatus); }); + connect(process, SIGNAL(finished(int)), process, SLOT(deleteLater())); QStringList parameters = QStringList() << filename; process->start(_pythonCommand, parameters); @@ -1076,6 +1079,7 @@ void TestRailInterface::getTestSectionsFromTestRail() { QProcess* process = new QProcess(); connect(process, static_cast(&QProcess::finished), this, [=](int exitCode, QProcess::ExitStatus exitStatus) { updateSectionsComboData(exitCode, exitStatus); }); + connect(process, SIGNAL(finished(int)), process, SLOT(deleteLater())); QStringList parameters = QStringList() << filename; process->start(_pythonCommand, parameters); @@ -1114,6 +1118,7 @@ void TestRailInterface::getRunsFromTestRail() { QProcess* process = new QProcess(); connect(process, static_cast(&QProcess::finished), this, [=](int exitCode, QProcess::ExitStatus exitStatus) { updateRunsComboData(exitCode, exitStatus); }); + connect(process, SIGNAL(finished(int)), process, SLOT(deleteLater())); QStringList parameters = QStringList() << filename; diff --git a/tools/auto-tester/src/TestRunner.cpp b/tools/auto-tester/src/TestRunner.cpp index e135fdae45..624dce17eb 100644 --- a/tools/auto-tester/src/TestRunner.cpp +++ b/tools/auto-tester/src/TestRunner.cpp @@ -25,16 +25,44 @@ TestRunner::TestRunner(std::vector dayCheckboxes, std::vector timeEditCheckboxes, std::vector timeEdits, QLabel* workingFolderLabel, + QCheckBox* runServerless, + QCheckBox* runLatest, + QTextEdit* url, QObject* parent) : - QObject(parent) -{ + QObject(parent) { _dayCheckboxes = dayCheckboxes; _timeEditCheckboxes = timeEditCheckboxes; _timeEdits = timeEdits; _workingFolderLabel = workingFolderLabel; + _runServerless = runServerless; + _runLatest = runLatest; + _url = url; installerThread = new QThread(); + installerWorker = new Worker(); + installerWorker->moveToThread(installerThread); + installerThread->start(); + connect(this, SIGNAL(startInstaller()), installerWorker, SLOT(runCommand())); + connect(installerWorker, SIGNAL(commandComplete()), this, SLOT(installationComplete())); + interfaceThread = new QThread(); + interfaceWorker = new Worker(); + interfaceThread->start(); + interfaceWorker->moveToThread(interfaceThread); + connect(this, SIGNAL(startInterface()), interfaceWorker, SLOT(runCommand())); + connect(interfaceWorker, SIGNAL(commandComplete()), this, SLOT(interfaceExecutionComplete())); +} + +TestRunner::~TestRunner() { + delete installerThread; + delete interfaceThread; + + delete interfaceThread; + delete interfaceWorker; + + if (_timer) { + delete _timer; + } } void TestRunner::setWorkingFolder() { @@ -46,7 +74,7 @@ void TestRunner::setWorkingFolder() { } _workingFolder = QFileDialog::getExistingDirectory(nullptr, "Please select a temporary folder for installation", parent, - QFileDialog::ShowDirsOnly); + QFileDialog::ShowDirsOnly); // If user canceled then restore previous selection and return if (_workingFolder == "") { @@ -60,7 +88,6 @@ void TestRunner::setWorkingFolder() { autoTester->enableRunTabControls(); _workingFolderLabel->setText(QDir::toNativeSeparators(_workingFolder)); - // The time is checked every 30 seconds for automatic test start _timer = new QTimer(this); connect(_timer, SIGNAL(timeout()), this, SLOT(checkTime())); _timer->start(30 * 1000); //time specified in ms @@ -79,10 +106,19 @@ void TestRunner::run() { // Download the latest High Fidelity installer and build XML. QStringList urls; - urls << INSTALLER_URL << BUILD_XML_URL; - QStringList filenames; - filenames << INSTALLER_FILENAME << BUILD_XML_FILENAME; + if (_runLatest->isChecked()) { + _installerFilename = INSTALLER_FILENAME_LATEST; + + urls << INSTALLER_URL_LATEST << BUILD_XML_URL; + filenames << _installerFilename << BUILD_XML_FILENAME; + } else { + QString urlText = _url->toPlainText(); + urls << urlText; + _installerFilename = getInstallerNameFromURL(urlText); + filenames << _installerFilename; + } + updateStatusLabel("Downloading installer"); @@ -110,29 +146,24 @@ void TestRunner::runInstaller() { QStringList arguments{ QStringList() << QString("/S") << QString("/D=") + QDir::toNativeSeparators(_installationFolder) }; - QString installerFullPath = _workingFolder + "/" + INSTALLER_FILENAME; + QString installerFullPath = _workingFolder + "/" + _installerFilename; QString commandLine = QDir::toNativeSeparators(installerFullPath) + " /S /D=" + QDir::toNativeSeparators(_installationFolder); - worker = new Worker(commandLine); - - worker->moveToThread(installerThread); - connect(installerThread, SIGNAL(started()), worker, SLOT(process())); - connect(worker, SIGNAL(finished()), this, SLOT(installationComplete())); - installerThread->start(); + installerWorker->setCommandLine(commandLine); + emit startInstaller(); } void TestRunner::installationComplete() { - disconnect(installerThread, SIGNAL(started()), worker, SLOT(process())); - disconnect(worker, SIGNAL(finished()), this, SLOT(installationComplete())); - delete worker; - createSnapshotFolder(); updateStatusLabel("Running tests"); - startLocalServerProcesses(); + if (!_runServerless->isChecked()) { + startLocalServerProcesses(); + } + runInterfaceWithTestScript(); } @@ -239,24 +270,28 @@ void TestRunner::startLocalServerProcesses() { } void TestRunner::runInterfaceWithTestScript() { - QString commandLine = QString("\"") + QDir::toNativeSeparators(_installationFolder) + - "\\interface.exe\" --url hifi://localhost --testScript https://raw.githubusercontent.com/" + _user + - "/hifi_tests/" + _branch + "/tests/testRecursive.js quitWhenFinished --testResultsLocation " + - _snapshotFolder; + QString commandLine; - worker = new Worker(commandLine); + if (_runServerless->isChecked()) { + // Move to an empty area + commandLine = + QString("\"") + QDir::toNativeSeparators(_installationFolder) + + "\\interface.exe\" --url hifi://localhost/9999,9999,9999/0.0,0.0,0.0,1.0 --testScript https://raw.githubusercontent.com/" + _user + + "/hifi_tests/" + _branch + "/tests/testRecursive.js quitWhenFinished --testResultsLocation " + + _snapshotFolder; + } else { + // There is no content, so no need to move + commandLine = QString("\"") + QDir::toNativeSeparators(_installationFolder) + + "\\interface.exe\" --url hifi://localhost --testScript https://raw.githubusercontent.com/" + _user + + "/hifi_tests/" + _branch + + "/tests/content/entity/zone/testRecursive.js quitWhenFinished --testResultsLocation " + _snapshotFolder; + } - worker->moveToThread(interfaceThread); - connect(interfaceThread, SIGNAL(started()), worker, SLOT(process())); - connect(worker, SIGNAL(finished()), this, SLOT(interfaceExecutionComplete())); - interfaceThread->start(); + interfaceWorker->setCommandLine(commandLine); + emit startInterface(); } void TestRunner::interfaceExecutionComplete() { - disconnect(interfaceThread, SIGNAL(started()), worker, SLOT(process())); - disconnect(worker, SIGNAL(finished()), this, SLOT(interfaceExecutionComplete())); - delete worker; - killProcesses(); evaluateResults(); @@ -285,6 +320,13 @@ void TestRunner::automaticTestRunEvaluationComplete(QString zippedFolder) { } void TestRunner::addBuildNumberToResults(QString zippedFolderName) { + if (!_runLatest->isChecked()) { + QStringList filenameParts = zippedFolderName.split("."); + QString augmentedFilename = filenameParts[0] + "(" + getPRNumberFromURL(_url->toPlainText()) + ")." + filenameParts[1]; + QFile::rename(zippedFolderName, augmentedFilename); + + return; + } try { QDomDocument domDocument; QString filename{ _workingFolder + "/" + BUILD_XML_FILENAME }; @@ -444,11 +486,46 @@ void TestRunner::appendLog(const QString& message) { autoTester->appendLogWindow(message); } -Worker::Worker(const QString commandLine) { +QString TestRunner::getInstallerNameFromURL(const QString& url) { + // An example URL: https://deployment.highfidelity.com/jobs/pr-build/label%3Dwindows/13023/HighFidelity-Beta-Interface-PR14006-be76c43.exe + try { + QStringList urlParts = url.split("/"); + int rr = urlParts.size(); + if (urlParts.size() != 8) { + throw "URL not in expected format, should look like `https://deployment.highfidelity.com/jobs/pr-build/label%3Dwindows/13023/HighFidelity-Beta-Interface-PR14006-be76c43.exe`"; + } + return urlParts[urlParts.size() - 1]; + } catch (QString errorMessage) { + QMessageBox::critical(0, "Internal error: " + QString(__FILE__) + ":" + QString::number(__LINE__), errorMessage); + exit(-1); + } catch (...) { + QMessageBox::critical(0, "Internal error: " + QString(__FILE__) + ":" + QString::number(__LINE__), "unknown error"); + exit(-1); + } +} + +QString TestRunner::getPRNumberFromURL(const QString& url) { + try { + QStringList urlParts = url.split("/"); + QStringList filenameParts = urlParts[urlParts.size() - 1].split("-"); + if (filenameParts.size() != 5) { + throw "URL not in expected format, should look like `https://deployment.highfidelity.com/jobs/pr-build/label%3Dwindows/13023/HighFidelity-Beta-Interface-PR14006-be76c43.exe`"; + } + return filenameParts[3]; + } catch (QString errorMessage) { + QMessageBox::critical(0, "Internal error: " + QString(__FILE__) + ":" + QString::number(__LINE__), errorMessage); + exit(-1); + } catch (...) { + QMessageBox::critical(0, "Internal error: " + QString(__FILE__) + ":" + QString::number(__LINE__), "unknown error"); + exit(-1); + } +} + +void Worker::setCommandLine(const QString& commandLine) { _commandLine = commandLine; } -void Worker::process() { +void Worker::runCommand() { system(_commandLine.toStdString().c_str()); - emit finished(); + emit commandComplete(); } diff --git a/tools/auto-tester/src/TestRunner.h b/tools/auto-tester/src/TestRunner.h index abf4e87880..0843b438c8 100644 --- a/tools/auto-tester/src/TestRunner.h +++ b/tools/auto-tester/src/TestRunner.h @@ -15,6 +15,7 @@ #include #include #include +#include #include #include #include @@ -28,8 +29,13 @@ public: std::vector timeEditCheckboxes, std::vector timeEdits, QLabel* workingFolderLabel, + QCheckBox* runServerless, + QCheckBox* runLatest, + QTextEdit* url, QObject* parent = 0); + ~TestRunner(); + void setWorkingFolder(); void run(); @@ -56,17 +62,26 @@ public: void updateStatusLabel(const QString& message); void appendLog(const QString& message); + QString getInstallerNameFromURL(const QString& url); + QString getPRNumberFromURL(const QString& url); + private slots: void checkTime(); void installationComplete(); void interfaceExecutionComplete(); +signals: + void startInstaller(); + void startInterface(); + private: bool _automatedTestIsRunning{ false }; - const QString INSTALLER_URL{ "http://builds.highfidelity.com/HighFidelity-Beta-latest-dev.exe" }; - const QString INSTALLER_FILENAME{ "HighFidelity-Beta-latest-dev.exe" }; + const QString INSTALLER_URL_LATEST{ "http://builds.highfidelity.com/HighFidelity-Beta-latest-dev.exe" }; + const QString INSTALLER_FILENAME_LATEST{ "HighFidelity-Beta-latest-dev.exe" }; + QString _installerURL; + QString _installerFilename; const QString BUILD_XML_URL{ "https://highfidelity.com/dev-builds.xml" }; const QString BUILD_XML_FILENAME{ "dev-builds.xml" }; @@ -87,6 +102,9 @@ private: std::vector _timeEditCheckboxes; std::vector _timeEdits; QLabel* _workingFolderLabel; + QCheckBox* _runServerless; + QCheckBox* _runLatest; + QTextEdit* _url; QTimer* _timer; @@ -96,18 +114,22 @@ private: QThread* installerThread; QThread* interfaceThread; - Worker* worker; + Worker* installerWorker; + Worker* interfaceWorker; }; class Worker : public QObject { Q_OBJECT public: - Worker(const QString commandLine); + void setCommandLine(const QString& commandLine); + public slots: - void process(); + void runCommand(); signals: - void finished(); + void commandComplete(); + void startInstaller(); + void startInterface(); private: QString _commandLine; diff --git a/tools/auto-tester/src/ui/AutoTester.cpp b/tools/auto-tester/src/ui/AutoTester.cpp index 05eee2957a..3a438a5fb9 100644 --- a/tools/auto-tester/src/ui/AutoTester.cpp +++ b/tools/auto-tester/src/ui/AutoTester.cpp @@ -40,7 +40,22 @@ AutoTester::AutoTester(QWidget* parent) : QMainWindow(parent) { //// _helpWindow.textBrowser->setText() } +AutoTester::~AutoTester() { + delete _signalMapper; + + if (_test) { + delete _test; + } + + if (_testRunner) { + delete _testRunner; + } +} + void AutoTester::setup() { + if (_test) { + delete _test; + } _test = new Test(_ui.progressBar, _ui.checkBoxInteractiveMode); std::vector dayCheckboxes; @@ -64,7 +79,10 @@ void AutoTester::setup() { timeEdits.emplace_back(_ui.timeEdit3); timeEdits.emplace_back(_ui.timeEdit4); - _testRunner = new TestRunner(dayCheckboxes, timeEditCheckboxes, timeEdits, _ui.workingFolderLabel); + if (_testRunner) { + delete _testRunner; + } + _testRunner = new TestRunner(dayCheckboxes, timeEditCheckboxes, timeEdits, _ui.workingFolderLabel, _ui.checkBoxServerless, _ui.checkBoxRunLatest, _ui.urlTextEdit); } void AutoTester::startTestsEvaluation(const bool isRunningFromCommandLine, @@ -144,6 +162,10 @@ void AutoTester::on_runNowButton_clicked() { _testRunner->run(); } +void AutoTester::on_checkBoxRunLatest_clicked() { + _ui.urlTextEdit->setEnabled(!_ui.checkBoxRunLatest->isChecked()); +} + void AutoTester::automaticTestRunEvaluationComplete(QString zippedFolderName) { _testRunner->automaticTestRunEvaluationComplete(zippedFolderName); } @@ -213,6 +235,10 @@ void AutoTester::downloadFiles(const QStringList& URLs, const QString& directory _ui.progressBar->setValue(0); _ui.progressBar->setVisible(true); + foreach (auto downloader, _downloaders) { + delete downloader; + } + _downloaders.clear(); for (int i = 0; i < _numberOfFilesToDownload; ++i) { downloadFile(URLs[i]); diff --git a/tools/auto-tester/src/ui/AutoTester.h b/tools/auto-tester/src/ui/AutoTester.h index de0622c670..f59d39e851 100644 --- a/tools/auto-tester/src/ui/AutoTester.h +++ b/tools/auto-tester/src/ui/AutoTester.h @@ -21,12 +21,12 @@ #include "HelpWindow.h" #include "../TestRunner.h" - class AutoTester : public QMainWindow { Q_OBJECT public: - AutoTester(QWidget *parent = Q_NULLPTR); + AutoTester(QWidget* parent = Q_NULLPTR); + ~AutoTester(); void setup(); @@ -39,7 +39,7 @@ public: void automaticTestRunEvaluationComplete(QString zippedFolderName); void downloadFile(const QUrl& url); - void downloadFiles(const QStringList& URLs, const QString& directoryName, const QStringList& filenames, void *caller); + void downloadFiles(const QStringList& URLs, const QString& directoryName, const QStringList& filenames, void* caller); void setUserText(const QString& user); QString getSelectedUser(); @@ -58,7 +58,7 @@ private slots: void on_evaluateTestsButton_clicked(); void on_createRecursiveScriptButton_clicked(); void on_createAllRecursiveScriptsButton_clicked(); - void on_createTestsButton_clicked(); + void on_createTestsButton_clicked(); void on_createMDFileButton_clicked(); void on_createAllMDFilesButton_clicked(); @@ -74,6 +74,8 @@ private slots: void on_setWorkingFolderButton_clicked(); void on_runNowButton_clicked(); + void on_checkBoxRunLatest_clicked(); + void on_updateTestRailRunResultsButton_clicked(); void on_hideTaskbarButton_clicked(); @@ -91,8 +93,8 @@ private slots: private: Ui::AutoTesterClass _ui; - Test* _test; - TestRunner* _testRunner; + Test* _test{ nullptr }; + TestRunner* _testRunner{ nullptr }; std::vector _downloaders; @@ -103,15 +105,15 @@ private: // Used to enable passing a parameter to slots QSignalMapper* _signalMapper; - int _numberOfFilesToDownload { 0 }; - int _numberOfFilesDownloaded { 0 }; - int _index { 0 }; + int _numberOfFilesToDownload{ 0 }; + int _numberOfFilesDownloaded{ 0 }; + int _index{ 0 }; - bool _isRunningFromCommandline { false }; + bool _isRunningFromCommandline{ false }; HelpWindow _helpWindow; void* _caller; }; -#endif // hifi_AutoTester_h \ No newline at end of file +#endif // hifi_AutoTester_h \ No newline at end of file diff --git a/tools/auto-tester/src/ui/AutoTester.ui b/tools/auto-tester/src/ui/AutoTester.ui index fa9569a768..e241acc6f3 100644 --- a/tools/auto-tester/src/ui/AutoTester.ui +++ b/tools/auto-tester/src/ui/AutoTester.ui @@ -6,8 +6,8 @@ 0 0 - 707 - 796 + 737 + 864 @@ -24,7 +24,7 @@ 470 - 660 + 750 100 40 @@ -36,10 +36,10 @@ - 30 + 40 140 631 - 501 + 581 @@ -196,7 +196,7 @@ 10 - 70 + 160 161 28 @@ -212,7 +212,7 @@ 20 - 150 + 240 91 241 @@ -319,7 +319,7 @@ 130 - 150 + 240 161 191 @@ -443,14 +443,14 @@ - ####### + (not set...) 300 - 120 + 210 311 331 @@ -460,7 +460,7 @@ 300 - 80 + 170 41 31 @@ -473,7 +473,7 @@ 350 - 80 + 170 271 31 @@ -482,6 +482,70 @@ ####### + + + + 20 + 70 + 120 + 20 + + + + <html><head/><body><p>If unchecked, will not show results during evaluation</p></body></html> + + + Serveless + + + true + + + + + + 20 + 100 + 120 + 20 + + + + <html><head/><body><p>If unchecked, will not show results during evaluation</p></body></html> + + + Run Latest + + + true + + + + + false + + + + 150 + 98 + 461 + 24 + + + + + + + 128 + 95 + 21 + 31 + + + + URL + + @@ -651,7 +715,7 @@ 80 - 670 + 760 255 23 @@ -666,7 +730,7 @@ 0 0 - 707 + 737 21 diff --git a/tools/auto-tester/src/ui/TestRailRunSelectorWindow.cpp b/tools/auto-tester/src/ui/TestRailRunSelectorWindow.cpp index 54a3985a8b..2247fe33cc 100644 --- a/tools/auto-tester/src/ui/TestRailRunSelectorWindow.cpp +++ b/tools/auto-tester/src/ui/TestRailRunSelectorWindow.cpp @@ -19,7 +19,6 @@ TestRailRunSelectorWindow::TestRailRunSelectorWindow(QWidget *parent) { projectIDLineEdit->setValidator(new QIntValidator(1, 999, this)); } - void TestRailRunSelectorWindow::reset() { urlLineEdit->setDisabled(false); userLineEdit->setDisabled(false);