Delete allocated memory where needed.

Added option to run server-less
Added option to select build.

tools/auto-tester/src/Test.cpp

@@ -593,6 +593,12 @@ bool Test::createMDFile(const QString& directory) {
     }
 
     mdFile.close();
+
+    foreach (auto test, testScriptLines.stepList) {
+        delete test;
+    }
+    testScriptLines.stepList.clear();
+
     return true;
 }
 

tools/auto-tester/src/TestRailInterface.cpp

@@ -367,6 +367,7 @@ void TestRailInterface::createAddTestCasesPythonScript(const QString& testDirect
         QProcess* process = new QProcess();
 
         connect(process, &QProcess::started, this, [=]() { _busyWindow.exec(); });
+        connect(process, SIGNAL(finished(int)), process, SLOT(deleteLater()));
         connect(process, static_cast<void (QProcess::*)(int, QProcess::ExitStatus)>(&QProcess::finished), this,
                 [=](int exitCode, QProcess::ExitStatus exitStatus) { _busyWindow.hide(); });
 
@@ -491,7 +492,7 @@ void TestRailInterface::addRun() {
     ) {
         QProcess* process = new QProcess();
         connect(process, &QProcess::started, this, [=]() { _busyWindow.exec(); });
-
+        connect(process, SIGNAL(finished(int)), process, SLOT(deleteLater()));
         connect(process, static_cast<void (QProcess::*)(int, QProcess::ExitStatus)>(&QProcess::finished), this,
                 [=](int exitCode, QProcess::ExitStatus exitStatus) { _busyWindow.hide(); });
 
@@ -499,6 +500,7 @@ void TestRailInterface::addRun() {
         process->start(_pythonCommand, parameters);
     }
 }
+
 void TestRailInterface::updateRunWithResults() {
     QString filename = _outputDirectory + "/updateRunWithResults.py";
     if (QFile::exists(filename)) {
@@ -578,7 +580,7 @@ void TestRailInterface::updateRunWithResults() {
     ) {
         QProcess* process = new QProcess();
         connect(process, &QProcess::started, this, [=]() { _busyWindow.exec(); });
-
+        connect(process, SIGNAL(finished(int)), process, SLOT(deleteLater()));
         connect(process, static_cast<void (QProcess::*)(int, QProcess::ExitStatus)>(&QProcess::finished), this,
                 [=](int exitCode, QProcess::ExitStatus exitStatus) { _busyWindow.hide(); });
 
@@ -753,6 +755,7 @@ void TestRailInterface::getReleasesFromTestRail() {
     QProcess* process = new QProcess();
     connect(process, static_cast<void (QProcess::*)(int, QProcess::ExitStatus)>(&QProcess::finished), this,
             [=](int exitCode, QProcess::ExitStatus exitStatus) { updateReleasesComboData(exitCode, exitStatus); });
+    connect(process, SIGNAL(finished(int)), process, SLOT(deleteLater()));
 
     QStringList parameters = QStringList() << filename;
     process->start(_pythonCommand, parameters);
@@ -1076,6 +1079,7 @@ void TestRailInterface::getTestSectionsFromTestRail() {
     QProcess* process = new QProcess();
     connect(process, static_cast<void (QProcess::*)(int, QProcess::ExitStatus)>(&QProcess::finished), this,
             [=](int exitCode, QProcess::ExitStatus exitStatus) { updateSectionsComboData(exitCode, exitStatus); });
+    connect(process, SIGNAL(finished(int)), process, SLOT(deleteLater()));
 
     QStringList parameters = QStringList() << filename;
     process->start(_pythonCommand, parameters);
@@ -1114,6 +1118,7 @@ void TestRailInterface::getRunsFromTestRail() {
     QProcess* process = new QProcess();
     connect(process, static_cast<void (QProcess::*)(int, QProcess::ExitStatus)>(&QProcess::finished), this,
             [=](int exitCode, QProcess::ExitStatus exitStatus) { updateRunsComboData(exitCode, exitStatus); });
+    connect(process, SIGNAL(finished(int)), process, SLOT(deleteLater()));
 
     QStringList parameters = QStringList() << filename;
 

tools/auto-tester/src/TestRunner.cpp

@@ -25,16 +25,44 @@ TestRunner::TestRunner(std::vector<QCheckBox*> dayCheckboxes,
                        std::vector<QCheckBox*> timeEditCheckboxes,
                        std::vector<QTimeEdit*> timeEdits,
                        QLabel* workingFolderLabel,
+                       QCheckBox* runServerless,
+                       QCheckBox* runLatest,
+                       QTextEdit* url,
                        QObject* parent) :
-    QObject(parent) 
+    QObject(parent) {
-{
     _dayCheckboxes = dayCheckboxes;
     _timeEditCheckboxes = timeEditCheckboxes;
     _timeEdits = timeEdits;
     _workingFolderLabel = workingFolderLabel;
+    _runServerless = runServerless;
+    _runLatest = runLatest;
+    _url = url;
 
     installerThread = new QThread();
+    installerWorker = new Worker();
+    installerWorker->moveToThread(installerThread);
+    installerThread->start();
+    connect(this, SIGNAL(startInstaller()), installerWorker, SLOT(runCommand()));
+    connect(installerWorker, SIGNAL(commandComplete()), this, SLOT(installationComplete()));
+
     interfaceThread = new QThread();
+    interfaceWorker = new Worker();
+    interfaceThread->start();
+    interfaceWorker->moveToThread(interfaceThread);
+    connect(this, SIGNAL(startInterface()), interfaceWorker, SLOT(runCommand()));
+    connect(interfaceWorker, SIGNAL(commandComplete()), this, SLOT(interfaceExecutionComplete()));
+}
+
+TestRunner::~TestRunner() {
+    delete installerThread;
+    delete interfaceThread;
+
+    delete interfaceThread;
+    delete interfaceWorker;
+
+    if (_timer) {
+        delete _timer;
+    }
 }
 
 void TestRunner::setWorkingFolder() {
@@ -60,7 +88,6 @@ void TestRunner::setWorkingFolder() {
     autoTester->enableRunTabControls();
     _workingFolderLabel->setText(QDir::toNativeSeparators(_workingFolder));
 
-    // The time is checked every 30 seconds for automatic test start
     _timer = new QTimer(this);
     connect(_timer, SIGNAL(timeout()), this, SLOT(checkTime()));
     _timer->start(30 * 1000);  //time specified in ms
@@ -79,10 +106,19 @@ void TestRunner::run() {
 
     // Download the latest High Fidelity installer and build XML.
     QStringList urls;
-    urls << INSTALLER_URL << BUILD_XML_URL;
-
     QStringList filenames;
-    filenames << INSTALLER_FILENAME << BUILD_XML_FILENAME;
+    if (_runLatest->isChecked()) {
+        _installerFilename = INSTALLER_FILENAME_LATEST;
+
+        urls << INSTALLER_URL_LATEST << BUILD_XML_URL;
+        filenames << _installerFilename << BUILD_XML_FILENAME;
+    } else {
+        QString urlText = _url->toPlainText();
+        urls << urlText;
+        _installerFilename = getInstallerNameFromURL(urlText);
+        filenames << _installerFilename;
+    }
+
 
     updateStatusLabel("Downloading installer");
 
@@ -110,29 +146,24 @@ void TestRunner::runInstaller() {
 
     QStringList arguments{ QStringList() << QString("/S") << QString("/D=") + QDir::toNativeSeparators(_installationFolder) };
 
-    QString installerFullPath = _workingFolder + "/" + INSTALLER_FILENAME;
+    QString installerFullPath = _workingFolder + "/" + _installerFilename;
 
     QString commandLine =
         QDir::toNativeSeparators(installerFullPath) + " /S /D=" + QDir::toNativeSeparators(_installationFolder);
 
-    worker = new Worker(commandLine);
+    installerWorker->setCommandLine(commandLine);
-
+    emit startInstaller();
-    worker->moveToThread(installerThread);
-    connect(installerThread, SIGNAL(started()), worker, SLOT(process()));
-    connect(worker, SIGNAL(finished()), this, SLOT(installationComplete()));
-    installerThread->start();
 }
 
 void TestRunner::installationComplete() {
-    disconnect(installerThread, SIGNAL(started()), worker, SLOT(process()));
-    disconnect(worker, SIGNAL(finished()), this, SLOT(installationComplete()));
-    delete worker;
-
     createSnapshotFolder();
 
     updateStatusLabel("Running tests");
 
+    if (!_runServerless->isChecked()) {
         startLocalServerProcesses();
+    }
+
     runInterfaceWithTestScript();
 }
 
@@ -239,24 +270,28 @@ void TestRunner::startLocalServerProcesses() {
 }
 
 void TestRunner::runInterfaceWithTestScript() {
-    QString commandLine = QString("\"") + QDir::toNativeSeparators(_installationFolder) +
+    QString commandLine;
-                          "\\interface.exe\" --url hifi://localhost --testScript https://raw.githubusercontent.com/" + _user +
+
+    if (_runServerless->isChecked()) {
+        // Move to an empty area
+        commandLine =
+            QString("\"") + QDir::toNativeSeparators(_installationFolder) +
+            "\\interface.exe\" --url hifi://localhost/9999,9999,9999/0.0,0.0,0.0,1.0 --testScript https://raw.githubusercontent.com/" + _user +
             "/hifi_tests/" + _branch + "/tests/testRecursive.js quitWhenFinished --testResultsLocation " +
             _snapshotFolder;
+    } else {
+        // There is no content, so no need to move
+        commandLine = QString("\"") + QDir::toNativeSeparators(_installationFolder) +
+                      "\\interface.exe\" --url hifi://localhost --testScript https://raw.githubusercontent.com/" + _user +
+                      "/hifi_tests/" + _branch +
+                      "/tests/content/entity/zone/testRecursive.js quitWhenFinished --testResultsLocation " + _snapshotFolder;
+    }
 
-    worker = new Worker(commandLine);
+    interfaceWorker->setCommandLine(commandLine);
-
+    emit startInterface();
-    worker->moveToThread(interfaceThread);
-    connect(interfaceThread, SIGNAL(started()), worker, SLOT(process()));
-    connect(worker, SIGNAL(finished()), this, SLOT(interfaceExecutionComplete()));
-    interfaceThread->start();
 }
 
 void TestRunner::interfaceExecutionComplete() {
-    disconnect(interfaceThread, SIGNAL(started()), worker, SLOT(process()));
-    disconnect(worker, SIGNAL(finished()), this, SLOT(interfaceExecutionComplete()));
-    delete worker;
-
     killProcesses();
 
     evaluateResults();
@@ -285,6 +320,13 @@ void TestRunner::automaticTestRunEvaluationComplete(QString zippedFolder) {
 }
 
 void TestRunner::addBuildNumberToResults(QString zippedFolderName) {
+    if (!_runLatest->isChecked()) {
+        QStringList filenameParts = zippedFolderName.split(".");
+        QString augmentedFilename = filenameParts[0] + "(" + getPRNumberFromURL(_url->toPlainText()) + ")." + filenameParts[1];
+        QFile::rename(zippedFolderName, augmentedFilename);
+
+        return;
+    }
     try {
         QDomDocument domDocument;
         QString filename{ _workingFolder + "/" + BUILD_XML_FILENAME };
@@ -444,11 +486,46 @@ void TestRunner::appendLog(const QString& message) {
     autoTester->appendLogWindow(message);
 }
 
-Worker::Worker(const QString commandLine) { 
+QString TestRunner::getInstallerNameFromURL(const QString& url) {
+    // An example URL: https://deployment.highfidelity.com/jobs/pr-build/label%3Dwindows/13023/HighFidelity-Beta-Interface-PR14006-be76c43.exe
+    try {
+        QStringList urlParts = url.split("/");
+        int rr = urlParts.size(); 
+        if (urlParts.size() != 8) {
+            throw "URL not in expected format, should look like `https://deployment.highfidelity.com/jobs/pr-build/label%3Dwindows/13023/HighFidelity-Beta-Interface-PR14006-be76c43.exe`";
+        }
+        return urlParts[urlParts.size() - 1];
+    } catch (QString errorMessage) {
+        QMessageBox::critical(0, "Internal error: " + QString(__FILE__) + ":" + QString::number(__LINE__), errorMessage);
+        exit(-1);
+    } catch (...) {
+        QMessageBox::critical(0, "Internal error: " + QString(__FILE__) + ":" + QString::number(__LINE__), "unknown error");
+        exit(-1);
+    }
+}
+
+QString TestRunner::getPRNumberFromURL(const QString& url) {
+    try {
+        QStringList urlParts = url.split("/");
+        QStringList filenameParts = urlParts[urlParts.size() - 1].split("-");
+        if (filenameParts.size() != 5) {
+            throw "URL not in expected format, should look like `https://deployment.highfidelity.com/jobs/pr-build/label%3Dwindows/13023/HighFidelity-Beta-Interface-PR14006-be76c43.exe`";
+        }
+        return filenameParts[3];
+    } catch (QString errorMessage) {
+        QMessageBox::critical(0, "Internal error: " + QString(__FILE__) + ":" + QString::number(__LINE__), errorMessage);
+        exit(-1);
+    } catch (...) {
+        QMessageBox::critical(0, "Internal error: " + QString(__FILE__) + ":" + QString::number(__LINE__), "unknown error");
+        exit(-1);
+    }
+}
+
+void Worker::setCommandLine(const QString& commandLine) {
     _commandLine = commandLine;
 }
 
-void Worker::process() {
+void Worker::runCommand() {
     system(_commandLine.toStdString().c_str());
-    emit finished();
+    emit commandComplete();
 }

tools/auto-tester/src/TestRunner.h

@@ -15,6 +15,7 @@
 #include <QDir>
 #include <QLabel>
 #include <QObject>
+#include <QTextEdit>
 #include <QThread>
 #include <QTimeEdit>
 #include <QTimer>
@@ -28,8 +29,13 @@ public:
                         std::vector<QCheckBox*> timeEditCheckboxes,
                         std::vector<QTimeEdit*> timeEdits,
                         QLabel* workingFolderLabel,
+                        QCheckBox* runServerless,
+                        QCheckBox* runLatest,
+                        QTextEdit* url,
                         QObject* parent = 0);
 
+    ~TestRunner();
+
     void setWorkingFolder();
 
     void run();
@@ -56,17 +62,26 @@ public:
     void updateStatusLabel(const QString& message);
     void appendLog(const QString& message);
 
+    QString getInstallerNameFromURL(const QString& url);
+    QString getPRNumberFromURL(const QString& url);
+
 private slots:
     void checkTime();
     void installationComplete();
     void interfaceExecutionComplete();
 
+signals:
+    void startInstaller();
+    void startInterface();
+
 private:
     bool _automatedTestIsRunning{ false };
 
-    const QString INSTALLER_URL{ "http://builds.highfidelity.com/HighFidelity-Beta-latest-dev.exe" };
+    const QString INSTALLER_URL_LATEST{ "http://builds.highfidelity.com/HighFidelity-Beta-latest-dev.exe" };
-    const QString INSTALLER_FILENAME{ "HighFidelity-Beta-latest-dev.exe" };
+    const QString INSTALLER_FILENAME_LATEST{ "HighFidelity-Beta-latest-dev.exe" };
 
+    QString _installerURL;
+    QString _installerFilename;
     const QString BUILD_XML_URL{ "https://highfidelity.com/dev-builds.xml" };
     const QString BUILD_XML_FILENAME{ "dev-builds.xml" };
 
@@ -87,6 +102,9 @@ private:
     std::vector<QCheckBox*> _timeEditCheckboxes;
     std::vector<QTimeEdit*> _timeEdits;
     QLabel* _workingFolderLabel;
+    QCheckBox* _runServerless;
+    QCheckBox* _runLatest;
+    QTextEdit* _url;
 
     QTimer* _timer;
 
@@ -96,18 +114,22 @@ private:
 
     QThread* installerThread;
     QThread* interfaceThread;
-    Worker* worker;
+    Worker* installerWorker;
+    Worker* interfaceWorker;
 };
 
 class Worker : public QObject {
     Q_OBJECT
 public:
-    Worker(const QString commandLine);
+    void setCommandLine(const QString& commandLine);
+
 public slots:
-    void process();
+    void runCommand();
 
 signals:
-    void finished();
+    void commandComplete();
+    void startInstaller();
+    void startInterface();
 
 private:
     QString _commandLine;

tools/auto-tester/src/ui/AutoTester.cpp

@@ -40,7 +40,22 @@ AutoTester::AutoTester(QWidget* parent) : QMainWindow(parent) {
    //// _helpWindow.textBrowser->setText()
 }
 
+AutoTester::~AutoTester() {
+    delete _signalMapper;
+
+    if (_test) {
+        delete _test;
+    }
+
+    if (_testRunner) {
+        delete _testRunner;
+    }
+}
+
 void AutoTester::setup() {
+    if (_test) {
+        delete _test;
+    }
     _test = new Test(_ui.progressBar, _ui.checkBoxInteractiveMode);
 
     std::vector<QCheckBox*> dayCheckboxes;
@@ -64,7 +79,10 @@ void AutoTester::setup() {
     timeEdits.emplace_back(_ui.timeEdit3);
     timeEdits.emplace_back(_ui.timeEdit4);
 
-    _testRunner = new TestRunner(dayCheckboxes, timeEditCheckboxes, timeEdits, _ui.workingFolderLabel);
+    if (_testRunner) {
+        delete _testRunner;
+    }
+    _testRunner = new TestRunner(dayCheckboxes, timeEditCheckboxes, timeEdits, _ui.workingFolderLabel, _ui.checkBoxServerless, _ui.checkBoxRunLatest, _ui.urlTextEdit);
 }
 
 void AutoTester::startTestsEvaluation(const bool isRunningFromCommandLine,
@@ -144,6 +162,10 @@ void AutoTester::on_runNowButton_clicked() {
     _testRunner->run();
 }
 
+void AutoTester::on_checkBoxRunLatest_clicked() {
+    _ui.urlTextEdit->setEnabled(!_ui.checkBoxRunLatest->isChecked());
+}
+
 void AutoTester::automaticTestRunEvaluationComplete(QString zippedFolderName) {
     _testRunner->automaticTestRunEvaluationComplete(zippedFolderName);
 }
@@ -213,6 +235,10 @@ void AutoTester::downloadFiles(const QStringList& URLs, const QString& directory
     _ui.progressBar->setValue(0);
     _ui.progressBar->setVisible(true);
 
+    foreach (auto downloader, _downloaders) {
+        delete downloader;
+    }
+
     _downloaders.clear();
     for (int i = 0; i < _numberOfFilesToDownload; ++i) {
         downloadFile(URLs[i]);

tools/auto-tester/src/ui/AutoTester.h

@@ -21,12 +21,12 @@
 #include "HelpWindow.h"
 #include "../TestRunner.h"
 
-
 class AutoTester : public QMainWindow {
     Q_OBJECT
 
 public:
     AutoTester(QWidget* parent = Q_NULLPTR);
+    ~AutoTester();
 
     void setup();
 
@@ -74,6 +74,8 @@ private slots:
     void on_setWorkingFolderButton_clicked();
     void on_runNowButton_clicked();
 
+    void on_checkBoxRunLatest_clicked();
+
     void on_updateTestRailRunResultsButton_clicked();
 
     void on_hideTaskbarButton_clicked();
@@ -91,8 +93,8 @@ private slots:
 
 private:
     Ui::AutoTesterClass _ui;
-    Test* _test;
+    Test* _test{ nullptr };
-    TestRunner* _testRunner;
+    TestRunner* _testRunner{ nullptr };
 
     std::vector<Downloader*> _downloaders;
 

tools/auto-tester/src/ui/AutoTester.ui

@@ -6,8 +6,8 @@
    <rect>
     <x>0</x>
     <y>0</y>
-    <width>707</width>
+    <width>737</width>
-    <height>796</height>
+    <height>864</height>
    </rect>
   </property>
   <property name="sizePolicy">
@@ -24,7 +24,7 @@
     <property name="geometry">
      <rect>
       <x>470</x>
-      <y>660</y>
+      <y>750</y>
       <width>100</width>
       <height>40</height>
      </rect>
@@ -36,10 +36,10 @@
    <widget class="QTabWidget" name="tabWidget">
     <property name="geometry">
      <rect>
-      <x>30</x>
+      <x>40</x>
       <y>140</y>
       <width>631</width>
-      <height>501</height>
+      <height>581</height>
      </rect>
     </property>
     <property name="currentIndex">
@@ -196,7 +196,7 @@
       <property name="geometry">
        <rect>
         <x>10</x>
-        <y>70</y>
+        <y>160</y>
         <width>161</width>
         <height>28</height>
        </rect>
@@ -212,7 +212,7 @@
       <property name="geometry">
        <rect>
         <x>20</x>
-        <y>150</y>
+        <y>240</y>
         <width>91</width>
         <height>241</height>
        </rect>
@@ -319,7 +319,7 @@
       <property name="geometry">
        <rect>
         <x>130</x>
-        <y>150</y>
+        <y>240</y>
         <width>161</width>
         <height>191</height>
        </rect>
@@ -443,14 +443,14 @@
        </rect>
       </property>
       <property name="text">
-       <string>#######</string>
+       <string>(not set...)</string>
       </property>
      </widget>
      <widget class="QPlainTextEdit" name="plainTextEdit">
       <property name="geometry">
        <rect>
         <x>300</x>
-        <y>120</y>
+        <y>210</y>
         <width>311</width>
         <height>331</height>
        </rect>
@@ -460,7 +460,7 @@
       <property name="geometry">
        <rect>
         <x>300</x>
-        <y>80</y>
+        <y>170</y>
         <width>41</width>
         <height>31</height>
        </rect>
@@ -473,7 +473,7 @@
       <property name="geometry">
        <rect>
         <x>350</x>
-        <y>80</y>
+        <y>170</y>
         <width>271</width>
         <height>31</height>
        </rect>
@@ -482,6 +482,70 @@
        <string>#######</string>
       </property>
      </widget>
+     <widget class="QCheckBox" name="checkBoxServerless">
+      <property name="geometry">
+       <rect>
+        <x>20</x>
+        <y>70</y>
+        <width>120</width>
+        <height>20</height>
+       </rect>
+      </property>
+      <property name="toolTip">
+       <string>&lt;html&gt;&lt;head/&gt;&lt;body&gt;&lt;p&gt;If unchecked, will not show results during evaluation&lt;/p&gt;&lt;/body&gt;&lt;/html&gt;</string>
+      </property>
+      <property name="text">
+       <string>Serveless</string>
+      </property>
+      <property name="checked">
+       <bool>true</bool>
+      </property>
+     </widget>
+     <widget class="QCheckBox" name="checkBoxRunLatest">
+      <property name="geometry">
+       <rect>
+        <x>20</x>
+        <y>100</y>
+        <width>120</width>
+        <height>20</height>
+       </rect>
+      </property>
+      <property name="toolTip">
+       <string>&lt;html&gt;&lt;head/&gt;&lt;body&gt;&lt;p&gt;If unchecked, will not show results during evaluation&lt;/p&gt;&lt;/body&gt;&lt;/html&gt;</string>
+      </property>
+      <property name="text">
+       <string>Run Latest</string>
+      </property>
+      <property name="checked">
+       <bool>true</bool>
+      </property>
+     </widget>
+     <widget class="QTextEdit" name="urlTextEdit">
+      <property name="enabled">
+       <bool>false</bool>
+      </property>
+      <property name="geometry">
+       <rect>
+        <x>150</x>
+        <y>98</y>
+        <width>461</width>
+        <height>24</height>
+       </rect>
+      </property>
+     </widget>
+     <widget class="QLabel" name="workingFolderLabel_3">
+      <property name="geometry">
+       <rect>
+        <x>128</x>
+        <y>95</y>
+        <width>21</width>
+        <height>31</height>
+       </rect>
+      </property>
+      <property name="text">
+       <string>URL</string>
+      </property>
+     </widget>
     </widget>
     <widget class="QWidget" name="tab_2">
      <attribute name="title">
@@ -651,7 +715,7 @@
     <property name="geometry">
      <rect>
       <x>80</x>
-      <y>670</y>
+      <y>760</y>
       <width>255</width>
       <height>23</height>
      </rect>
@@ -666,7 +730,7 @@
     <rect>
      <x>0</x>
      <y>0</y>
-     <width>707</width>
+     <width>737</width>
      <height>21</height>
     </rect>
    </property>

tools/auto-tester/src/ui/TestRailRunSelectorWindow.cpp

@@ -19,7 +19,6 @@ TestRailRunSelectorWindow::TestRailRunSelectorWindow(QWidget *parent) {
     projectIDLineEdit->setValidator(new QIntValidator(1, 999, this));
 }
 
-
 void TestRailRunSelectorWindow::reset() {
     urlLineEdit->setDisabled(false);
     userLineEdit->setDisabled(false);