Merge branch 'addDailyTests' of https://github.com/NissimHadar/hifi into addDailyTests

tools/auto-tester/AppDataHighFidelity/Interface.json

@@ -86,6 +86,8 @@
     "Developer/Entities/Show Realtime Entity Stats": false,
     "Developer/Hands/Show Hand Targets": false,
     "Developer/Network/Disable Activity Logger": false,
+    "Developer/Network/Send wrong DS connect version": false,
+    "Developer/Network/Send wrong protocol version": false,
     "Developer/Physics/Highlight Simulation Ownership": false,
     "Developer/Physics/Show Bullet Bounding Boxes": false,
     "Developer/Physics/Show Bullet Collision": false,
@@ -118,6 +120,7 @@
     "Developer/Render/Temporal Antialiasing (FXAA if disabled)": true,
     "Developer/Render/Throttle FPS If Not Focus": true,
     "Developer/Render/World Axes": false,
+    "Developer/Show Animation Stats": false,
     "Developer/Show Overlays": true,
     "Developer/Show Statistics": false,
     "Developer/Timing/Log Extra Timing Details": false,
@@ -174,6 +177,8 @@
     "Maximum Texture Memory/8192 MB": false,
     "Maximum Texture Memory/Automatic Texture Memory": true,
     "Network/Disable Activity Logger": false,
+    "Network/Send wrong DS connect version": false,
+    "Network/Send wrong protocol version": false,
     "Perception Neuron/enabled": false,
     "Perception Neuron/serverAddress": "localhost",
     "Perception Neuron/serverPort": 7001,
@@ -267,12 +272,14 @@
     "hmdLODDecreaseFPS": 34,
     "io.highfidelity.attachPoints": "{}",
     "io.highfidelity.isEditing": false,
-    "sessionRunTime": 77,
+    "loginDialogPoppedUp": false,
+    "sessionRunTime": 42,
     "showLightsAndParticlesInEditMode": true,
     "showZonesInEditMode": true,
     "staticJitterBufferFrames": 1,
     "toolbar/com.highfidelity.interface.toolbar.system/desktopHeight": 1059,
     "toolbar/com.highfidelity.interface.toolbar.system/x": 655,
     "toolbar/com.highfidelity.interface.toolbar.system/y": 953,
-    "toolbar/constrainToolbarToCenterX": true
+    "toolbar/constrainToolbarToCenterX": true,
+    "wallet/autoLogout": true
 }

tools/auto-tester/src/Test.cpp

@@ -63,7 +63,7 @@ QString Test::zipAndDeleteTestResultsFolder() {
     return zippedResultsFileName;
 }
 
-bool Test::compareImageLists() {
+int Test::compareImageLists() {
     _progressBar->setMinimum(0);
     _progressBar->setMaximum(_expectedImagesFullFilenames.length() - 1);
     _progressBar->setValue(0);
@@ -71,8 +71,8 @@ bool Test::compareImageLists() {
 
     // Loop over both lists and compare each pair of images
     // Quit loop if user has aborted due to a failed test.
-    bool success{ true };
     bool keepOn{ true };
+    int numberOfFailures{ 0 };
     for (int i = 0; keepOn && i < _expectedImagesFullFilenames.length(); ++i) {
         // First check that images are the same size
         QImage resultImage(_resultImagesFullFilenames[i]);
@@ -91,6 +91,7 @@ bool Test::compareImageLists() {
         }
 
         if (similarityIndex < THRESHOLD) {
+            ++numberOfFailures;
             TestFailure testFailure = TestFailure{
                 (float)similarityIndex,
                 _expectedImagesFullFilenames[i].left(_expectedImagesFullFilenames[i].lastIndexOf("/") + 1), // path to the test (including trailing /)
@@ -102,7 +103,6 @@ bool Test::compareImageLists() {
 
             if (!isInteractiveMode) {
                 appendTestResultsToFile(_testResultsFolderPath, testFailure, _mismatchWindow.getComparisonImage());
-                success = false;
             } else {
                 _mismatchWindow.exec();
 
@@ -111,11 +111,9 @@ bool Test::compareImageLists() {
                         break;
                     case USE_RESPONSE_FAIL:
                         appendTestResultsToFile(_testResultsFolderPath, testFailure, _mismatchWindow.getComparisonImage());
-                        success = false;
                         break;
                     case USER_RESPONSE_ABORT:
                         keepOn = false;
-                        success = false;
                         break;
                     default:
                         assert(false);
@@ -128,7 +126,7 @@ bool Test::compareImageLists() {
     }
 
     _progressBar->setVisible(false);
-    return success;
+    return numberOfFailures;
 }
 
 void Test::appendTestResultsToFile(const QString& _testResultsFolderPath, TestFailure testFailure, QPixmap comparisonImage) {
@@ -190,7 +188,7 @@ void Test::startTestsEvaluation(const bool isRunningFromCommandLine,
     _isRunningInAutomaticTestRun = isRunningInAutomaticTestRun;
 
     if (snapshotDirectory.isNull()) {
-        // Get list of JPEG images in folder, sorted by name
+        // Get list of PNG images in folder, sorted by name
         QString previousSelection = _snapshotDirectory;
         QString parent = previousSelection.left(previousSelection.lastIndexOf('/'));
         if (!parent.isNull() && parent.right(1) != "/") {
@@ -256,10 +254,10 @@ void Test::startTestsEvaluation(const bool isRunningFromCommandLine,
     autoTester->downloadFiles(expectedImagesURLs, _snapshotDirectory, _expectedImagesFilenames, (void *)this);
 }
 void Test::finishTestsEvaluation() {
-    bool success = compareImageLists();
+    int numberOfFailures = compareImageLists();
     
     if (!_isRunningFromCommandLine && !_isRunningInAutomaticTestRun) {
-        if (success) {
+        if (numberOfFailures == 0) {
             QMessageBox::information(0, "Success", "All images are as expected");
         } else {
             QMessageBox::information(0, "Failure", "One or more images are not as expected");
@@ -273,7 +271,7 @@ void Test::finishTestsEvaluation() {
     }
 
     if (_isRunningInAutomaticTestRun) {
-        autoTester->automaticTestRunEvaluationComplete(zippedFolderName);
+        autoTester->automaticTestRunEvaluationComplete(zippedFolderName, numberOfFailures);
     }
 }
 
@@ -593,6 +591,12 @@ bool Test::createMDFile(const QString& directory) {
     }
 
     mdFile.close();
+
+    foreach (auto test, testScriptLines.stepList) {
+        delete test;
+    }
+    testScriptLines.stepList.clear();
+
     return true;
 }
 

tools/auto-tester/src/Test.h

@@ -75,7 +75,7 @@ public:
     void createAllRecursiveScripts();
     void createRecursiveScript(const QString& topLevelDirectory, bool interactiveMode);
 
-    bool compareImageLists();
+    int compareImageLists();
 
     QStringList createListOfAll_imagesInDirectory(const QString& imageFormat, const QString& pathToImageDirectory);
 

tools/auto-tester/src/TestRailInterface.cpp

@@ -367,6 +367,7 @@ void TestRailInterface::createAddTestCasesPythonScript(const QString& testDirect
         QProcess* process = new QProcess();
 
         connect(process, &QProcess::started, this, [=]() { _busyWindow.exec(); });
+        connect(process, SIGNAL(finished(int)), process, SLOT(deleteLater()));
         connect(process, static_cast<void (QProcess::*)(int, QProcess::ExitStatus)>(&QProcess::finished), this,
                 [=](int exitCode, QProcess::ExitStatus exitStatus) { _busyWindow.hide(); });
 
@@ -491,7 +492,7 @@ void TestRailInterface::addRun() {
     ) {
         QProcess* process = new QProcess();
         connect(process, &QProcess::started, this, [=]() { _busyWindow.exec(); });
-
+        connect(process, SIGNAL(finished(int)), process, SLOT(deleteLater()));
         connect(process, static_cast<void (QProcess::*)(int, QProcess::ExitStatus)>(&QProcess::finished), this,
                 [=](int exitCode, QProcess::ExitStatus exitStatus) { _busyWindow.hide(); });
 
@@ -499,6 +500,7 @@ void TestRailInterface::addRun() {
         process->start(_pythonCommand, parameters);
     }
 }
+
 void TestRailInterface::updateRunWithResults() {
     QString filename = _outputDirectory + "/updateRunWithResults.py";
     if (QFile::exists(filename)) {
@@ -578,7 +580,7 @@ void TestRailInterface::updateRunWithResults() {
     ) {
         QProcess* process = new QProcess();
         connect(process, &QProcess::started, this, [=]() { _busyWindow.exec(); });
-
+        connect(process, SIGNAL(finished(int)), process, SLOT(deleteLater()));
         connect(process, static_cast<void (QProcess::*)(int, QProcess::ExitStatus)>(&QProcess::finished), this,
                 [=](int exitCode, QProcess::ExitStatus exitStatus) { _busyWindow.hide(); });
 
@@ -753,6 +755,7 @@ void TestRailInterface::getReleasesFromTestRail() {
     QProcess* process = new QProcess();
     connect(process, static_cast<void (QProcess::*)(int, QProcess::ExitStatus)>(&QProcess::finished), this,
             [=](int exitCode, QProcess::ExitStatus exitStatus) { updateReleasesComboData(exitCode, exitStatus); });
+    connect(process, SIGNAL(finished(int)), process, SLOT(deleteLater()));
 
     QStringList parameters = QStringList() << filename;
     process->start(_pythonCommand, parameters);
@@ -1076,6 +1079,7 @@ void TestRailInterface::getTestSectionsFromTestRail() {
     QProcess* process = new QProcess();
     connect(process, static_cast<void (QProcess::*)(int, QProcess::ExitStatus)>(&QProcess::finished), this,
             [=](int exitCode, QProcess::ExitStatus exitStatus) { updateSectionsComboData(exitCode, exitStatus); });
+    connect(process, SIGNAL(finished(int)), process, SLOT(deleteLater()));
 
     QStringList parameters = QStringList() << filename;
     process->start(_pythonCommand, parameters);
@@ -1114,6 +1118,7 @@ void TestRailInterface::getRunsFromTestRail() {
     QProcess* process = new QProcess();
     connect(process, static_cast<void (QProcess::*)(int, QProcess::ExitStatus)>(&QProcess::finished), this,
             [=](int exitCode, QProcess::ExitStatus exitStatus) { updateRunsComboData(exitCode, exitStatus); });
+    connect(process, SIGNAL(finished(int)), process, SLOT(deleteLater()));
 
     QStringList parameters = QStringList() << filename;
 

tools/auto-tester/src/TestRunner.cpp

@@ -34,17 +34,44 @@ TestRunner::TestRunner(std::vector<QCheckBox*> dayCheckboxes,
                        std::vector<QCheckBox*> timeEditCheckboxes,
                        std::vector<QTimeEdit*> timeEdits,
                        QLabel* workingFolderLabel,
+                       QCheckBox* runServerless,
+                       QCheckBox* runLatest,
+                       QTextEdit* url,
                        QObject* parent) :
-    QObject(parent) 
+    QObject(parent) {
-{
     _dayCheckboxes = dayCheckboxes;
     _timeEditCheckboxes = timeEditCheckboxes;
     _timeEdits = timeEdits;
     _workingFolderLabel = workingFolderLabel;
+    _runServerless = runServerless;
+    _runLatest = runLatest;
+    _url = url;
+
+    installerThread = new QThread();
+    installerWorker = new Worker();
+    installerWorker->moveToThread(installerThread);
+    installerThread->start();
+    connect(this, SIGNAL(startInstaller()), installerWorker, SLOT(runCommand()));
+    connect(installerWorker, SIGNAL(commandComplete()), this, SLOT(installationComplete()));
+
+    interfaceThread = new QThread();
+    interfaceWorker = new Worker();
+    interfaceThread->start();
+    interfaceWorker->moveToThread(interfaceThread);
+    connect(this, SIGNAL(startInterface()), interfaceWorker, SLOT(runCommand()));
+    connect(interfaceWorker, SIGNAL(commandComplete()), this, SLOT(interfaceExecutionComplete()));
 }
 
 TestRunner::~TestRunner() {
-    disconnect(_timer, SIGNAL(timeout()), this, SLOT(checkTime()));
+    delete installerThread;
+    delete interfaceThread;
+
+    delete interfaceThread;
+    delete interfaceWorker;
+
+    if (_timer) {
+        delete _timer;
+    }
 }
 
 void TestRunner::setWorkingFolder() {
@@ -56,7 +83,7 @@ void TestRunner::setWorkingFolder() {
     }
 
     _workingFolder = QFileDialog::getExistingDirectory(nullptr, "Please select a temporary folder for installation", parent,
-                                                    QFileDialog::ShowDirsOnly);
+                                                       QFileDialog::ShowDirsOnly);
 
     // If user canceled then restore previous selection and return
     if (_workingFolder == "") {
@@ -70,7 +97,6 @@ void TestRunner::setWorkingFolder() {
     autoTester->enableRunTabControls();
     _workingFolderLabel->setText(QDir::toNativeSeparators(_workingFolder));
 
-    // The time is checked every 30 seconds for automatic test start
     _timer = new QTimer(this);
     connect(_timer, SIGNAL(timeout()), this, SLOT(checkTime()));
     _timer->start(30 * 1000);  //time specified in ms
@@ -87,7 +113,24 @@ void TestRunner::run() {
     // This will be restored at the end of the tests
     saveExistingHighFidelityAppDataFolder();
 
+<<<<<<< HEAD
     QThread* thread = new QThread;
+=======
+    // Download the latest High Fidelity installer and build XML.
+    QStringList urls;
+    QStringList filenames;
+    if (_runLatest->isChecked()) {
+        _installerFilename = INSTALLER_FILENAME_LATEST;
+
+        urls << INSTALLER_URL_LATEST << BUILD_XML_URL;
+        filenames << _installerFilename << BUILD_XML_FILENAME;
+    } else {
+        QString urlText = _url->toPlainText();
+        urls << urlText;
+        _installerFilename = getInstallerNameFromURL(urlText);
+        filenames << _installerFilename;
+    }
+>>>>>>> 8301b472feeaf857d4273f65eedb97957bc13755
 
     Runner* runner = new Runner();
     runner->moveToThread(thread);
@@ -100,16 +143,25 @@ void TestRunner::run() {
 }
 
 void TestRunner::installerDownloadComplete() {
+<<<<<<< HEAD
     appendLog(QString("Test started at ") + QString::number(runner->_testStartDateTime.time().hour()) + ":" +
               QString("%1").arg(runner->_testStartDateTime.time().minute(), 2, 10, QChar('0')) + ", on " +
               runner->_testStartDateTime.date().toString("ddd, MMM d, yyyy"));
 
     autoTester->updateStatusLabel("Installing");
+=======
+    appendLog(QString("Tests started at ") + QString::number(_testStartDateTime.time().hour()) + ":" +
+              QString("%1").arg(_testStartDateTime.time().minute(), 2, 10, QChar('0')) + ", on " +
+              _testStartDateTime.date().toString("ddd, MMM d, yyyy"));
+
+    updateStatusLabel("Installing");
+>>>>>>> 8301b472feeaf857d4273f65eedb97957bc13755
 
     // Kill any existing processes that would interfere with installation
     killProcesses();
 
     runInstaller();
+<<<<<<< HEAD
 
     createSnapshotFolder();
 
@@ -122,19 +174,35 @@ void TestRunner::installerDownloadComplete() {
     evaluateResults();
 
     // The High Fidelity AppData folder will be restored after evaluation has completed
+=======
+>>>>>>> 8301b472feeaf857d4273f65eedb97957bc13755
 }
 
 void TestRunner::runInstaller() {
     // Qt cannot start an installation process using QProcess::start (Qt Bug 9761)
     // To allow installation, the installer is run using the `system` command
+
     QStringList arguments{ QStringList() << QString("/S") << QString("/D=") + QDir::toNativeSeparators(_installationFolder) };
 
-    QString installerFullPath = _workingFolder + "/" + INSTALLER_FILENAME;
+    QString installerFullPath = _workingFolder + "/" + _installerFilename;
 
     QString commandLine =
         QDir::toNativeSeparators(installerFullPath) + " /S /D=" + QDir::toNativeSeparators(_installationFolder);
 
-    system(commandLine.toStdString().c_str());
+    installerWorker->setCommandLine(commandLine);
+    emit startInstaller();
+}
+
+void TestRunner::installationComplete() {
+    createSnapshotFolder();
+
+    updateStatusLabel("Running tests");
+
+    if (!_runServerless->isChecked()) {
+        startLocalServerProcesses();
+    }
+
+    runInterfaceWithTestScript();
 }
 
 void TestRunner::saveExistingHighFidelityAppDataFolder() {
@@ -144,11 +212,20 @@ void TestRunner::saveExistingHighFidelityAppDataFolder() {
     dataDirectory = qgetenv("USERPROFILE") + "\\AppData\\Roaming";
 #endif
 
-    _appDataFolder = dataDirectory + "\\High Fidelity";
+    if (_runLatest->isChecked()) {
+        _appDataFolder = dataDirectory + "\\High Fidelity";
+    } else {
+        // We are running a PR build
+        _appDataFolder = dataDirectory + "\\High Fidelity - " + getPRNumberFromURL(_url->toPlainText());
+    }
+
+    _savedAppDataFolder = dataDirectory + "/" + UNIQUE_FOLDER_NAME;
+    if (_savedAppDataFolder.exists()) {
+        _savedAppDataFolder.removeRecursively();
+    }
 
     if (_appDataFolder.exists()) {
         // The original folder is saved in a unique name
-        _savedAppDataFolder = dataDirectory + "/" + UNIQUE_FOLDER_NAME;
         _appDataFolder.rename(_appDataFolder.path(), _savedAppDataFolder.path());
     }
 
@@ -240,14 +317,30 @@ void TestRunner::startLocalServerProcesses() {
 }
 
 void TestRunner::runInterfaceWithTestScript() {
-#ifdef Q_OS_WIN
+    QString exeFile = QString("\"") + QDir::toNativeSeparators(_installationFolder) + "\\interface.exe\"";
-    QString commandLine = QString("\"") + QDir::toNativeSeparators(_installationFolder) +
-                          "\\interface.exe\" --url hifi://localhost --testScript https://raw.githubusercontent.com/" + _user +
-                          "/hifi_tests/" + _branch + "/tests/testRecursive.js quitWhenFinished --testResultsLocation " +
-                          _snapshotFolder;
 
-    system(commandLine.toStdString().c_str());
+    QString url = QString("hifi://localhost");
-#endif
+    if (_runServerless->isChecked()) {
+        // Move to an empty area
+        url = url + "/9999,9999,9999/0.0,0.0,0.0,1.0";
+    }
+
+    QString testScript =
+        QString("https://raw.githubusercontent.com/") + _user + "/hifi_tests/" + _branch + "/tests/testRecursive.js";
+
+    QString commandLine = exeFile + " --url " + url + " --testScript " + testScript +
+                          " quitWhenFinished --testResultsLocation " + _snapshotFolder;
+
+    interfaceWorker->setCommandLine(commandLine);
+    emit startInterface();
+}
+
+void TestRunner::interfaceExecutionComplete() {
+    killProcesses();
+
+    evaluateResults();
+
+    // The High Fidelity AppData folder will be restored after evaluation has completed
 }
 
 void TestRunner::evaluateResults() {
@@ -255,15 +348,42 @@ void TestRunner::evaluateResults() {
     autoTester->startTestsEvaluation(false, true, _snapshotFolder, _branch, _user);
 }
 
-void TestRunner::automaticTestRunEvaluationComplete(QString zippedFolder) {
+void TestRunner::automaticTestRunEvaluationComplete(QString zippedFolder, int numberOfFailures) {
     addBuildNumberToResults(zippedFolder);
     restoreHighFidelityAppDataFolder();
 
+<<<<<<< HEAD
     autoTester->updateStatusLabel("Testing complete");
+=======
+    updateStatusLabel("Testing complete");
+
+    QDateTime currentDateTime = QDateTime::currentDateTime();
+
+    QString completionText = QString("Tests completed at ") + QString::number(currentDateTime.time().hour()) + ":" +
+                             QString("%1").arg(currentDateTime.time().minute(), 2, 10, QChar('0')) + ", on " +
+                             currentDateTime.date().toString("ddd, MMM d, yyyy");
+
+    if (numberOfFailures == 0) {
+        completionText += "; no failures";
+    } else if (numberOfFailures == 1) {
+        completionText += "; 1 failure";
+    } else {
+        completionText += QString("; ") + QString::number(numberOfFailures) + " failures";
+    }
+    appendLog(completionText);
+
+>>>>>>> 8301b472feeaf857d4273f65eedb97957bc13755
     _automatedTestIsRunning = false;
 }
 
 void TestRunner::addBuildNumberToResults(QString zippedFolderName) {
+    if (!_runLatest->isChecked()) {
+        QStringList filenameParts = zippedFolderName.split(".");
+        QString augmentedFilename = filenameParts[0] + "(" + getPRNumberFromURL(_url->toPlainText()) + ")." + filenameParts[1];
+        QFile::rename(zippedFolderName, augmentedFilename);
+
+        return;
+    }
     try {
         QDomDocument domDocument;
         QString filename{ _workingFolder + "/" + BUILD_XML_FILENAME };
@@ -419,6 +539,7 @@ void TestRunner::appendLog(const QString& message) {
     autoTester->appendLogWindow(message);
 }
 
+<<<<<<< HEAD
 Runner::Runner() {
 }
 
@@ -439,4 +560,48 @@ void Runner::process() {
 
     // `installerDownloadComplete` will run after download has completed
     emit finished();
+=======
+QString TestRunner::getInstallerNameFromURL(const QString& url) {
+    // An example URL: https://deployment.highfidelity.com/jobs/pr-build/label%3Dwindows/13023/HighFidelity-Beta-Interface-PR14006-be76c43.exe
+    try {
+        QStringList urlParts = url.split("/");
+        int rr = urlParts.size();
+        if (urlParts.size() != 8) {
+            throw "URL not in expected format, should look like `https://deployment.highfidelity.com/jobs/pr-build/label%3Dwindows/13023/HighFidelity-Beta-Interface-PR14006-be76c43.exe`";
+        }
+        return urlParts[urlParts.size() - 1];
+    } catch (QString errorMessage) {
+        QMessageBox::critical(0, "Internal error: " + QString(__FILE__) + ":" + QString::number(__LINE__), errorMessage);
+        exit(-1);
+    } catch (...) {
+        QMessageBox::critical(0, "Internal error: " + QString(__FILE__) + ":" + QString::number(__LINE__), "unknown error");
+        exit(-1);
+    }
+}
+
+QString TestRunner::getPRNumberFromURL(const QString& url) {
+    try {
+        QStringList urlParts = url.split("/");
+        QStringList filenameParts = urlParts[urlParts.size() - 1].split("-");
+        if (filenameParts.size() <= 3) {
+            throw "URL not in expected format, should look like `https://deployment.highfidelity.com/jobs/pr-build/label%3Dwindows/13023/HighFidelity-Beta-Interface-PR14006-be76c43.exe`";
+        }
+        return filenameParts[filenameParts.size() - 2];
+    } catch (QString errorMessage) {
+        QMessageBox::critical(0, "Internal error: " + QString(__FILE__) + ":" + QString::number(__LINE__), errorMessage);
+        exit(-1);
+    } catch (...) {
+        QMessageBox::critical(0, "Internal error: " + QString(__FILE__) + ":" + QString::number(__LINE__), "unknown error");
+        exit(-1);
+    }
+}
+
+void Worker::setCommandLine(const QString& commandLine) {
+    _commandLine = commandLine;
+}
+
+void Worker::runCommand() {
+    system(_commandLine.toStdString().c_str());
+    emit commandComplete();
+>>>>>>> 8301b472feeaf857d4273f65eedb97957bc13755
 }

tools/auto-tester/src/TestRunner.h

@@ -15,11 +15,12 @@
 #include <QDir>
 #include <QLabel>
 #include <QObject>
-#include <QProcess>
+#include <QTextEdit>
+#include <QThread>
 #include <QTimeEdit>
 #include <QTimer>
 
-#include "Downloader.h"
+class Worker;
 
 class Runner;
 
@@ -30,7 +31,11 @@ public:
                         std::vector<QCheckBox*> timeEditCheckboxes,
                         std::vector<QTimeEdit*> timeEdits,
                         QLabel* workingFolderLabel,
+                        QCheckBox* runServerless,
+                        QCheckBox* runLatest,
+                        QTextEdit* url,
                         QObject* parent = 0);
+
     ~TestRunner();
 
     void setWorkingFolder();
@@ -44,31 +49,54 @@ public:
     void restoreHighFidelityAppDataFolder();
 
     void createSnapshotFolder();
+    
     void killProcesses();
     void startLocalServerProcesses();
+    
     void runInterfaceWithTestScript();
+
     void evaluateResults();
-    void automaticTestRunEvaluationComplete(QString zippedFolderName);
+    void automaticTestRunEvaluationComplete(QString zippedFolderName, int numberOfFailures);
     void addBuildNumberToResults(QString zippedFolderName);
 
     void copyFolder(const QString& source, const QString& destination);
 
     void appendLog(const QString& message);
 
+    QString getInstallerNameFromURL(const QString& url);
+    QString getPRNumberFromURL(const QString& url);
+
 private slots:
     void checkTime();
+    void installationComplete();
+    void interfaceExecutionComplete();
+
+signals:
+    void startInstaller();
+    void startInterface();
 
 private:
     bool _automatedTestIsRunning{ false };
 
+    const QString INSTALLER_URL_LATEST{ "http://builds.highfidelity.com/HighFidelity-Beta-latest-dev.exe" };
+    const QString INSTALLER_FILENAME_LATEST{ "HighFidelity-Beta-latest-dev.exe" };
+
+    QString _installerURL;
+    QString _installerFilename;
+    const QString BUILD_XML_URL{ "https://highfidelity.com/dev-builds.xml" };
+    const QString BUILD_XML_FILENAME{ "dev-builds.xml" };
+
     QDir _appDataFolder;
     QDir _savedAppDataFolder;
 
+<<<<<<< HEAD
     QString _snapshotFolder;
 
+=======
+    QString _workingFolder;
+>>>>>>> 8301b472feeaf857d4273f65eedb97957bc13755
     QString _installationFolder;
-
+    QString _snapshotFolder;
-    Downloader* _downloader;
 
     const QString UNIQUE_FOLDER_NAME{ "fgadhcUDHSFaidsfh3478JJJFSDFIUSOEIrf" };
     const QString SNAPSHOT_FOLDER_NAME{ "snapshots" };
@@ -79,6 +107,13 @@ private:
     std::vector<QCheckBox*> _dayCheckboxes;
     std::vector<QCheckBox*> _timeEditCheckboxes;
     std::vector<QTimeEdit*> _timeEdits;
+<<<<<<< HEAD
+=======
+    QLabel* _workingFolderLabel;
+    QCheckBox* _runServerless;
+    QCheckBox* _runLatest;
+    QTextEdit* _url;
+>>>>>>> 8301b472feeaf857d4273f65eedb97957bc13755
 
     QTimer* _timer;
 
@@ -106,6 +141,27 @@ signals:
 
 private:
     QDateTime _testStartDateTime;
+
+    QThread* installerThread;
+    QThread* interfaceThread;
+    Worker* installerWorker;
+    Worker* interfaceWorker;
 };
 
+class Worker : public QObject {
+    Q_OBJECT
+public:
+    void setCommandLine(const QString& commandLine);
+
+public slots:
+    void runCommand();
+
+signals:
+    void commandComplete();
+    void startInstaller();
+    void startInterface();
+
+private:
+    QString _commandLine;
+};
 #endif  // hifi_testRunner_h

tools/auto-tester/src/ui/AutoTester.cpp

@@ -40,7 +40,22 @@ AutoTester::AutoTester(QWidget* parent) : QMainWindow(parent) {
    //// _helpWindow.textBrowser->setText()
 }
 
+AutoTester::~AutoTester() {
+    delete _signalMapper;
+
+    if (_test) {
+        delete _test;
+    }
+
+    if (_testRunner) {
+        delete _testRunner;
+    }
+}
+
 void AutoTester::setup() {
+    if (_test) {
+        delete _test;
+    }
     _test = new Test(_ui.progressBar, _ui.checkBoxInteractiveMode);
 
     std::vector<QCheckBox*> dayCheckboxes;
@@ -64,7 +79,10 @@ void AutoTester::setup() {
     timeEdits.emplace_back(_ui.timeEdit3);
     timeEdits.emplace_back(_ui.timeEdit4);
 
-    _testRunner = new TestRunner(dayCheckboxes, timeEditCheckboxes, timeEdits, _ui.workingFolderLabel);
+    if (_testRunner) {
+        delete _testRunner;
+    }
+    _testRunner = new TestRunner(dayCheckboxes, timeEditCheckboxes, timeEdits, _ui.workingFolderLabel, _ui.checkBoxServerless, _ui.checkBoxRunLatest, _ui.urlTextEdit);
 }
 
 void AutoTester::startTestsEvaluation(const bool isRunningFromCommandLine,
@@ -144,8 +162,12 @@ void AutoTester::on_runNowButton_clicked() {
     _testRunner->run();
 }
 
-void AutoTester::automaticTestRunEvaluationComplete(QString zippedFolderName) {
+void AutoTester::on_checkBoxRunLatest_clicked() {
-    _testRunner->automaticTestRunEvaluationComplete(zippedFolderName);
+    _ui.urlTextEdit->setEnabled(!_ui.checkBoxRunLatest->isChecked());
+}
+
+void AutoTester::automaticTestRunEvaluationComplete(QString zippedFolderName, int numberOfFailures) {
+    _testRunner->automaticTestRunEvaluationComplete(zippedFolderName, numberOfFailures);
 }
 
 void AutoTester::on_updateTestRailRunResultsButton_clicked() {
@@ -213,6 +235,10 @@ void AutoTester::downloadFiles(const QStringList& URLs, const QString& directory
     _ui.progressBar->setValue(0);
     _ui.progressBar->setVisible(true);
 
+    foreach (auto downloader, _downloaders) {
+        delete downloader;
+    }
+
     _downloaders.clear();
     for (int i = 0; i < _numberOfFilesToDownload; ++i) {
         downloadFile(URLs[i]);

tools/auto-tester/src/ui/AutoTester.h

@@ -21,12 +21,12 @@
 #include "HelpWindow.h"
 #include "../TestRunner.h"
 
-
 class AutoTester : public QMainWindow {
     Q_OBJECT
 
 public:
-    AutoTester(QWidget *parent = Q_NULLPTR);
+    AutoTester(QWidget* parent = Q_NULLPTR);
+    ~AutoTester();
 
     void setup();
 
@@ -36,10 +36,10 @@ public:
                               const QString& branch,
                               const QString& user);
 
-    void automaticTestRunEvaluationComplete(QString zippedFolderName);
+    void automaticTestRunEvaluationComplete(QString zippedFolderName, int numberOfFailures);
 
     void downloadFile(const QUrl& url);
-    void downloadFiles(const QStringList& URLs, const QString& directoryName, const QStringList& filenames, void *caller);
+    void downloadFiles(const QStringList& URLs, const QString& directoryName, const QStringList& filenames, void* caller);
 
     void setUserText(const QString& user);
     QString getSelectedUser();
@@ -58,7 +58,7 @@ private slots:
     void on_evaluateTestsButton_clicked();
     void on_createRecursiveScriptButton_clicked();
     void on_createAllRecursiveScriptsButton_clicked();
-	void on_createTestsButton_clicked();
+    void on_createTestsButton_clicked();
 
     void on_createMDFileButton_clicked();
     void on_createAllMDFilesButton_clicked();
@@ -74,6 +74,8 @@ private slots:
     void on_setWorkingFolderButton_clicked();
     void on_runNowButton_clicked();
 
+    void on_checkBoxRunLatest_clicked();
+
     void on_updateTestRailRunResultsButton_clicked();
 
     void on_hideTaskbarButton_clicked();
@@ -91,8 +93,8 @@ private slots:
 
 private:
     Ui::AutoTesterClass _ui;
-    Test* _test;
+    Test* _test{ nullptr };
-    TestRunner* _testRunner;
+    TestRunner* _testRunner{ nullptr };
 
     std::vector<Downloader*> _downloaders;
 
@@ -103,15 +105,15 @@ private:
     // Used to enable passing a parameter to slots
     QSignalMapper* _signalMapper;
 
-    int _numberOfFilesToDownload { 0 };
+    int _numberOfFilesToDownload{ 0 };
-    int _numberOfFilesDownloaded { 0 };
+    int _numberOfFilesDownloaded{ 0 };
-    int _index { 0 };
+    int _index{ 0 };
 
-    bool _isRunningFromCommandline { false };
+    bool _isRunningFromCommandline{ false };
 
     HelpWindow _helpWindow;
 
     void* _caller;
 };
 
-#endif // hifi_AutoTester_h
+#endif  // hifi_AutoTester_h

tools/auto-tester/src/ui/AutoTester.ui

@@ -6,8 +6,8 @@
    <rect>
     <x>0</x>
     <y>0</y>
-    <width>707</width>
+    <width>737</width>
-    <height>796</height>
+    <height>864</height>
    </rect>
   </property>
   <property name="sizePolicy">
@@ -24,7 +24,7 @@
     <property name="geometry">
      <rect>
       <x>470</x>
-      <y>660</y>
+      <y>750</y>
       <width>100</width>
       <height>40</height>
      </rect>
@@ -36,10 +36,10 @@
    <widget class="QTabWidget" name="tabWidget">
     <property name="geometry">
      <rect>
-      <x>30</x>
+      <x>40</x>
       <y>140</y>
       <width>631</width>
-      <height>501</height>
+      <height>581</height>
      </rect>
     </property>
     <property name="currentIndex">
@@ -196,7 +196,7 @@
       <property name="geometry">
        <rect>
         <x>10</x>
-        <y>70</y>
+        <y>160</y>
         <width>161</width>
         <height>28</height>
        </rect>
@@ -212,7 +212,7 @@
       <property name="geometry">
        <rect>
         <x>20</x>
-        <y>150</y>
+        <y>240</y>
         <width>91</width>
         <height>241</height>
        </rect>
@@ -319,7 +319,7 @@
       <property name="geometry">
        <rect>
         <x>130</x>
-        <y>150</y>
+        <y>240</y>
         <width>161</width>
         <height>191</height>
        </rect>
@@ -443,14 +443,14 @@
        </rect>
       </property>
       <property name="text">
-       <string>#######</string>
+       <string>(not set...)</string>
       </property>
      </widget>
      <widget class="QPlainTextEdit" name="plainTextEdit">
       <property name="geometry">
        <rect>
         <x>300</x>
-        <y>120</y>
+        <y>210</y>
         <width>311</width>
         <height>331</height>
        </rect>
@@ -460,7 +460,7 @@
       <property name="geometry">
        <rect>
         <x>300</x>
-        <y>80</y>
+        <y>170</y>
         <width>41</width>
         <height>31</height>
        </rect>
@@ -473,7 +473,7 @@
       <property name="geometry">
        <rect>
         <x>350</x>
-        <y>80</y>
+        <y>170</y>
         <width>271</width>
         <height>31</height>
        </rect>
@@ -482,6 +482,70 @@
        <string>#######</string>
       </property>
      </widget>
+     <widget class="QCheckBox" name="checkBoxServerless">
+      <property name="geometry">
+       <rect>
+        <x>20</x>
+        <y>70</y>
+        <width>120</width>
+        <height>20</height>
+       </rect>
+      </property>
+      <property name="toolTip">
+       <string>&lt;html&gt;&lt;head/&gt;&lt;body&gt;&lt;p&gt;If unchecked, will not show results during evaluation&lt;/p&gt;&lt;/body&gt;&lt;/html&gt;</string>
+      </property>
+      <property name="text">
+       <string>Server-less</string>
+      </property>
+      <property name="checked">
+       <bool>false</bool>
+      </property>
+     </widget>
+     <widget class="QCheckBox" name="checkBoxRunLatest">
+      <property name="geometry">
+       <rect>
+        <x>20</x>
+        <y>100</y>
+        <width>120</width>
+        <height>20</height>
+       </rect>
+      </property>
+      <property name="toolTip">
+       <string>&lt;html&gt;&lt;head/&gt;&lt;body&gt;&lt;p&gt;If unchecked, will not show results during evaluation&lt;/p&gt;&lt;/body&gt;&lt;/html&gt;</string>
+      </property>
+      <property name="text">
+       <string>Run Latest</string>
+      </property>
+      <property name="checked">
+       <bool>true</bool>
+      </property>
+     </widget>
+     <widget class="QTextEdit" name="urlTextEdit">
+      <property name="enabled">
+       <bool>false</bool>
+      </property>
+      <property name="geometry">
+       <rect>
+        <x>150</x>
+        <y>98</y>
+        <width>461</width>
+        <height>28</height>
+       </rect>
+      </property>
+     </widget>
+     <widget class="QLabel" name="workingFolderLabel_3">
+      <property name="geometry">
+       <rect>
+        <x>128</x>
+        <y>95</y>
+        <width>21</width>
+        <height>31</height>
+       </rect>
+      </property>
+      <property name="text">
+       <string>URL</string>
+      </property>
+     </widget>
     </widget>
     <widget class="QWidget" name="tab_2">
      <attribute name="title">
@@ -651,7 +715,7 @@
     <property name="geometry">
      <rect>
       <x>80</x>
-      <y>670</y>
+      <y>760</y>
       <width>255</width>
       <height>23</height>
      </rect>
@@ -666,7 +730,7 @@
     <rect>
      <x>0</x>
      <y>0</y>
-     <width>707</width>
+     <width>737</width>
      <height>21</height>
     </rect>
    </property>

tools/auto-tester/src/ui/TestRailRunSelectorWindow.cpp

@@ -19,7 +19,6 @@ TestRailRunSelectorWindow::TestRailRunSelectorWindow(QWidget *parent) {
     projectIDLineEdit->setValidator(new QIntValidator(1, 999, this));
 }
 
-
 void TestRailRunSelectorWindow::reset() {
     urlLineEdit->setDisabled(false);
     userLineEdit->setDisabled(false);