From 35eb01952408629d992800ccc2f5b2ece6fd9969 Mon Sep 17 00:00:00 2001 From: NissimHadar Date: Thu, 13 Sep 2018 15:34:10 -0700 Subject: [PATCH 1/8] Can run installer in a thread. --- tools/auto-tester/src/TestRunner.cpp | 51 ++++++++++++++++++++-------- tools/auto-tester/src/TestRunner.h | 44 ++++++++++++++++-------- 2 files changed, 66 insertions(+), 29 deletions(-) diff --git a/tools/auto-tester/src/TestRunner.cpp b/tools/auto-tester/src/TestRunner.cpp index dddc30e8a8..0d777fbecb 100644 --- a/tools/auto-tester/src/TestRunner.cpp +++ b/tools/auto-tester/src/TestRunner.cpp @@ -32,6 +32,8 @@ TestRunner::TestRunner(std::vector dayCheckboxes, _timeEditCheckboxes = timeEditCheckboxes; _timeEdits = timeEdits; _workingFolderLabel = workingFolderLabel; + + thread = new QThread(); } TestRunner::~TestRunner() { @@ -103,7 +105,32 @@ void TestRunner::installerDownloadComplete() { killProcesses(); runInstaller(); +} +void TestRunner::runInstaller() { + // Qt cannot start an installation process using QProcess::start (Qt Bug 9761) + // To allow installation, the installer is run using the `system` command + + QStringList arguments{ QStringList() << QString("/S") << QString("/D=") + QDir::toNativeSeparators(_installationFolder) }; + + QString installerFullPath = _workingFolder + "/" + INSTALLER_FILENAME; + + QString commandLine = + QDir::toNativeSeparators(installerFullPath) + " /S /D=" + QDir::toNativeSeparators(_installationFolder); + + + worker = new Worker(commandLine); + worker->moveToThread(thread); + connect(worker, SIGNAL(error(QString)), this, SLOT(errorString(QString))); + connect(thread, SIGNAL(started()), worker, SLOT(process())); + connect(worker, SIGNAL(finished()), this, SLOT(installationComplete())); + connect(worker, SIGNAL(finished()), thread, SLOT(quit())); + connect(worker, SIGNAL(finished()), worker, SLOT(deleteLater())); + connect(thread, SIGNAL(finished()), thread, SLOT(deleteLater())); + thread->start(); +} + +void TestRunner::installationComplete() { createSnapshotFolder(); updateStatusLabel("Running tests"); @@ -117,19 +144,6 @@ void TestRunner::installerDownloadComplete() { // The High Fidelity AppData folder will be restored after evaluation has completed } -void TestRunner::runInstaller() { - // Qt cannot start an installation process using QProcess::start (Qt Bug 9761) - // To allow installation, the installer is run using the `system` command - QStringList arguments{ QStringList() << QString("/S") << QString("/D=") + QDir::toNativeSeparators(_installationFolder) }; - - QString installerFullPath = _workingFolder + "/" + INSTALLER_FILENAME; - - QString commandLine = - QDir::toNativeSeparators(installerFullPath) + " /S /D=" + QDir::toNativeSeparators(_installationFolder); - - system(commandLine.toStdString().c_str()); -} - void TestRunner::saveExistingHighFidelityAppDataFolder() { QString dataDirectory{ "NOT FOUND" }; @@ -414,4 +428,13 @@ void TestRunner::appendLog(const QString& message) { _logFile.close(); autoTester->appendLogWindow(message); -} \ No newline at end of file +} + +Worker::Worker(const QString commandLine) { + _commandLine = commandLine; +} + +void Worker::process() { + system(_commandLine.toStdString().c_str()); + emit finished(); +} diff --git a/tools/auto-tester/src/TestRunner.h b/tools/auto-tester/src/TestRunner.h index e13b0be070..dfb670bb6d 100644 --- a/tools/auto-tester/src/TestRunner.h +++ b/tools/auto-tester/src/TestRunner.h @@ -15,11 +15,11 @@ #include #include #include -#include +#include #include #include -#include "Downloader.h" +class Worker; class TestRunner : public QObject { Q_OBJECT @@ -56,29 +56,27 @@ public: private slots: void checkTime(); + void installationComplete(); private: bool _automatedTestIsRunning{ false }; - QDir _appDataFolder; - QDir _savedAppDataFolder; - - QString _workingFolder; - QString _snapshotFolder; - - QString _installationFolder; - - Downloader* _downloader; - - const QString UNIQUE_FOLDER_NAME{ "fgadhcUDHSFaidsfh3478JJJFSDFIUSOEIrf" }; - const QString SNAPSHOT_FOLDER_NAME{ "snapshots" }; - const QString INSTALLER_URL{ "http://builds.highfidelity.com/HighFidelity-Beta-latest-dev.exe" }; const QString INSTALLER_FILENAME{ "HighFidelity-Beta-latest-dev.exe" }; const QString BUILD_XML_URL{ "https://highfidelity.com/dev-builds.xml" }; const QString BUILD_XML_FILENAME{ "dev-builds.xml" }; + QDir _appDataFolder; + QDir _savedAppDataFolder; + + QString _workingFolder; + QString _installationFolder; + QString _snapshotFolder; + + const QString UNIQUE_FOLDER_NAME{ "fgadhcUDHSFaidsfh3478JJJFSDFIUSOEIrf" }; + const QString SNAPSHOT_FOLDER_NAME{ "snapshots" }; + QString _branch; QString _user; @@ -92,6 +90,22 @@ private: QFile _logFile; QDateTime _testStartDateTime; + + QThread* thread; + Worker* worker; }; +class Worker : public QObject { + Q_OBJECT +public: + Worker(const QString commandLine); +public slots: + void process(); + +signals: + void finished(); + +private: + QString _commandLine; +}; #endif // hifi_testRunner_h \ No newline at end of file From 46b00535c808b098dbff9a9e1783984ee44f5730 Mon Sep 17 00:00:00 2001 From: NissimHadar Date: Thu, 13 Sep 2018 22:59:54 -0700 Subject: [PATCH 2/8] Running all night. --- tools/auto-tester/src/TestRunner.cpp | 60 +++++++++++++++++----------- tools/auto-tester/src/TestRunner.h | 8 +++- 2 files changed, 43 insertions(+), 25 deletions(-) diff --git a/tools/auto-tester/src/TestRunner.cpp b/tools/auto-tester/src/TestRunner.cpp index 0d777fbecb..e135fdae45 100644 --- a/tools/auto-tester/src/TestRunner.cpp +++ b/tools/auto-tester/src/TestRunner.cpp @@ -33,11 +33,8 @@ TestRunner::TestRunner(std::vector dayCheckboxes, _timeEdits = timeEdits; _workingFolderLabel = workingFolderLabel; - thread = new QThread(); -} - -TestRunner::~TestRunner() { - disconnect(_timer, SIGNAL(timeout()), this, SLOT(checkTime())); + installerThread = new QThread(); + interfaceThread = new QThread(); } void TestRunner::setWorkingFolder() { @@ -95,11 +92,11 @@ void TestRunner::run() { } void TestRunner::installerDownloadComplete() { - appendLog(QString("Test started at ") + QString::number(_testStartDateTime.time().hour()) + ":" + + appendLog(QString("Tests started at ") + QString::number(_testStartDateTime.time().hour()) + ":" + QString("%1").arg(_testStartDateTime.time().minute(), 2, 10, QChar('0')) + ", on " + _testStartDateTime.date().toString("ddd, MMM d, yyyy")); - updateStatusLabel("Installing"); + updateStatusLabel("Installing"); // Kill any existing processes that would interfere with installation killProcesses(); @@ -118,30 +115,25 @@ void TestRunner::runInstaller() { QString commandLine = QDir::toNativeSeparators(installerFullPath) + " /S /D=" + QDir::toNativeSeparators(_installationFolder); - worker = new Worker(commandLine); - worker->moveToThread(thread); - connect(worker, SIGNAL(error(QString)), this, SLOT(errorString(QString))); - connect(thread, SIGNAL(started()), worker, SLOT(process())); + + worker->moveToThread(installerThread); + connect(installerThread, SIGNAL(started()), worker, SLOT(process())); connect(worker, SIGNAL(finished()), this, SLOT(installationComplete())); - connect(worker, SIGNAL(finished()), thread, SLOT(quit())); - connect(worker, SIGNAL(finished()), worker, SLOT(deleteLater())); - connect(thread, SIGNAL(finished()), thread, SLOT(deleteLater())); - thread->start(); + installerThread->start(); } void TestRunner::installationComplete() { + disconnect(installerThread, SIGNAL(started()), worker, SLOT(process())); + disconnect(worker, SIGNAL(finished()), this, SLOT(installationComplete())); + delete worker; + createSnapshotFolder(); updateStatusLabel("Running tests"); startLocalServerProcesses(); runInterfaceWithTestScript(); - killProcesses(); - - evaluateResults(); - - // The High Fidelity AppData folder will be restored after evaluation has completed } void TestRunner::saveExistingHighFidelityAppDataFolder() { @@ -247,14 +239,29 @@ void TestRunner::startLocalServerProcesses() { } void TestRunner::runInterfaceWithTestScript() { -#ifdef Q_OS_WIN QString commandLine = QString("\"") + QDir::toNativeSeparators(_installationFolder) + "\\interface.exe\" --url hifi://localhost --testScript https://raw.githubusercontent.com/" + _user + "/hifi_tests/" + _branch + "/tests/testRecursive.js quitWhenFinished --testResultsLocation " + _snapshotFolder; - system(commandLine.toStdString().c_str()); -#endif + worker = new Worker(commandLine); + + worker->moveToThread(interfaceThread); + connect(interfaceThread, SIGNAL(started()), worker, SLOT(process())); + connect(worker, SIGNAL(finished()), this, SLOT(interfaceExecutionComplete())); + interfaceThread->start(); +} + +void TestRunner::interfaceExecutionComplete() { + disconnect(interfaceThread, SIGNAL(started()), worker, SLOT(process())); + disconnect(worker, SIGNAL(finished()), this, SLOT(interfaceExecutionComplete())); + delete worker; + + killProcesses(); + + evaluateResults(); + + // The High Fidelity AppData folder will be restored after evaluation has completed } void TestRunner::evaluateResults() { @@ -267,6 +274,13 @@ void TestRunner::automaticTestRunEvaluationComplete(QString zippedFolder) { restoreHighFidelityAppDataFolder(); updateStatusLabel("Testing complete"); + + QDateTime currentDateTime = QDateTime::currentDateTime(); + + appendLog(QString("Tests completed at ") + QString::number(currentDateTime.time().hour()) + ":" + + QString("%1").arg(currentDateTime.time().minute(), 2, 10, QChar('0')) + ", on " + + currentDateTime.date().toString("ddd, MMM d, yyyy")); + _automatedTestIsRunning = false; } diff --git a/tools/auto-tester/src/TestRunner.h b/tools/auto-tester/src/TestRunner.h index dfb670bb6d..abf4e87880 100644 --- a/tools/auto-tester/src/TestRunner.h +++ b/tools/auto-tester/src/TestRunner.h @@ -29,7 +29,6 @@ public: std::vector timeEdits, QLabel* workingFolderLabel, QObject* parent = 0); - ~TestRunner(); void setWorkingFolder(); @@ -42,9 +41,12 @@ public: void restoreHighFidelityAppDataFolder(); void createSnapshotFolder(); + void killProcesses(); void startLocalServerProcesses(); + void runInterfaceWithTestScript(); + void evaluateResults(); void automaticTestRunEvaluationComplete(QString zippedFolderName); void addBuildNumberToResults(QString zippedFolderName); @@ -57,6 +59,7 @@ public: private slots: void checkTime(); void installationComplete(); + void interfaceExecutionComplete(); private: bool _automatedTestIsRunning{ false }; @@ -91,7 +94,8 @@ private: QDateTime _testStartDateTime; - QThread* thread; + QThread* installerThread; + QThread* interfaceThread; Worker* worker; }; From f046d3b87dda1c42a9d5ac7ee38b7eb9bccf38d0 Mon Sep 17 00:00:00 2001 From: NissimHadar Date: Mon, 17 Sep 2018 13:21:01 -0700 Subject: [PATCH 3/8] Delete allocated memory where needed. Added option to run server-less Added option to select build. --- tools/auto-tester/src/Test.cpp | 6 + tools/auto-tester/src/TestRailInterface.cpp | 9 +- tools/auto-tester/src/TestRunner.cpp | 147 +++++++++++++----- tools/auto-tester/src/TestRunner.h | 34 +++- tools/auto-tester/src/ui/AutoTester.cpp | 28 +++- tools/auto-tester/src/ui/AutoTester.h | 24 +-- tools/auto-tester/src/ui/AutoTester.ui | 92 +++++++++-- .../src/ui/TestRailRunSelectorWindow.cpp | 1 - 8 files changed, 271 insertions(+), 70 deletions(-) diff --git a/tools/auto-tester/src/Test.cpp b/tools/auto-tester/src/Test.cpp index d3aa310d7d..aedf0be3d3 100644 --- a/tools/auto-tester/src/Test.cpp +++ b/tools/auto-tester/src/Test.cpp @@ -593,6 +593,12 @@ bool Test::createMDFile(const QString& directory) { } mdFile.close(); + + foreach (auto test, testScriptLines.stepList) { + delete test; + } + testScriptLines.stepList.clear(); + return true; } diff --git a/tools/auto-tester/src/TestRailInterface.cpp b/tools/auto-tester/src/TestRailInterface.cpp index e6f12100df..b2132cf85e 100644 --- a/tools/auto-tester/src/TestRailInterface.cpp +++ b/tools/auto-tester/src/TestRailInterface.cpp @@ -367,6 +367,7 @@ void TestRailInterface::createAddTestCasesPythonScript(const QString& testDirect QProcess* process = new QProcess(); connect(process, &QProcess::started, this, [=]() { _busyWindow.exec(); }); + connect(process, SIGNAL(finished(int)), process, SLOT(deleteLater())); connect(process, static_cast(&QProcess::finished), this, [=](int exitCode, QProcess::ExitStatus exitStatus) { _busyWindow.hide(); }); @@ -491,7 +492,7 @@ void TestRailInterface::addRun() { ) { QProcess* process = new QProcess(); connect(process, &QProcess::started, this, [=]() { _busyWindow.exec(); }); - + connect(process, SIGNAL(finished(int)), process, SLOT(deleteLater())); connect(process, static_cast(&QProcess::finished), this, [=](int exitCode, QProcess::ExitStatus exitStatus) { _busyWindow.hide(); }); @@ -499,6 +500,7 @@ void TestRailInterface::addRun() { process->start(_pythonCommand, parameters); } } + void TestRailInterface::updateRunWithResults() { QString filename = _outputDirectory + "/updateRunWithResults.py"; if (QFile::exists(filename)) { @@ -578,7 +580,7 @@ void TestRailInterface::updateRunWithResults() { ) { QProcess* process = new QProcess(); connect(process, &QProcess::started, this, [=]() { _busyWindow.exec(); }); - + connect(process, SIGNAL(finished(int)), process, SLOT(deleteLater())); connect(process, static_cast(&QProcess::finished), this, [=](int exitCode, QProcess::ExitStatus exitStatus) { _busyWindow.hide(); }); @@ -753,6 +755,7 @@ void TestRailInterface::getReleasesFromTestRail() { QProcess* process = new QProcess(); connect(process, static_cast(&QProcess::finished), this, [=](int exitCode, QProcess::ExitStatus exitStatus) { updateReleasesComboData(exitCode, exitStatus); }); + connect(process, SIGNAL(finished(int)), process, SLOT(deleteLater())); QStringList parameters = QStringList() << filename; process->start(_pythonCommand, parameters); @@ -1076,6 +1079,7 @@ void TestRailInterface::getTestSectionsFromTestRail() { QProcess* process = new QProcess(); connect(process, static_cast(&QProcess::finished), this, [=](int exitCode, QProcess::ExitStatus exitStatus) { updateSectionsComboData(exitCode, exitStatus); }); + connect(process, SIGNAL(finished(int)), process, SLOT(deleteLater())); QStringList parameters = QStringList() << filename; process->start(_pythonCommand, parameters); @@ -1114,6 +1118,7 @@ void TestRailInterface::getRunsFromTestRail() { QProcess* process = new QProcess(); connect(process, static_cast(&QProcess::finished), this, [=](int exitCode, QProcess::ExitStatus exitStatus) { updateRunsComboData(exitCode, exitStatus); }); + connect(process, SIGNAL(finished(int)), process, SLOT(deleteLater())); QStringList parameters = QStringList() << filename; diff --git a/tools/auto-tester/src/TestRunner.cpp b/tools/auto-tester/src/TestRunner.cpp index e135fdae45..624dce17eb 100644 --- a/tools/auto-tester/src/TestRunner.cpp +++ b/tools/auto-tester/src/TestRunner.cpp @@ -25,16 +25,44 @@ TestRunner::TestRunner(std::vector dayCheckboxes, std::vector timeEditCheckboxes, std::vector timeEdits, QLabel* workingFolderLabel, + QCheckBox* runServerless, + QCheckBox* runLatest, + QTextEdit* url, QObject* parent) : - QObject(parent) -{ + QObject(parent) { _dayCheckboxes = dayCheckboxes; _timeEditCheckboxes = timeEditCheckboxes; _timeEdits = timeEdits; _workingFolderLabel = workingFolderLabel; + _runServerless = runServerless; + _runLatest = runLatest; + _url = url; installerThread = new QThread(); + installerWorker = new Worker(); + installerWorker->moveToThread(installerThread); + installerThread->start(); + connect(this, SIGNAL(startInstaller()), installerWorker, SLOT(runCommand())); + connect(installerWorker, SIGNAL(commandComplete()), this, SLOT(installationComplete())); + interfaceThread = new QThread(); + interfaceWorker = new Worker(); + interfaceThread->start(); + interfaceWorker->moveToThread(interfaceThread); + connect(this, SIGNAL(startInterface()), interfaceWorker, SLOT(runCommand())); + connect(interfaceWorker, SIGNAL(commandComplete()), this, SLOT(interfaceExecutionComplete())); +} + +TestRunner::~TestRunner() { + delete installerThread; + delete interfaceThread; + + delete interfaceThread; + delete interfaceWorker; + + if (_timer) { + delete _timer; + } } void TestRunner::setWorkingFolder() { @@ -46,7 +74,7 @@ void TestRunner::setWorkingFolder() { } _workingFolder = QFileDialog::getExistingDirectory(nullptr, "Please select a temporary folder for installation", parent, - QFileDialog::ShowDirsOnly); + QFileDialog::ShowDirsOnly); // If user canceled then restore previous selection and return if (_workingFolder == "") { @@ -60,7 +88,6 @@ void TestRunner::setWorkingFolder() { autoTester->enableRunTabControls(); _workingFolderLabel->setText(QDir::toNativeSeparators(_workingFolder)); - // The time is checked every 30 seconds for automatic test start _timer = new QTimer(this); connect(_timer, SIGNAL(timeout()), this, SLOT(checkTime())); _timer->start(30 * 1000); //time specified in ms @@ -79,10 +106,19 @@ void TestRunner::run() { // Download the latest High Fidelity installer and build XML. QStringList urls; - urls << INSTALLER_URL << BUILD_XML_URL; - QStringList filenames; - filenames << INSTALLER_FILENAME << BUILD_XML_FILENAME; + if (_runLatest->isChecked()) { + _installerFilename = INSTALLER_FILENAME_LATEST; + + urls << INSTALLER_URL_LATEST << BUILD_XML_URL; + filenames << _installerFilename << BUILD_XML_FILENAME; + } else { + QString urlText = _url->toPlainText(); + urls << urlText; + _installerFilename = getInstallerNameFromURL(urlText); + filenames << _installerFilename; + } + updateStatusLabel("Downloading installer"); @@ -110,29 +146,24 @@ void TestRunner::runInstaller() { QStringList arguments{ QStringList() << QString("/S") << QString("/D=") + QDir::toNativeSeparators(_installationFolder) }; - QString installerFullPath = _workingFolder + "/" + INSTALLER_FILENAME; + QString installerFullPath = _workingFolder + "/" + _installerFilename; QString commandLine = QDir::toNativeSeparators(installerFullPath) + " /S /D=" + QDir::toNativeSeparators(_installationFolder); - worker = new Worker(commandLine); - - worker->moveToThread(installerThread); - connect(installerThread, SIGNAL(started()), worker, SLOT(process())); - connect(worker, SIGNAL(finished()), this, SLOT(installationComplete())); - installerThread->start(); + installerWorker->setCommandLine(commandLine); + emit startInstaller(); } void TestRunner::installationComplete() { - disconnect(installerThread, SIGNAL(started()), worker, SLOT(process())); - disconnect(worker, SIGNAL(finished()), this, SLOT(installationComplete())); - delete worker; - createSnapshotFolder(); updateStatusLabel("Running tests"); - startLocalServerProcesses(); + if (!_runServerless->isChecked()) { + startLocalServerProcesses(); + } + runInterfaceWithTestScript(); } @@ -239,24 +270,28 @@ void TestRunner::startLocalServerProcesses() { } void TestRunner::runInterfaceWithTestScript() { - QString commandLine = QString("\"") + QDir::toNativeSeparators(_installationFolder) + - "\\interface.exe\" --url hifi://localhost --testScript https://raw.githubusercontent.com/" + _user + - "/hifi_tests/" + _branch + "/tests/testRecursive.js quitWhenFinished --testResultsLocation " + - _snapshotFolder; + QString commandLine; - worker = new Worker(commandLine); + if (_runServerless->isChecked()) { + // Move to an empty area + commandLine = + QString("\"") + QDir::toNativeSeparators(_installationFolder) + + "\\interface.exe\" --url hifi://localhost/9999,9999,9999/0.0,0.0,0.0,1.0 --testScript https://raw.githubusercontent.com/" + _user + + "/hifi_tests/" + _branch + "/tests/testRecursive.js quitWhenFinished --testResultsLocation " + + _snapshotFolder; + } else { + // There is no content, so no need to move + commandLine = QString("\"") + QDir::toNativeSeparators(_installationFolder) + + "\\interface.exe\" --url hifi://localhost --testScript https://raw.githubusercontent.com/" + _user + + "/hifi_tests/" + _branch + + "/tests/content/entity/zone/testRecursive.js quitWhenFinished --testResultsLocation " + _snapshotFolder; + } - worker->moveToThread(interfaceThread); - connect(interfaceThread, SIGNAL(started()), worker, SLOT(process())); - connect(worker, SIGNAL(finished()), this, SLOT(interfaceExecutionComplete())); - interfaceThread->start(); + interfaceWorker->setCommandLine(commandLine); + emit startInterface(); } void TestRunner::interfaceExecutionComplete() { - disconnect(interfaceThread, SIGNAL(started()), worker, SLOT(process())); - disconnect(worker, SIGNAL(finished()), this, SLOT(interfaceExecutionComplete())); - delete worker; - killProcesses(); evaluateResults(); @@ -285,6 +320,13 @@ void TestRunner::automaticTestRunEvaluationComplete(QString zippedFolder) { } void TestRunner::addBuildNumberToResults(QString zippedFolderName) { + if (!_runLatest->isChecked()) { + QStringList filenameParts = zippedFolderName.split("."); + QString augmentedFilename = filenameParts[0] + "(" + getPRNumberFromURL(_url->toPlainText()) + ")." + filenameParts[1]; + QFile::rename(zippedFolderName, augmentedFilename); + + return; + } try { QDomDocument domDocument; QString filename{ _workingFolder + "/" + BUILD_XML_FILENAME }; @@ -444,11 +486,46 @@ void TestRunner::appendLog(const QString& message) { autoTester->appendLogWindow(message); } -Worker::Worker(const QString commandLine) { +QString TestRunner::getInstallerNameFromURL(const QString& url) { + // An example URL: https://deployment.highfidelity.com/jobs/pr-build/label%3Dwindows/13023/HighFidelity-Beta-Interface-PR14006-be76c43.exe + try { + QStringList urlParts = url.split("/"); + int rr = urlParts.size(); + if (urlParts.size() != 8) { + throw "URL not in expected format, should look like `https://deployment.highfidelity.com/jobs/pr-build/label%3Dwindows/13023/HighFidelity-Beta-Interface-PR14006-be76c43.exe`"; + } + return urlParts[urlParts.size() - 1]; + } catch (QString errorMessage) { + QMessageBox::critical(0, "Internal error: " + QString(__FILE__) + ":" + QString::number(__LINE__), errorMessage); + exit(-1); + } catch (...) { + QMessageBox::critical(0, "Internal error: " + QString(__FILE__) + ":" + QString::number(__LINE__), "unknown error"); + exit(-1); + } +} + +QString TestRunner::getPRNumberFromURL(const QString& url) { + try { + QStringList urlParts = url.split("/"); + QStringList filenameParts = urlParts[urlParts.size() - 1].split("-"); + if (filenameParts.size() != 5) { + throw "URL not in expected format, should look like `https://deployment.highfidelity.com/jobs/pr-build/label%3Dwindows/13023/HighFidelity-Beta-Interface-PR14006-be76c43.exe`"; + } + return filenameParts[3]; + } catch (QString errorMessage) { + QMessageBox::critical(0, "Internal error: " + QString(__FILE__) + ":" + QString::number(__LINE__), errorMessage); + exit(-1); + } catch (...) { + QMessageBox::critical(0, "Internal error: " + QString(__FILE__) + ":" + QString::number(__LINE__), "unknown error"); + exit(-1); + } +} + +void Worker::setCommandLine(const QString& commandLine) { _commandLine = commandLine; } -void Worker::process() { +void Worker::runCommand() { system(_commandLine.toStdString().c_str()); - emit finished(); + emit commandComplete(); } diff --git a/tools/auto-tester/src/TestRunner.h b/tools/auto-tester/src/TestRunner.h index abf4e87880..0843b438c8 100644 --- a/tools/auto-tester/src/TestRunner.h +++ b/tools/auto-tester/src/TestRunner.h @@ -15,6 +15,7 @@ #include #include #include +#include #include #include #include @@ -28,8 +29,13 @@ public: std::vector timeEditCheckboxes, std::vector timeEdits, QLabel* workingFolderLabel, + QCheckBox* runServerless, + QCheckBox* runLatest, + QTextEdit* url, QObject* parent = 0); + ~TestRunner(); + void setWorkingFolder(); void run(); @@ -56,17 +62,26 @@ public: void updateStatusLabel(const QString& message); void appendLog(const QString& message); + QString getInstallerNameFromURL(const QString& url); + QString getPRNumberFromURL(const QString& url); + private slots: void checkTime(); void installationComplete(); void interfaceExecutionComplete(); +signals: + void startInstaller(); + void startInterface(); + private: bool _automatedTestIsRunning{ false }; - const QString INSTALLER_URL{ "http://builds.highfidelity.com/HighFidelity-Beta-latest-dev.exe" }; - const QString INSTALLER_FILENAME{ "HighFidelity-Beta-latest-dev.exe" }; + const QString INSTALLER_URL_LATEST{ "http://builds.highfidelity.com/HighFidelity-Beta-latest-dev.exe" }; + const QString INSTALLER_FILENAME_LATEST{ "HighFidelity-Beta-latest-dev.exe" }; + QString _installerURL; + QString _installerFilename; const QString BUILD_XML_URL{ "https://highfidelity.com/dev-builds.xml" }; const QString BUILD_XML_FILENAME{ "dev-builds.xml" }; @@ -87,6 +102,9 @@ private: std::vector _timeEditCheckboxes; std::vector _timeEdits; QLabel* _workingFolderLabel; + QCheckBox* _runServerless; + QCheckBox* _runLatest; + QTextEdit* _url; QTimer* _timer; @@ -96,18 +114,22 @@ private: QThread* installerThread; QThread* interfaceThread; - Worker* worker; + Worker* installerWorker; + Worker* interfaceWorker; }; class Worker : public QObject { Q_OBJECT public: - Worker(const QString commandLine); + void setCommandLine(const QString& commandLine); + public slots: - void process(); + void runCommand(); signals: - void finished(); + void commandComplete(); + void startInstaller(); + void startInterface(); private: QString _commandLine; diff --git a/tools/auto-tester/src/ui/AutoTester.cpp b/tools/auto-tester/src/ui/AutoTester.cpp index 05eee2957a..3a438a5fb9 100644 --- a/tools/auto-tester/src/ui/AutoTester.cpp +++ b/tools/auto-tester/src/ui/AutoTester.cpp @@ -40,7 +40,22 @@ AutoTester::AutoTester(QWidget* parent) : QMainWindow(parent) { //// _helpWindow.textBrowser->setText() } +AutoTester::~AutoTester() { + delete _signalMapper; + + if (_test) { + delete _test; + } + + if (_testRunner) { + delete _testRunner; + } +} + void AutoTester::setup() { + if (_test) { + delete _test; + } _test = new Test(_ui.progressBar, _ui.checkBoxInteractiveMode); std::vector dayCheckboxes; @@ -64,7 +79,10 @@ void AutoTester::setup() { timeEdits.emplace_back(_ui.timeEdit3); timeEdits.emplace_back(_ui.timeEdit4); - _testRunner = new TestRunner(dayCheckboxes, timeEditCheckboxes, timeEdits, _ui.workingFolderLabel); + if (_testRunner) { + delete _testRunner; + } + _testRunner = new TestRunner(dayCheckboxes, timeEditCheckboxes, timeEdits, _ui.workingFolderLabel, _ui.checkBoxServerless, _ui.checkBoxRunLatest, _ui.urlTextEdit); } void AutoTester::startTestsEvaluation(const bool isRunningFromCommandLine, @@ -144,6 +162,10 @@ void AutoTester::on_runNowButton_clicked() { _testRunner->run(); } +void AutoTester::on_checkBoxRunLatest_clicked() { + _ui.urlTextEdit->setEnabled(!_ui.checkBoxRunLatest->isChecked()); +} + void AutoTester::automaticTestRunEvaluationComplete(QString zippedFolderName) { _testRunner->automaticTestRunEvaluationComplete(zippedFolderName); } @@ -213,6 +235,10 @@ void AutoTester::downloadFiles(const QStringList& URLs, const QString& directory _ui.progressBar->setValue(0); _ui.progressBar->setVisible(true); + foreach (auto downloader, _downloaders) { + delete downloader; + } + _downloaders.clear(); for (int i = 0; i < _numberOfFilesToDownload; ++i) { downloadFile(URLs[i]); diff --git a/tools/auto-tester/src/ui/AutoTester.h b/tools/auto-tester/src/ui/AutoTester.h index de0622c670..f59d39e851 100644 --- a/tools/auto-tester/src/ui/AutoTester.h +++ b/tools/auto-tester/src/ui/AutoTester.h @@ -21,12 +21,12 @@ #include "HelpWindow.h" #include "../TestRunner.h" - class AutoTester : public QMainWindow { Q_OBJECT public: - AutoTester(QWidget *parent = Q_NULLPTR); + AutoTester(QWidget* parent = Q_NULLPTR); + ~AutoTester(); void setup(); @@ -39,7 +39,7 @@ public: void automaticTestRunEvaluationComplete(QString zippedFolderName); void downloadFile(const QUrl& url); - void downloadFiles(const QStringList& URLs, const QString& directoryName, const QStringList& filenames, void *caller); + void downloadFiles(const QStringList& URLs, const QString& directoryName, const QStringList& filenames, void* caller); void setUserText(const QString& user); QString getSelectedUser(); @@ -58,7 +58,7 @@ private slots: void on_evaluateTestsButton_clicked(); void on_createRecursiveScriptButton_clicked(); void on_createAllRecursiveScriptsButton_clicked(); - void on_createTestsButton_clicked(); + void on_createTestsButton_clicked(); void on_createMDFileButton_clicked(); void on_createAllMDFilesButton_clicked(); @@ -74,6 +74,8 @@ private slots: void on_setWorkingFolderButton_clicked(); void on_runNowButton_clicked(); + void on_checkBoxRunLatest_clicked(); + void on_updateTestRailRunResultsButton_clicked(); void on_hideTaskbarButton_clicked(); @@ -91,8 +93,8 @@ private slots: private: Ui::AutoTesterClass _ui; - Test* _test; - TestRunner* _testRunner; + Test* _test{ nullptr }; + TestRunner* _testRunner{ nullptr }; std::vector _downloaders; @@ -103,15 +105,15 @@ private: // Used to enable passing a parameter to slots QSignalMapper* _signalMapper; - int _numberOfFilesToDownload { 0 }; - int _numberOfFilesDownloaded { 0 }; - int _index { 0 }; + int _numberOfFilesToDownload{ 0 }; + int _numberOfFilesDownloaded{ 0 }; + int _index{ 0 }; - bool _isRunningFromCommandline { false }; + bool _isRunningFromCommandline{ false }; HelpWindow _helpWindow; void* _caller; }; -#endif // hifi_AutoTester_h \ No newline at end of file +#endif // hifi_AutoTester_h \ No newline at end of file diff --git a/tools/auto-tester/src/ui/AutoTester.ui b/tools/auto-tester/src/ui/AutoTester.ui index fa9569a768..e241acc6f3 100644 --- a/tools/auto-tester/src/ui/AutoTester.ui +++ b/tools/auto-tester/src/ui/AutoTester.ui @@ -6,8 +6,8 @@ 0 0 - 707 - 796 + 737 + 864 @@ -24,7 +24,7 @@ 470 - 660 + 750 100 40 @@ -36,10 +36,10 @@ - 30 + 40 140 631 - 501 + 581 @@ -196,7 +196,7 @@ 10 - 70 + 160 161 28 @@ -212,7 +212,7 @@ 20 - 150 + 240 91 241 @@ -319,7 +319,7 @@ 130 - 150 + 240 161 191 @@ -443,14 +443,14 @@ - ####### + (not set...) 300 - 120 + 210 311 331 @@ -460,7 +460,7 @@ 300 - 80 + 170 41 31 @@ -473,7 +473,7 @@ 350 - 80 + 170 271 31 @@ -482,6 +482,70 @@ ####### + + + + 20 + 70 + 120 + 20 + + + + <html><head/><body><p>If unchecked, will not show results during evaluation</p></body></html> + + + Serveless + + + true + + + + + + 20 + 100 + 120 + 20 + + + + <html><head/><body><p>If unchecked, will not show results during evaluation</p></body></html> + + + Run Latest + + + true + + + + + false + + + + 150 + 98 + 461 + 24 + + + + + + + 128 + 95 + 21 + 31 + + + + URL + + @@ -651,7 +715,7 @@ 80 - 670 + 760 255 23 @@ -666,7 +730,7 @@ 0 0 - 707 + 737 21 diff --git a/tools/auto-tester/src/ui/TestRailRunSelectorWindow.cpp b/tools/auto-tester/src/ui/TestRailRunSelectorWindow.cpp index 54a3985a8b..2247fe33cc 100644 --- a/tools/auto-tester/src/ui/TestRailRunSelectorWindow.cpp +++ b/tools/auto-tester/src/ui/TestRailRunSelectorWindow.cpp @@ -19,7 +19,6 @@ TestRailRunSelectorWindow::TestRailRunSelectorWindow(QWidget *parent) { projectIDLineEdit->setValidator(new QIntValidator(1, 999, this)); } - void TestRailRunSelectorWindow::reset() { urlLineEdit->setDisabled(false); userLineEdit->setDisabled(false); From 0c3e3e977e9d27555070f4e3526740a130e8cf8e Mon Sep 17 00:00:00 2001 From: NissimHadar Date: Mon, 17 Sep 2018 16:06:11 -0700 Subject: [PATCH 4/8] Corrected pulling PR number from the executable name. --- tools/auto-tester/src/Test.cpp | 2 +- tools/auto-tester/src/TestRunner.cpp | 26 ++++++++++++-------------- tools/auto-tester/src/ui/AutoTester.ui | 4 ++-- 3 files changed, 15 insertions(+), 17 deletions(-) diff --git a/tools/auto-tester/src/Test.cpp b/tools/auto-tester/src/Test.cpp index aedf0be3d3..9d77d4ed60 100644 --- a/tools/auto-tester/src/Test.cpp +++ b/tools/auto-tester/src/Test.cpp @@ -190,7 +190,7 @@ void Test::startTestsEvaluation(const bool isRunningFromCommandLine, _isRunningInAutomaticTestRun = isRunningInAutomaticTestRun; if (snapshotDirectory.isNull()) { - // Get list of JPEG images in folder, sorted by name + // Get list of PNG images in folder, sorted by name QString previousSelection = _snapshotDirectory; QString parent = previousSelection.left(previousSelection.lastIndexOf('/')); if (!parent.isNull() && parent.right(1) != "/") { diff --git a/tools/auto-tester/src/TestRunner.cpp b/tools/auto-tester/src/TestRunner.cpp index 624dce17eb..71199782e9 100644 --- a/tools/auto-tester/src/TestRunner.cpp +++ b/tools/auto-tester/src/TestRunner.cpp @@ -270,23 +270,21 @@ void TestRunner::startLocalServerProcesses() { } void TestRunner::runInterfaceWithTestScript() { - QString commandLine; + QString exeFile = QString("\"") + QDir::toNativeSeparators(_installationFolder) + + "\\interface.exe\""; + QString url = QString("hifi://localhost"); if (_runServerless->isChecked()) { // Move to an empty area - commandLine = - QString("\"") + QDir::toNativeSeparators(_installationFolder) + - "\\interface.exe\" --url hifi://localhost/9999,9999,9999/0.0,0.0,0.0,1.0 --testScript https://raw.githubusercontent.com/" + _user + - "/hifi_tests/" + _branch + "/tests/testRecursive.js quitWhenFinished --testResultsLocation " + - _snapshotFolder; - } else { - // There is no content, so no need to move - commandLine = QString("\"") + QDir::toNativeSeparators(_installationFolder) + - "\\interface.exe\" --url hifi://localhost --testScript https://raw.githubusercontent.com/" + _user + - "/hifi_tests/" + _branch + - "/tests/content/entity/zone/testRecursive.js quitWhenFinished --testResultsLocation " + _snapshotFolder; + url = url + "/9999,9999,9999/0.0,0.0,0.0,1.0"; } + QString testScript = + QString("https://raw.githubusercontent.com/") + _user + "/hifi_tests/" + _branch + "/tests/testRecursive.js"; + + QString commandLine = exeFile + " --url " + url + " --testScript " + testScript + + " quitWhenFinished --testResultsLocation " + _snapshotFolder; + interfaceWorker->setCommandLine(commandLine); emit startInterface(); } @@ -508,10 +506,10 @@ QString TestRunner::getPRNumberFromURL(const QString& url) { try { QStringList urlParts = url.split("/"); QStringList filenameParts = urlParts[urlParts.size() - 1].split("-"); - if (filenameParts.size() != 5) { + if (filenameParts.size() <= 3) { throw "URL not in expected format, should look like `https://deployment.highfidelity.com/jobs/pr-build/label%3Dwindows/13023/HighFidelity-Beta-Interface-PR14006-be76c43.exe`"; } - return filenameParts[3]; + return filenameParts[filenameParts.size() - 2]; } catch (QString errorMessage) { QMessageBox::critical(0, "Internal error: " + QString(__FILE__) + ":" + QString::number(__LINE__), errorMessage); exit(-1); diff --git a/tools/auto-tester/src/ui/AutoTester.ui b/tools/auto-tester/src/ui/AutoTester.ui index e241acc6f3..5357a7e612 100644 --- a/tools/auto-tester/src/ui/AutoTester.ui +++ b/tools/auto-tester/src/ui/AutoTester.ui @@ -498,7 +498,7 @@ Serveless - true + false @@ -529,7 +529,7 @@ 150 98 461 - 24 + 28 From f872a95a7a9901900ecb730aaa814eea8473076b Mon Sep 17 00:00:00 2001 From: NissimHadar Date: Tue, 18 Sep 2018 08:49:11 -0700 Subject: [PATCH 5/8] Added number of failures to log. --- tools/auto-tester/src/Test.cpp | 16 ++++++------- tools/auto-tester/src/Test.h | 2 +- tools/auto-tester/src/TestRunner.cpp | 30 +++++++++++++++++-------- tools/auto-tester/src/TestRunner.h | 2 +- tools/auto-tester/src/ui/AutoTester.cpp | 4 ++-- tools/auto-tester/src/ui/AutoTester.h | 2 +- tools/auto-tester/src/ui/AutoTester.ui | 2 +- 7 files changed, 34 insertions(+), 24 deletions(-) diff --git a/tools/auto-tester/src/Test.cpp b/tools/auto-tester/src/Test.cpp index 9d77d4ed60..5e3dd50650 100644 --- a/tools/auto-tester/src/Test.cpp +++ b/tools/auto-tester/src/Test.cpp @@ -63,7 +63,7 @@ QString Test::zipAndDeleteTestResultsFolder() { return zippedResultsFileName; } -bool Test::compareImageLists() { +int Test::compareImageLists() { _progressBar->setMinimum(0); _progressBar->setMaximum(_expectedImagesFullFilenames.length() - 1); _progressBar->setValue(0); @@ -71,8 +71,8 @@ bool Test::compareImageLists() { // Loop over both lists and compare each pair of images // Quit loop if user has aborted due to a failed test. - bool success{ true }; bool keepOn{ true }; + int numberOfFailures{ 0 }; for (int i = 0; keepOn && i < _expectedImagesFullFilenames.length(); ++i) { // First check that images are the same size QImage resultImage(_resultImagesFullFilenames[i]); @@ -91,6 +91,7 @@ bool Test::compareImageLists() { } if (similarityIndex < THRESHOLD) { + ++numberOfFailures; TestFailure testFailure = TestFailure{ (float)similarityIndex, _expectedImagesFullFilenames[i].left(_expectedImagesFullFilenames[i].lastIndexOf("/") + 1), // path to the test (including trailing /) @@ -102,7 +103,6 @@ bool Test::compareImageLists() { if (!isInteractiveMode) { appendTestResultsToFile(_testResultsFolderPath, testFailure, _mismatchWindow.getComparisonImage()); - success = false; } else { _mismatchWindow.exec(); @@ -111,11 +111,9 @@ bool Test::compareImageLists() { break; case USE_RESPONSE_FAIL: appendTestResultsToFile(_testResultsFolderPath, testFailure, _mismatchWindow.getComparisonImage()); - success = false; break; case USER_RESPONSE_ABORT: keepOn = false; - success = false; break; default: assert(false); @@ -128,7 +126,7 @@ bool Test::compareImageLists() { } _progressBar->setVisible(false); - return success; + return numberOfFailures; } void Test::appendTestResultsToFile(const QString& _testResultsFolderPath, TestFailure testFailure, QPixmap comparisonImage) { @@ -256,10 +254,10 @@ void Test::startTestsEvaluation(const bool isRunningFromCommandLine, autoTester->downloadFiles(expectedImagesURLs, _snapshotDirectory, _expectedImagesFilenames, (void *)this); } void Test::finishTestsEvaluation() { - bool success = compareImageLists(); + int numberOfFailures = compareImageLists(); if (!_isRunningFromCommandLine && !_isRunningInAutomaticTestRun) { - if (success) { + if (numberOfFailures == 0) { QMessageBox::information(0, "Success", "All images are as expected"); } else { QMessageBox::information(0, "Failure", "One or more images are not as expected"); @@ -273,7 +271,7 @@ void Test::finishTestsEvaluation() { } if (_isRunningInAutomaticTestRun) { - autoTester->automaticTestRunEvaluationComplete(zippedFolderName); + autoTester->automaticTestRunEvaluationComplete(zippedFolderName, numberOfFailures); } } diff --git a/tools/auto-tester/src/Test.h b/tools/auto-tester/src/Test.h index c6ef83e7e8..8ea00c909b 100644 --- a/tools/auto-tester/src/Test.h +++ b/tools/auto-tester/src/Test.h @@ -75,7 +75,7 @@ public: void createAllRecursiveScripts(); void createRecursiveScript(const QString& topLevelDirectory, bool interactiveMode); - bool compareImageLists(); + int compareImageLists(); QStringList createListOfAll_imagesInDirectory(const QString& imageFormat, const QString& pathToImageDirectory); diff --git a/tools/auto-tester/src/TestRunner.cpp b/tools/auto-tester/src/TestRunner.cpp index 71199782e9..89b9bf27d6 100644 --- a/tools/auto-tester/src/TestRunner.cpp +++ b/tools/auto-tester/src/TestRunner.cpp @@ -119,7 +119,6 @@ void TestRunner::run() { filenames << _installerFilename; } - updateStatusLabel("Downloading installer"); autoTester->downloadFiles(urls, _workingFolder, filenames, (void*)this); @@ -174,7 +173,12 @@ void TestRunner::saveExistingHighFidelityAppDataFolder() { dataDirectory = qgetenv("USERPROFILE") + "\\AppData\\Roaming"; #endif - _appDataFolder = dataDirectory + "\\High Fidelity"; + if (!_runLatest->isChecked()) { + // We are running a PR build + _appDataFolder = dataDirectory + "\\High Fidelity - " + getPRNumberFromURL(_url->toPlainText()); + } else { + _appDataFolder = dataDirectory + "\\High Fidelity"; + } if (_appDataFolder.exists()) { // The original folder is saved in a unique name @@ -270,8 +274,7 @@ void TestRunner::startLocalServerProcesses() { } void TestRunner::runInterfaceWithTestScript() { - QString exeFile = QString("\"") + QDir::toNativeSeparators(_installationFolder) + - "\\interface.exe\""; + QString exeFile = QString("\"") + QDir::toNativeSeparators(_installationFolder) + "\\interface.exe\""; QString url = QString("hifi://localhost"); if (_runServerless->isChecked()) { @@ -302,7 +305,7 @@ void TestRunner::evaluateResults() { autoTester->startTestsEvaluation(false, true, _snapshotFolder, _branch, _user); } -void TestRunner::automaticTestRunEvaluationComplete(QString zippedFolder) { +void TestRunner::automaticTestRunEvaluationComplete(QString zippedFolder, int numberOfFailures) { addBuildNumberToResults(zippedFolder); restoreHighFidelityAppDataFolder(); @@ -310,9 +313,18 @@ void TestRunner::automaticTestRunEvaluationComplete(QString zippedFolder) { QDateTime currentDateTime = QDateTime::currentDateTime(); - appendLog(QString("Tests completed at ") + QString::number(currentDateTime.time().hour()) + ":" + - QString("%1").arg(currentDateTime.time().minute(), 2, 10, QChar('0')) + ", on " + - currentDateTime.date().toString("ddd, MMM d, yyyy")); + QString completionText = QString("Tests completed at ") + QString::number(currentDateTime.time().hour()) + ":" + + QString("%1").arg(currentDateTime.time().minute(), 2, 10, QChar('0')) + ", on " + + currentDateTime.date().toString("ddd, MMM d, yyyy"); + + if (numberOfFailures == 0) { + completionText += "; no failures"; + } else if (numberOfFailures == 1) { + completionText += "; 1 failure"; + } else { + completionText += QString("; ") + QString::number(numberOfFailures) + " failures"; + } + appendLog(completionText); _automatedTestIsRunning = false; } @@ -488,7 +500,7 @@ QString TestRunner::getInstallerNameFromURL(const QString& url) { // An example URL: https://deployment.highfidelity.com/jobs/pr-build/label%3Dwindows/13023/HighFidelity-Beta-Interface-PR14006-be76c43.exe try { QStringList urlParts = url.split("/"); - int rr = urlParts.size(); + int rr = urlParts.size(); if (urlParts.size() != 8) { throw "URL not in expected format, should look like `https://deployment.highfidelity.com/jobs/pr-build/label%3Dwindows/13023/HighFidelity-Beta-Interface-PR14006-be76c43.exe`"; } diff --git a/tools/auto-tester/src/TestRunner.h b/tools/auto-tester/src/TestRunner.h index 0843b438c8..f54ae98e65 100644 --- a/tools/auto-tester/src/TestRunner.h +++ b/tools/auto-tester/src/TestRunner.h @@ -54,7 +54,7 @@ public: void runInterfaceWithTestScript(); void evaluateResults(); - void automaticTestRunEvaluationComplete(QString zippedFolderName); + void automaticTestRunEvaluationComplete(QString zippedFolderName, int numberOfFailures); void addBuildNumberToResults(QString zippedFolderName); void copyFolder(const QString& source, const QString& destination); diff --git a/tools/auto-tester/src/ui/AutoTester.cpp b/tools/auto-tester/src/ui/AutoTester.cpp index 3a438a5fb9..502a447641 100644 --- a/tools/auto-tester/src/ui/AutoTester.cpp +++ b/tools/auto-tester/src/ui/AutoTester.cpp @@ -166,8 +166,8 @@ void AutoTester::on_checkBoxRunLatest_clicked() { _ui.urlTextEdit->setEnabled(!_ui.checkBoxRunLatest->isChecked()); } -void AutoTester::automaticTestRunEvaluationComplete(QString zippedFolderName) { - _testRunner->automaticTestRunEvaluationComplete(zippedFolderName); +void AutoTester::automaticTestRunEvaluationComplete(QString zippedFolderName, int numberOfFailures) { + _testRunner->automaticTestRunEvaluationComplete(zippedFolderName, numberOfFailures); } void AutoTester::on_updateTestRailRunResultsButton_clicked() { diff --git a/tools/auto-tester/src/ui/AutoTester.h b/tools/auto-tester/src/ui/AutoTester.h index f59d39e851..939a03acf4 100644 --- a/tools/auto-tester/src/ui/AutoTester.h +++ b/tools/auto-tester/src/ui/AutoTester.h @@ -36,7 +36,7 @@ public: const QString& branch, const QString& user); - void automaticTestRunEvaluationComplete(QString zippedFolderName); + void automaticTestRunEvaluationComplete(QString zippedFolderName, int numberOfFailures); void downloadFile(const QUrl& url); void downloadFiles(const QStringList& URLs, const QString& directoryName, const QStringList& filenames, void* caller); diff --git a/tools/auto-tester/src/ui/AutoTester.ui b/tools/auto-tester/src/ui/AutoTester.ui index 5357a7e612..bfa9ca587e 100644 --- a/tools/auto-tester/src/ui/AutoTester.ui +++ b/tools/auto-tester/src/ui/AutoTester.ui @@ -495,7 +495,7 @@ <html><head/><body><p>If unchecked, will not show results during evaluation</p></body></html> - Serveless + Server-less false From 69f5e9110a14cd8edac906312c8178b0656fa424 Mon Sep 17 00:00:00 2001 From: NissimHadar Date: Tue, 18 Sep 2018 10:16:42 -0700 Subject: [PATCH 6/8] Minor cleanup --- tools/auto-tester/src/TestRunner.cpp | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/tools/auto-tester/src/TestRunner.cpp b/tools/auto-tester/src/TestRunner.cpp index 89b9bf27d6..eca851a8c9 100644 --- a/tools/auto-tester/src/TestRunner.cpp +++ b/tools/auto-tester/src/TestRunner.cpp @@ -173,11 +173,11 @@ void TestRunner::saveExistingHighFidelityAppDataFolder() { dataDirectory = qgetenv("USERPROFILE") + "\\AppData\\Roaming"; #endif - if (!_runLatest->isChecked()) { + if (_runLatest->isChecked()) { + _appDataFolder = dataDirectory + "\\High Fidelity"; + } else { // We are running a PR build _appDataFolder = dataDirectory + "\\High Fidelity - " + getPRNumberFromURL(_url->toPlainText()); - } else { - _appDataFolder = dataDirectory + "\\High Fidelity"; } if (_appDataFolder.exists()) { From 719176b931d690f35f3555fee4ae142ecefcff5b Mon Sep 17 00:00:00 2001 From: NissimHadar Date: Tue, 18 Sep 2018 12:01:13 -0700 Subject: [PATCH 7/8] Newer version. --- tools/auto-tester/AppDataHighFidelity/Interface.json | 11 +++++++++-- 1 file changed, 9 insertions(+), 2 deletions(-) diff --git a/tools/auto-tester/AppDataHighFidelity/Interface.json b/tools/auto-tester/AppDataHighFidelity/Interface.json index 429d6f109e..ca91a092c8 100644 --- a/tools/auto-tester/AppDataHighFidelity/Interface.json +++ b/tools/auto-tester/AppDataHighFidelity/Interface.json @@ -86,6 +86,8 @@ "Developer/Entities/Show Realtime Entity Stats": false, "Developer/Hands/Show Hand Targets": false, "Developer/Network/Disable Activity Logger": false, + "Developer/Network/Send wrong DS connect version": false, + "Developer/Network/Send wrong protocol version": false, "Developer/Physics/Highlight Simulation Ownership": false, "Developer/Physics/Show Bullet Bounding Boxes": false, "Developer/Physics/Show Bullet Collision": false, @@ -118,6 +120,7 @@ "Developer/Render/Temporal Antialiasing (FXAA if disabled)": true, "Developer/Render/Throttle FPS If Not Focus": true, "Developer/Render/World Axes": false, + "Developer/Show Animation Stats": false, "Developer/Show Overlays": true, "Developer/Show Statistics": false, "Developer/Timing/Log Extra Timing Details": false, @@ -174,6 +177,8 @@ "Maximum Texture Memory/8192 MB": false, "Maximum Texture Memory/Automatic Texture Memory": true, "Network/Disable Activity Logger": false, + "Network/Send wrong DS connect version": false, + "Network/Send wrong protocol version": false, "Perception Neuron/enabled": false, "Perception Neuron/serverAddress": "localhost", "Perception Neuron/serverPort": 7001, @@ -267,12 +272,14 @@ "hmdLODDecreaseFPS": 34, "io.highfidelity.attachPoints": "{}", "io.highfidelity.isEditing": false, - "sessionRunTime": 77, + "loginDialogPoppedUp": false, + "sessionRunTime": 42, "showLightsAndParticlesInEditMode": true, "showZonesInEditMode": true, "staticJitterBufferFrames": 1, "toolbar/com.highfidelity.interface.toolbar.system/desktopHeight": 1059, "toolbar/com.highfidelity.interface.toolbar.system/x": 655, "toolbar/com.highfidelity.interface.toolbar.system/y": 953, - "toolbar/constrainToolbarToCenterX": true + "toolbar/constrainToolbarToCenterX": true, + "wallet/autoLogout": true } From 8301b472feeaf857d4273f65eedb97957bc13755 Mon Sep 17 00:00:00 2001 From: NissimHadar Date: Tue, 18 Sep 2018 14:22:10 -0700 Subject: [PATCH 8/8] Automated login. --- .../AppDataHighFidelity/Interface/AccountInfo.bin | Bin 0 -> 489 bytes tools/auto-tester/src/TestRunner.cpp | 6 +++++- 2 files changed, 5 insertions(+), 1 deletion(-) create mode 100644 tools/auto-tester/AppDataHighFidelity/Interface/AccountInfo.bin diff --git a/tools/auto-tester/AppDataHighFidelity/Interface/AccountInfo.bin b/tools/auto-tester/AppDataHighFidelity/Interface/AccountInfo.bin new file mode 100644 index 0000000000000000000000000000000000000000..65c971ea793926e8f7518d36c6079a59b4cc08e8 GIT binary patch literal 489 zcmZwDIZgvX6a>(RL`XzHz#(8ZGh;F&5@N{FEZx`=U}O(SWQ3fJvv3Dcw;T{q(O;|T ze;nZ2%53&x=hC@zPl8$RO|Z2SoLO0WuoA6Jf=6q!cV|W$PaH}(x$fuwHf-O*_G%nA zug~+%bKIoPhf&apCcROkQCf8hNg_vhkUa5_0l~HGDQko1ZiyTL-sL>kK znJMirv{|oHyD*6y#LuteZyXr}y=^`)c}q-5WyrjPP9q9hg-($@q>Wmg)jJm&efAoZ zd-b5o4i}jz`AWUYq}a!$LZQw-XjJ){#)wl_Ud=Y{-3fN)?CxIqvq%2$AEff1^k1Zz EU+;ia(EtDd literal 0 HcmV?d00001 diff --git a/tools/auto-tester/src/TestRunner.cpp b/tools/auto-tester/src/TestRunner.cpp index eca851a8c9..0417fd8d04 100644 --- a/tools/auto-tester/src/TestRunner.cpp +++ b/tools/auto-tester/src/TestRunner.cpp @@ -180,9 +180,13 @@ void TestRunner::saveExistingHighFidelityAppDataFolder() { _appDataFolder = dataDirectory + "\\High Fidelity - " + getPRNumberFromURL(_url->toPlainText()); } + _savedAppDataFolder = dataDirectory + "/" + UNIQUE_FOLDER_NAME; + if (_savedAppDataFolder.exists()) { + _savedAppDataFolder.removeRecursively(); + } + if (_appDataFolder.exists()) { // The original folder is saved in a unique name - _savedAppDataFolder = dataDirectory + "/" + UNIQUE_FOLDER_NAME; _appDataFolder.rename(_appDataFolder.path(), _savedAppDataFolder.path()); }