From 4a702ed8d49c99149f3df44ba8a764a992a887a2 Mon Sep 17 00:00:00 2001 From: NissimHadar Date: Sat, 8 Sep 2018 21:27:21 -0700 Subject: [PATCH] add build number to the automated test results --- tools/auto-tester/src/Test.cpp | 8 +- tools/auto-tester/src/Test.h | 2 +- tools/auto-tester/src/TestRunner.cpp | 98 +++++++++++++++++++++---- tools/auto-tester/src/TestRunner.h | 6 +- tools/auto-tester/src/ui/AutoTester.cpp | 4 +- tools/auto-tester/src/ui/AutoTester.h | 2 +- 6 files changed, 97 insertions(+), 23 deletions(-) diff --git a/tools/auto-tester/src/Test.cpp b/tools/auto-tester/src/Test.cpp index f3526b67a2..d3aa310d7d 100644 --- a/tools/auto-tester/src/Test.cpp +++ b/tools/auto-tester/src/Test.cpp @@ -44,7 +44,7 @@ bool Test::createTestResultsFolderPath(const QString& directory) { return QDir().mkdir(_testResultsFolderPath); } -void Test::zipAndDeleteTestResultsFolder() { +QString Test::zipAndDeleteTestResultsFolder() { QString zippedResultsFileName { _testResultsFolderPath + ".zip" }; QFileInfo fileInfo(zippedResultsFileName); if (fileInfo.exists()) { @@ -59,6 +59,8 @@ void Test::zipAndDeleteTestResultsFolder() { //In all cases, for the next evaluation _testResultsFolderPath = ""; _index = 1; + + return zippedResultsFileName; } bool Test::compareImageLists() { @@ -264,14 +266,14 @@ void Test::finishTestsEvaluation() { } } - zipAndDeleteTestResultsFolder(); + QString zippedFolderName = zipAndDeleteTestResultsFolder(); if (_exitWhenComplete) { exit(0); } if (_isRunningInAutomaticTestRun) { - autoTester->automaticTestRunEvaluationComplete(); + autoTester->automaticTestRunEvaluationComplete(zippedFolderName); } } diff --git a/tools/auto-tester/src/Test.h b/tools/auto-tester/src/Test.h index e68ff4b295..c6ef83e7e8 100644 --- a/tools/auto-tester/src/Test.h +++ b/tools/auto-tester/src/Test.h @@ -86,7 +86,7 @@ public: void appendTestResultsToFile(const QString& testResultsFolderPath, TestFailure testFailure, QPixmap comparisonImage); bool createTestResultsFolderPath(const QString& directory); - void zipAndDeleteTestResultsFolder(); + QString zipAndDeleteTestResultsFolder(); static bool isAValidDirectory(const QString& pathname); QString extractPathFromTestsDown(const QString& fullPath); diff --git a/tools/auto-tester/src/TestRunner.cpp b/tools/auto-tester/src/TestRunner.cpp index 4ca263a7f3..5c7bb97055 100644 --- a/tools/auto-tester/src/TestRunner.cpp +++ b/tools/auto-tester/src/TestRunner.cpp @@ -16,7 +16,7 @@ #include "ui/AutoTester.h" extern AutoTester* autoTester; -TestRunner::TestRunner(QObject *parent) : QObject(parent) { +TestRunner::TestRunner(QObject* parent) : QObject(parent) { } void TestRunner::run() { @@ -30,14 +30,14 @@ void TestRunner::run() { // This will be restored at the end of the tests saveExistingHighFidelityAppDataFolder(); - // Download the latest High Fidelity installer + // Download the latest High Fidelity installer and build XML. QStringList urls; - urls << INSTALLER_URL; + urls << INSTALLER_URL << BUILD_XML_URL; QStringList filenames; - filenames << INSTALLER_FILENAME; + filenames << INSTALLER_FILENAME << BUILD_XML_FILENAME; - autoTester->downloadFiles(urls, _tempFolder, filenames, (void *)this); + autoTester->downloadFiles(urls, _tempFolder, filenames, (void*)this); // `installerDownloadComplete` will run after download has completed } @@ -47,7 +47,7 @@ void TestRunner::installerDownloadComplete() { killProcesses(); runInstaller(); - + createSnapshotFolder(); startLocalServerProcesses(); @@ -63,9 +63,9 @@ void TestRunner::runInstaller() { // Qt cannot start an installation process using QProcess::start (Qt Bug 9761) // To allow installation, the installer is run using the `system` command QStringList arguments{ QStringList() << QString("/S") << QString("/D=") + QDir::toNativeSeparators(_installationFolder) }; - + QString installerFullPath = _tempFolder + "/" + INSTALLER_FILENAME; - + QString commandLine = QDir::toNativeSeparators(installerFullPath) + " /S /D=" + QDir::toNativeSeparators(_installationFolder); @@ -88,7 +88,7 @@ void TestRunner::saveExistingHighFidelityAppDataFolder() { } // Copy an "empty" AppData folder (i.e. no entities) - copyFolder(QDir::currentPath() + "/AppDataHighFidelity", _appDataFolder.path()); + copyFolder(QDir::currentPath() + "/AppDataHighFidelity", _appDataFolder.path()); } void TestRunner::restoreHighFidelityAppDataFolder() { @@ -106,8 +106,8 @@ void TestRunner::selectTemporaryFolder() { parent += "/"; } - _tempFolder = - QFileDialog::getExistingDirectory(nullptr, "Please select a temporary folder for installation", parent, QFileDialog::ShowDirsOnly); + _tempFolder = QFileDialog::getExistingDirectory(nullptr, "Please select a temporary folder for installation", parent, + QFileDialog::ShowDirsOnly); // If user canceled then restore previous selection and return if (_tempFolder == "") { @@ -168,9 +168,9 @@ void TestRunner::startLocalServerProcesses() { void TestRunner::runInterfaceWithTestScript() { #ifdef Q_OS_WIN QString commandLine = QString("\"") + QDir::toNativeSeparators(_installationFolder) + - "\\interface.exe\" --url hifi://localhost --testScript https://raw.githubusercontent.com/" + _user + - "/hifi_tests/" + _branch + "/tests/testRecursive.js quitWhenFinished --testResultsLocation " + - _snapshotFolder; + "\\interface.exe\" --url hifi://localhost --testScript https://raw.githubusercontent.com/" + _user + + "/hifi_tests/" + _branch + "/tests/testRecursive.js quitWhenFinished --testResultsLocation " + + _snapshotFolder; system(commandLine.toStdString().c_str()); #endif @@ -180,10 +180,78 @@ void TestRunner::evaluateResults() { autoTester->startTestsEvaluation(false, true, _snapshotFolder, _branch, _user); } -void TestRunner::automaticTestRunEvaluationComplete() { +void TestRunner::automaticTestRunEvaluationComplete(QString zippedFolder) { + addBuildNumberToResults(zippedFolder); restoreHighFidelityAppDataFolder(); } +void TestRunner::addBuildNumberToResults(QString zippedFolderName) { + try { + QDomDocument domDocument; + QString filename{ _tempFolder + "/" + BUILD_XML_FILENAME }; + QFile file(filename); + if (!file.open(QIODevice::ReadOnly) || !domDocument.setContent(&file)) { + throw QString("Could not open " + filename); + } + + QString platformOfInterest; +#ifdef Q_OS_WIN + platformOfInterest = "windows"; +#else if Q_OS_MAC + platformOfInterest = "mac"; +#endif + QDomElement element = domDocument.documentElement(); + + // Verify first element is "projects" + if (element.tagName() != "projects") { + throw("File seems to be in wrong format"); + } + + element = element.firstChild().toElement(); + if (element.tagName() != "project") { + throw("File seems to be in wrong format"); + } + + if (element.attribute("name") != "interface") { + throw("File is not from 'interface' build"); + } + + // Now loop over the platforms + while (!element.isNull()) { + element = element.firstChild().toElement(); + QString sdf = element.tagName(); + if (element.tagName() != "platform" || element.attribute("name") != platformOfInterest) { + continue; + } + + // Next element should be the build + element = element.firstChild().toElement(); + if (element.tagName() != "build") { + throw("File seems to be in wrong format"); + } + + // Next element should be the version + element = element.firstChild().toElement(); + if (element.tagName() != "version") { + throw("File seems to be in wrong format"); + } + + // Add the build number to the end of the filename + QString build = element.text(); + QStringList filenameParts = zippedFolderName.split("."); + QString augmentedFilename = filenameParts[0] + "_" + build + "." + filenameParts[1]; + QFile::rename(zippedFolderName, augmentedFilename); + } + + } catch (QString errorMessage) { + QMessageBox::critical(0, "Internal error: " + QString(__FILE__) + ":" + QString::number(__LINE__), errorMessage); + exit(-1); + } catch (...) { + QMessageBox::critical(0, "Internal error: " + QString(__FILE__) + ":" + QString::number(__LINE__), "unknown error"); + exit(-1); + } +} + // Copies a folder recursively void TestRunner::copyFolder(const QString& source, const QString& destination) { try { diff --git a/tools/auto-tester/src/TestRunner.h b/tools/auto-tester/src/TestRunner.h index 2d9058e71a..5a5cbf1cf3 100644 --- a/tools/auto-tester/src/TestRunner.h +++ b/tools/auto-tester/src/TestRunner.h @@ -35,7 +35,8 @@ public: void startLocalServerProcesses(); void runInterfaceWithTestScript(); void evaluateResults(); - void automaticTestRunEvaluationComplete(); + void automaticTestRunEvaluationComplete(QString zippedFolderName); + void addBuildNumberToResults(QString zippedFolderName); void copyFolder(const QString& source, const QString& destination); @@ -56,6 +57,9 @@ private: const QString INSTALLER_URL{ "http://builds.highfidelity.com/HighFidelity-Beta-latest-dev.exe" }; const QString INSTALLER_FILENAME{ "HighFidelity-Beta-latest-dev.exe" }; + const QString BUILD_XML_URL{ "https://highfidelity.com/dev-builds.xml" }; + const QString BUILD_XML_FILENAME{ "dev-builds.xml" }; + QString _branch; QString _user; }; diff --git a/tools/auto-tester/src/ui/AutoTester.cpp b/tools/auto-tester/src/ui/AutoTester.cpp index b76bb6b7ab..3c8be24f69 100644 --- a/tools/auto-tester/src/ui/AutoTester.cpp +++ b/tools/auto-tester/src/ui/AutoTester.cpp @@ -107,8 +107,8 @@ void AutoTester::on_runNowButton_clicked() { _testRunner->run(); } -void AutoTester::automaticTestRunEvaluationComplete() { - _testRunner->automaticTestRunEvaluationComplete(); +void AutoTester::automaticTestRunEvaluationComplete(QString zippedFolderName) { + _testRunner->automaticTestRunEvaluationComplete(zippedFolderName); } void AutoTester::on_updateTestRailRunResultsButton_clicked() { diff --git a/tools/auto-tester/src/ui/AutoTester.h b/tools/auto-tester/src/ui/AutoTester.h index 1418365c07..6c8641fa49 100644 --- a/tools/auto-tester/src/ui/AutoTester.h +++ b/tools/auto-tester/src/ui/AutoTester.h @@ -36,7 +36,7 @@ public: const QString& branch, const QString& user); - void automaticTestRunEvaluationComplete(); + void automaticTestRunEvaluationComplete(QString zippedFolderName); void downloadFile(const QUrl& url); void downloadFiles(const QStringList& URLs, const QString& directoryName, const QStringList& filenames, void *caller);