From f872a95a7a9901900ecb730aaa814eea8473076b Mon Sep 17 00:00:00 2001 From: NissimHadar Date: Tue, 18 Sep 2018 08:49:11 -0700 Subject: [PATCH] Added number of failures to log. --- tools/auto-tester/src/Test.cpp | 16 ++++++------- tools/auto-tester/src/Test.h | 2 +- tools/auto-tester/src/TestRunner.cpp | 30 +++++++++++++++++-------- tools/auto-tester/src/TestRunner.h | 2 +- tools/auto-tester/src/ui/AutoTester.cpp | 4 ++-- tools/auto-tester/src/ui/AutoTester.h | 2 +- tools/auto-tester/src/ui/AutoTester.ui | 2 +- 7 files changed, 34 insertions(+), 24 deletions(-) diff --git a/tools/auto-tester/src/Test.cpp b/tools/auto-tester/src/Test.cpp index 9d77d4ed60..5e3dd50650 100644 --- a/tools/auto-tester/src/Test.cpp +++ b/tools/auto-tester/src/Test.cpp @@ -63,7 +63,7 @@ QString Test::zipAndDeleteTestResultsFolder() { return zippedResultsFileName; } -bool Test::compareImageLists() { +int Test::compareImageLists() { _progressBar->setMinimum(0); _progressBar->setMaximum(_expectedImagesFullFilenames.length() - 1); _progressBar->setValue(0); @@ -71,8 +71,8 @@ bool Test::compareImageLists() { // Loop over both lists and compare each pair of images // Quit loop if user has aborted due to a failed test. - bool success{ true }; bool keepOn{ true }; + int numberOfFailures{ 0 }; for (int i = 0; keepOn && i < _expectedImagesFullFilenames.length(); ++i) { // First check that images are the same size QImage resultImage(_resultImagesFullFilenames[i]); @@ -91,6 +91,7 @@ bool Test::compareImageLists() { } if (similarityIndex < THRESHOLD) { + ++numberOfFailures; TestFailure testFailure = TestFailure{ (float)similarityIndex, _expectedImagesFullFilenames[i].left(_expectedImagesFullFilenames[i].lastIndexOf("/") + 1), // path to the test (including trailing /) @@ -102,7 +103,6 @@ bool Test::compareImageLists() { if (!isInteractiveMode) { appendTestResultsToFile(_testResultsFolderPath, testFailure, _mismatchWindow.getComparisonImage()); - success = false; } else { _mismatchWindow.exec(); @@ -111,11 +111,9 @@ bool Test::compareImageLists() { break; case USE_RESPONSE_FAIL: appendTestResultsToFile(_testResultsFolderPath, testFailure, _mismatchWindow.getComparisonImage()); - success = false; break; case USER_RESPONSE_ABORT: keepOn = false; - success = false; break; default: assert(false); @@ -128,7 +126,7 @@ bool Test::compareImageLists() { } _progressBar->setVisible(false); - return success; + return numberOfFailures; } void Test::appendTestResultsToFile(const QString& _testResultsFolderPath, TestFailure testFailure, QPixmap comparisonImage) { @@ -256,10 +254,10 @@ void Test::startTestsEvaluation(const bool isRunningFromCommandLine, autoTester->downloadFiles(expectedImagesURLs, _snapshotDirectory, _expectedImagesFilenames, (void *)this); } void Test::finishTestsEvaluation() { - bool success = compareImageLists(); + int numberOfFailures = compareImageLists(); if (!_isRunningFromCommandLine && !_isRunningInAutomaticTestRun) { - if (success) { + if (numberOfFailures == 0) { QMessageBox::information(0, "Success", "All images are as expected"); } else { QMessageBox::information(0, "Failure", "One or more images are not as expected"); @@ -273,7 +271,7 @@ void Test::finishTestsEvaluation() { } if (_isRunningInAutomaticTestRun) { - autoTester->automaticTestRunEvaluationComplete(zippedFolderName); + autoTester->automaticTestRunEvaluationComplete(zippedFolderName, numberOfFailures); } } diff --git a/tools/auto-tester/src/Test.h b/tools/auto-tester/src/Test.h index c6ef83e7e8..8ea00c909b 100644 --- a/tools/auto-tester/src/Test.h +++ b/tools/auto-tester/src/Test.h @@ -75,7 +75,7 @@ public: void createAllRecursiveScripts(); void createRecursiveScript(const QString& topLevelDirectory, bool interactiveMode); - bool compareImageLists(); + int compareImageLists(); QStringList createListOfAll_imagesInDirectory(const QString& imageFormat, const QString& pathToImageDirectory); diff --git a/tools/auto-tester/src/TestRunner.cpp b/tools/auto-tester/src/TestRunner.cpp index 71199782e9..89b9bf27d6 100644 --- a/tools/auto-tester/src/TestRunner.cpp +++ b/tools/auto-tester/src/TestRunner.cpp @@ -119,7 +119,6 @@ void TestRunner::run() { filenames << _installerFilename; } - updateStatusLabel("Downloading installer"); autoTester->downloadFiles(urls, _workingFolder, filenames, (void*)this); @@ -174,7 +173,12 @@ void TestRunner::saveExistingHighFidelityAppDataFolder() { dataDirectory = qgetenv("USERPROFILE") + "\\AppData\\Roaming"; #endif - _appDataFolder = dataDirectory + "\\High Fidelity"; + if (!_runLatest->isChecked()) { + // We are running a PR build + _appDataFolder = dataDirectory + "\\High Fidelity - " + getPRNumberFromURL(_url->toPlainText()); + } else { + _appDataFolder = dataDirectory + "\\High Fidelity"; + } if (_appDataFolder.exists()) { // The original folder is saved in a unique name @@ -270,8 +274,7 @@ void TestRunner::startLocalServerProcesses() { } void TestRunner::runInterfaceWithTestScript() { - QString exeFile = QString("\"") + QDir::toNativeSeparators(_installationFolder) + - "\\interface.exe\""; + QString exeFile = QString("\"") + QDir::toNativeSeparators(_installationFolder) + "\\interface.exe\""; QString url = QString("hifi://localhost"); if (_runServerless->isChecked()) { @@ -302,7 +305,7 @@ void TestRunner::evaluateResults() { autoTester->startTestsEvaluation(false, true, _snapshotFolder, _branch, _user); } -void TestRunner::automaticTestRunEvaluationComplete(QString zippedFolder) { +void TestRunner::automaticTestRunEvaluationComplete(QString zippedFolder, int numberOfFailures) { addBuildNumberToResults(zippedFolder); restoreHighFidelityAppDataFolder(); @@ -310,9 +313,18 @@ void TestRunner::automaticTestRunEvaluationComplete(QString zippedFolder) { QDateTime currentDateTime = QDateTime::currentDateTime(); - appendLog(QString("Tests completed at ") + QString::number(currentDateTime.time().hour()) + ":" + - QString("%1").arg(currentDateTime.time().minute(), 2, 10, QChar('0')) + ", on " + - currentDateTime.date().toString("ddd, MMM d, yyyy")); + QString completionText = QString("Tests completed at ") + QString::number(currentDateTime.time().hour()) + ":" + + QString("%1").arg(currentDateTime.time().minute(), 2, 10, QChar('0')) + ", on " + + currentDateTime.date().toString("ddd, MMM d, yyyy"); + + if (numberOfFailures == 0) { + completionText += "; no failures"; + } else if (numberOfFailures == 1) { + completionText += "; 1 failure"; + } else { + completionText += QString("; ") + QString::number(numberOfFailures) + " failures"; + } + appendLog(completionText); _automatedTestIsRunning = false; } @@ -488,7 +500,7 @@ QString TestRunner::getInstallerNameFromURL(const QString& url) { // An example URL: https://deployment.highfidelity.com/jobs/pr-build/label%3Dwindows/13023/HighFidelity-Beta-Interface-PR14006-be76c43.exe try { QStringList urlParts = url.split("/"); - int rr = urlParts.size(); + int rr = urlParts.size(); if (urlParts.size() != 8) { throw "URL not in expected format, should look like `https://deployment.highfidelity.com/jobs/pr-build/label%3Dwindows/13023/HighFidelity-Beta-Interface-PR14006-be76c43.exe`"; } diff --git a/tools/auto-tester/src/TestRunner.h b/tools/auto-tester/src/TestRunner.h index 0843b438c8..f54ae98e65 100644 --- a/tools/auto-tester/src/TestRunner.h +++ b/tools/auto-tester/src/TestRunner.h @@ -54,7 +54,7 @@ public: void runInterfaceWithTestScript(); void evaluateResults(); - void automaticTestRunEvaluationComplete(QString zippedFolderName); + void automaticTestRunEvaluationComplete(QString zippedFolderName, int numberOfFailures); void addBuildNumberToResults(QString zippedFolderName); void copyFolder(const QString& source, const QString& destination); diff --git a/tools/auto-tester/src/ui/AutoTester.cpp b/tools/auto-tester/src/ui/AutoTester.cpp index 3a438a5fb9..502a447641 100644 --- a/tools/auto-tester/src/ui/AutoTester.cpp +++ b/tools/auto-tester/src/ui/AutoTester.cpp @@ -166,8 +166,8 @@ void AutoTester::on_checkBoxRunLatest_clicked() { _ui.urlTextEdit->setEnabled(!_ui.checkBoxRunLatest->isChecked()); } -void AutoTester::automaticTestRunEvaluationComplete(QString zippedFolderName) { - _testRunner->automaticTestRunEvaluationComplete(zippedFolderName); +void AutoTester::automaticTestRunEvaluationComplete(QString zippedFolderName, int numberOfFailures) { + _testRunner->automaticTestRunEvaluationComplete(zippedFolderName, numberOfFailures); } void AutoTester::on_updateTestRailRunResultsButton_clicked() { diff --git a/tools/auto-tester/src/ui/AutoTester.h b/tools/auto-tester/src/ui/AutoTester.h index f59d39e851..939a03acf4 100644 --- a/tools/auto-tester/src/ui/AutoTester.h +++ b/tools/auto-tester/src/ui/AutoTester.h @@ -36,7 +36,7 @@ public: const QString& branch, const QString& user); - void automaticTestRunEvaluationComplete(QString zippedFolderName); + void automaticTestRunEvaluationComplete(QString zippedFolderName, int numberOfFailures); void downloadFile(const QUrl& url); void downloadFiles(const QStringList& URLs, const QString& directoryName, const QStringList& filenames, void* caller); diff --git a/tools/auto-tester/src/ui/AutoTester.ui b/tools/auto-tester/src/ui/AutoTester.ui index 5357a7e612..bfa9ca587e 100644 --- a/tools/auto-tester/src/ui/AutoTester.ui +++ b/tools/auto-tester/src/ui/AutoTester.ui @@ -495,7 +495,7 @@ <html><head/><body><p>If unchecked, will not show results during evaluation</p></body></html> - Serveless + Server-less false