Mirrors/overte
3
0
Fork 0
mirror of https://github.com/overte-org/overte.git synced 2025-08-06 19:59:28 +02:00
Code Issues Projects Releases Packages Wiki Activity Actions

Added number of failures to log.

Browse source
This commit is contained in:
NissimHadar 2018-09-18 08:49:11 -07:00
parent 0c3e3e977e
commit f872a95a7a
7 changed files with 34 additions and 24 deletions
Show stats Download patch file Download diff file Expand all files Collapse all files

16
tools/auto-tester/src/Test.cpp
View file

@ -63,7 +63,7 @@ QString Test::zipAndDeleteTestResultsFolder() {
return zippedResultsFileName; return zippedResultsFileName;
} }
bool Test::compareImageLists() { int Test::compareImageLists() {
_progressBar->setMinimum(0); _progressBar->setMinimum(0);
_progressBar->setMaximum(_expectedImagesFullFilenames.length() - 1); _progressBar->setMaximum(_expectedImagesFullFilenames.length() - 1);
_progressBar->setValue(0); _progressBar->setValue(0);
@ -71,8 +71,8 @@ bool Test::compareImageLists() {
// Loop over both lists and compare each pair of images // Loop over both lists and compare each pair of images
// Quit loop if user has aborted due to a failed test. // Quit loop if user has aborted due to a failed test.
bool success{ true };
bool keepOn{ true }; bool keepOn{ true };
int numberOfFailures{ 0 };
for (int i = 0; keepOn && i < _expectedImagesFullFilenames.length(); ++i) { for (int i = 0; keepOn && i < _expectedImagesFullFilenames.length(); ++i) {
// First check that images are the same size // First check that images are the same size
QImage resultImage(_resultImagesFullFilenames[i]); QImage resultImage(_resultImagesFullFilenames[i]);
@ -91,6 +91,7 @@ bool Test::compareImageLists() {
} }
if (similarityIndex < THRESHOLD) { if (similarityIndex < THRESHOLD) {
++numberOfFailures;
TestFailure testFailure = TestFailure{ TestFailure testFailure = TestFailure{
(float)similarityIndex, (float)similarityIndex,
_expectedImagesFullFilenames[i].left(_expectedImagesFullFilenames[i].lastIndexOf("/") + 1), // path to the test (including trailing /) _expectedImagesFullFilenames[i].left(_expectedImagesFullFilenames[i].lastIndexOf("/") + 1), // path to the test (including trailing /)
@ -102,7 +103,6 @@ bool Test::compareImageLists() {
if (!isInteractiveMode) { if (!isInteractiveMode) {
appendTestResultsToFile(_testResultsFolderPath, testFailure, _mismatchWindow.getComparisonImage()); appendTestResultsToFile(_testResultsFolderPath, testFailure, _mismatchWindow.getComparisonImage());
success = false;
} else { } else {
_mismatchWindow.exec(); _mismatchWindow.exec();
@ -111,11 +111,9 @@ bool Test::compareImageLists() {
break; break;
case USE_RESPONSE_FAIL: case USE_RESPONSE_FAIL:
appendTestResultsToFile(_testResultsFolderPath, testFailure, _mismatchWindow.getComparisonImage()); appendTestResultsToFile(_testResultsFolderPath, testFailure, _mismatchWindow.getComparisonImage());
success = false;
break; break;
case USER_RESPONSE_ABORT: case USER_RESPONSE_ABORT:
keepOn = false; keepOn = false;
success = false;
break; break;
default: default:
assert(false); assert(false);
@ -128,7 +126,7 @@ bool Test::compareImageLists() {
} }
_progressBar->setVisible(false); _progressBar->setVisible(false);
return success; return numberOfFailures;
} }
void Test::appendTestResultsToFile(const QString& _testResultsFolderPath, TestFailure testFailure, QPixmap comparisonImage) { void Test::appendTestResultsToFile(const QString& _testResultsFolderPath, TestFailure testFailure, QPixmap comparisonImage) {
@ -256,10 +254,10 @@ void Test::startTestsEvaluation(const bool isRunningFromCommandLine,
autoTester->downloadFiles(expectedImagesURLs, _snapshotDirectory, _expectedImagesFilenames, (void *)this); autoTester->downloadFiles(expectedImagesURLs, _snapshotDirectory, _expectedImagesFilenames, (void *)this);
} }
void Test::finishTestsEvaluation() { void Test::finishTestsEvaluation() {
bool success = compareImageLists(); int numberOfFailures = compareImageLists();
if (!_isRunningFromCommandLine && !_isRunningInAutomaticTestRun) { if (!_isRunningFromCommandLine && !_isRunningInAutomaticTestRun) {
if (success) { if (numberOfFailures == 0) {
QMessageBox::information(0, "Success", "All images are as expected"); QMessageBox::information(0, "Success", "All images are as expected");
} else { } else {
QMessageBox::information(0, "Failure", "One or more images are not as expected"); QMessageBox::information(0, "Failure", "One or more images are not as expected");
@ -273,7 +271,7 @@ void Test::finishTestsEvaluation() {
} }
if (_isRunningInAutomaticTestRun) { if (_isRunningInAutomaticTestRun) {
autoTester->automaticTestRunEvaluationComplete(zippedFolderName); autoTester->automaticTestRunEvaluationComplete(zippedFolderName, numberOfFailures);
} }
} }

2
tools/auto-tester/src/Test.h
View file

@ -75,7 +75,7 @@ public:
void createAllRecursiveScripts(); void createAllRecursiveScripts();
void createRecursiveScript(const QString& topLevelDirectory, bool interactiveMode); void createRecursiveScript(const QString& topLevelDirectory, bool interactiveMode);
bool compareImageLists(); int compareImageLists();
QStringList createListOfAll_imagesInDirectory(const QString& imageFormat, const QString& pathToImageDirectory); QStringList createListOfAll_imagesInDirectory(const QString& imageFormat, const QString& pathToImageDirectory);

30
tools/auto-tester/src/TestRunner.cpp
View file

@ -119,7 +119,6 @@ void TestRunner::run() {
filenames << _installerFilename; filenames << _installerFilename;
} }
updateStatusLabel("Downloading installer"); updateStatusLabel("Downloading installer");
autoTester->downloadFiles(urls, _workingFolder, filenames, (void*)this); autoTester->downloadFiles(urls, _workingFolder, filenames, (void*)this);
@ -174,7 +173,12 @@ void TestRunner::saveExistingHighFidelityAppDataFolder() {
dataDirectory = qgetenv("USERPROFILE") + "\\AppData\\Roaming"; dataDirectory = qgetenv("USERPROFILE") + "\\AppData\\Roaming";
#endif #endif
_appDataFolder = dataDirectory + "\\High Fidelity"; if (!_runLatest->isChecked()) {
// We are running a PR build
_appDataFolder = dataDirectory + "\\High Fidelity - " + getPRNumberFromURL(_url->toPlainText());
} else {
_appDataFolder = dataDirectory + "\\High Fidelity";
}
if (_appDataFolder.exists()) { if (_appDataFolder.exists()) {
// The original folder is saved in a unique name // The original folder is saved in a unique name
@ -270,8 +274,7 @@ void TestRunner::startLocalServerProcesses() {
} }
void TestRunner::runInterfaceWithTestScript() { void TestRunner::runInterfaceWithTestScript() {
QString exeFile = QString("\"") + QDir::toNativeSeparators(_installationFolder) + QString exeFile = QString("\"") + QDir::toNativeSeparators(_installationFolder) + "\\interface.exe\"";
"\\interface.exe\"";
QString url = QString("hifi://localhost"); QString url = QString("hifi://localhost");
if (_runServerless->isChecked()) { if (_runServerless->isChecked()) {
@ -302,7 +305,7 @@ void TestRunner::evaluateResults() {
autoTester->startTestsEvaluation(false, true, _snapshotFolder, _branch, _user); autoTester->startTestsEvaluation(false, true, _snapshotFolder, _branch, _user);
} }
void TestRunner::automaticTestRunEvaluationComplete(QString zippedFolder) { void TestRunner::automaticTestRunEvaluationComplete(QString zippedFolder, int numberOfFailures) {
addBuildNumberToResults(zippedFolder); addBuildNumberToResults(zippedFolder);
restoreHighFidelityAppDataFolder(); restoreHighFidelityAppDataFolder();
@ -310,9 +313,18 @@ void TestRunner::automaticTestRunEvaluationComplete(QString zippedFolder) {
QDateTime currentDateTime = QDateTime::currentDateTime(); QDateTime currentDateTime = QDateTime::currentDateTime();
appendLog(QString("Tests completed at ") + QString::number(currentDateTime.time().hour()) + ":" + QString completionText = QString("Tests completed at ") + QString::number(currentDateTime.time().hour()) + ":" +
QString("%1").arg(currentDateTime.time().minute(), 2, 10, QChar('0')) + ", on " + QString("%1").arg(currentDateTime.time().minute(), 2, 10, QChar('0')) + ", on " +
currentDateTime.date().toString("ddd, MMM d, yyyy")); currentDateTime.date().toString("ddd, MMM d, yyyy");
if (numberOfFailures == 0) {
completionText += "; no failures";
} else if (numberOfFailures == 1) {
completionText += "; 1 failure";
} else {
completionText += QString("; ") + QString::number(numberOfFailures) + " failures";
}
appendLog(completionText);
_automatedTestIsRunning = false; _automatedTestIsRunning = false;
} }
@ -488,7 +500,7 @@ QString TestRunner::getInstallerNameFromURL(const QString& url) {
// An example URL: https://deployment.highfidelity.com/jobs/pr-build/label%3Dwindows/13023/HighFidelity-Beta-Interface-PR14006-be76c43.exe // An example URL: https://deployment.highfidelity.com/jobs/pr-build/label%3Dwindows/13023/HighFidelity-Beta-Interface-PR14006-be76c43.exe
try { try {
QStringList urlParts = url.split("/"); QStringList urlParts = url.split("/");
int rr = urlParts.size(); int rr = urlParts.size();
if (urlParts.size() != 8) { if (urlParts.size() != 8) {
throw "URL not in expected format, should look like `https://deployment.highfidelity.com/jobs/pr-build/label%3Dwindows/13023/HighFidelity-Beta-Interface-PR14006-be76c43.exe`"; throw "URL not in expected format, should look like `https://deployment.highfidelity.com/jobs/pr-build/label%3Dwindows/13023/HighFidelity-Beta-Interface-PR14006-be76c43.exe`";
} }

2
tools/auto-tester/src/TestRunner.h
View file

@ -54,7 +54,7 @@ public:
void runInterfaceWithTestScript(); void runInterfaceWithTestScript();
void evaluateResults(); void evaluateResults();
void automaticTestRunEvaluationComplete(QString zippedFolderName); void automaticTestRunEvaluationComplete(QString zippedFolderName, int numberOfFailures);
void addBuildNumberToResults(QString zippedFolderName); void addBuildNumberToResults(QString zippedFolderName);
void copyFolder(const QString& source, const QString& destination); void copyFolder(const QString& source, const QString& destination);

4
tools/auto-tester/src/ui/AutoTester.cpp
View file

@ -166,8 +166,8 @@ void AutoTester::on_checkBoxRunLatest_clicked() {
_ui.urlTextEdit->setEnabled(!_ui.checkBoxRunLatest->isChecked()); _ui.urlTextEdit->setEnabled(!_ui.checkBoxRunLatest->isChecked());
} }
void AutoTester::automaticTestRunEvaluationComplete(QString zippedFolderName) { void AutoTester::automaticTestRunEvaluationComplete(QString zippedFolderName, int numberOfFailures) {
_testRunner->automaticTestRunEvaluationComplete(zippedFolderName); _testRunner->automaticTestRunEvaluationComplete(zippedFolderName, numberOfFailures);
} }
void AutoTester::on_updateTestRailRunResultsButton_clicked() { void AutoTester::on_updateTestRailRunResultsButton_clicked() {

2
tools/auto-tester/src/ui/AutoTester.h
View file

@ -36,7 +36,7 @@ public:
const QString& branch, const QString& branch,
const QString& user); const QString& user);
void automaticTestRunEvaluationComplete(QString zippedFolderName); void automaticTestRunEvaluationComplete(QString zippedFolderName, int numberOfFailures);
void downloadFile(const QUrl& url); void downloadFile(const QUrl& url);
void downloadFiles(const QStringList& URLs, const QString& directoryName, const QStringList& filenames, void* caller); void downloadFiles(const QStringList& URLs, const QString& directoryName, const QStringList& filenames, void* caller);

2
tools/auto-tester/src/ui/AutoTester.ui
View file

@ -495,7 +495,7 @@
<string>&lt;html&gt;&lt;head/&gt;&lt;body&gt;&lt;p&gt;If unchecked, will not show results during evaluation&lt;/p&gt;&lt;/body&gt;&lt;/html&gt;</string> <string>&lt;html&gt;&lt;head/&gt;&lt;body&gt;&lt;p&gt;If unchecked, will not show results during evaluation&lt;/p&gt;&lt;/body&gt;&lt;/html&gt;</string>
</property> </property>
<property name="text"> <property name="text">
<string>Serveless</string> <string>Server-less</string>
</property> </property>
<property name="checked"> <property name="checked">
<bool>false</bool> <bool>false</bool>