diff --git a/tools/nitpick/README.md b/tools/nitpick/README.md index 7d75d660d7..23105a0e02 100644 --- a/tools/nitpick/README.md +++ b/tools/nitpick/README.md @@ -5,7 +5,7 @@ Nitpick is a stand alone application that provides a mechanism for regression te * The snapshots are compared to a 'canonical' set of images that have been produced beforehand. * The result, if any test failed, is a zipped folder describing the failure. -Nitpick has 5 functions, separated into 4 tabs: +Nitpick has 5 functions, separated into separate tabs: 1. Creating tests, MD files and recursive scripts 1. Windows task bar utility (Windows only) 1. Running tests @@ -22,9 +22,9 @@ Nitpick is built as part of the High Fidelity build. 1. Select all, right-click and select 7-Zip->Add to archive... 1. Set Archive format to 7z 1. Check "Create SFX archive -1. Enter installer name (i.e. `nitpick-installer-v1.1.exe`) +1. Enter installer name (i.e. `nitpick-installer-v1.2.exe`) 1. Click "OK" -1. Copy created installer to https://hifi-qa.s3.amazonaws.com/nitpick/Windows/nitpick-installer-v1.1.exe: aws s3 cp nitpick-installer-v1.1.exe s3://hifi-qa/nitpick/Mac/nitpick-installer-v1.1.exe +1. Copy created installer to https://hifi-qa.s3.amazonaws.com/nitpick/Windows/nitpick-installer-v1.2.exe: aws s3 cp nitpick-installer-v1.2.exe s3://hifi-qa/nitpick/Mac/nitpick-installer-v1.2.exe #### Mac These steps assume the hifi repository has been cloned to `~/hifi`. 1. (first time) Install brew @@ -37,12 +37,12 @@ These steps assume the hifi repository has been cloned to `~/hifi`. 1. Change the loader instruction to find the dynamic library locally In a terminal: `install_name_tool -change ~/hifi/build/ext/Xcode/quazip/project/lib/libquazip5.1.dylib libquazip5.1.dylib nitpick` 1. Delete any existing disk images. In a terminal: `rm *.dmg` -1. Create installer (note final period).In a terminal: `create-dmg --volname nitpick-installer-v1.1 nitpick-installer-v1.1.dmg .` +1. Create installer (note final period).In a terminal: `create-dmg --volname nitpick-installer-v1.2 nitpick-installer-v1.2.dmg .` Make sure to wait for completion. -1. Copy created installer to AWS: `~/Library/Python/3.7/bin/aws s3 cp nitpick-installer-v1.1.dmg s3://hifi-qa/nitpick/Mac/nitpick-installer-v1.1.dmg` +1. Copy created installer to AWS: `~/Library/Python/3.7/bin/aws s3 cp nitpick-installer-v1.2.dmg s3://hifi-qa/nitpick/Mac/nitpick-installer-v1.2.dmg` ### Installation #### Windows -1. (First time) download and install vc_redist.x64.exe (available at https://hifi-qa.s3.amazonaws.com/nitpick/Windows/nitpick-installer-v1.1.exe) +1. (First time) download and install vc_redist.x64.exe (available at https://hifi-qa.s3.amazonaws.com/nitpick/Windows/nitpick-installer-v1.2.exe) 1. (First time) download and install Python 3 from https://hifi-qa.s3.amazonaws.com/nitpick/Windows/python-3.7.0-amd64.exe (also located at https://www.python.org/downloads/) 1. After installation - create an environment variable called PYTHON_PATH and set it to the folder containing the Python executable. 1. (First time) download and install AWS CLI from https://hifi-qa.s3.amazonaws.com/nitpick/Windows/AWSCLI64PY3.msi (also available at https://aws.amazon.com/cli/ @@ -52,7 +52,7 @@ These steps assume the hifi repository has been cloned to `~/hifi`. 1. Leave region name and ouput format as default [None] 1. Install the latest release of Boto3 via pip: `pip install boto3` -1. Download the installer by browsing to [here]() +1. Download the installer by browsing to [here]() 1. Double click on the installer and install to a convenient location ![](./setup_7z.PNG) @@ -76,14 +76,14 @@ In a terminal: `python3 get-pip.py --user` 1. Enter the secret key 1. Leave region name and ouput format as default [None] 1. Install the latest release of Boto3 via pip: pip3 install boto3 -1. Download the installer by browsing to [here](). +1. Download the installer by browsing to [here](). 1. Double-click on the downloaded image to mount it 1. Create a folder for the nitpick files (e.g. ~/nitpick) If this folder exists then delete all it's contents. 1. Copy the downloaded files to the folder In a terminal: `cd ~/nitpick` - `cp -r /Volumes/nitpick-installer-v1.1/* .` + `cp -r /Volumes/nitpick-installer-v1.2/* .` 1. __To run nitpick, cd to the folder that you copied to and run `./nitpick`__ # Usage diff --git a/tools/nitpick/src/AWSInterface.cpp b/tools/nitpick/src/AWSInterface.cpp index e43ef8dc75..0b93ce44e5 100644 --- a/tools/nitpick/src/AWSInterface.cpp +++ b/tools/nitpick/src/AWSInterface.cpp @@ -10,6 +10,8 @@ #include "AWSInterface.h" #include +#include +#include #include #include @@ -22,12 +24,12 @@ AWSInterface::AWSInterface(QObject* parent) : QObject(parent) { } void AWSInterface::createWebPageFromResults(const QString& testResults, - const QString& snapshotDirectory, + const QString& workingDirectory, QCheckBox* updateAWSCheckBox, QLineEdit* urlLineEdit) { _testResults = testResults; - _snapshotDirectory = snapshotDirectory; - + _workingDirectory = workingDirectory; + _urlLineEdit = urlLineEdit; _urlLineEdit->setEnabled(false); @@ -36,6 +38,9 @@ void AWSInterface::createWebPageFromResults(const QString& testResults, if (updateAWSCheckBox->isChecked()) { updateAWS(); + QMessageBox::information(0, "Success", "HTML file has been created and copied to AWS"); + } else { + QMessageBox::information(0, "Success", "HTML file has been created"); } } @@ -43,14 +48,14 @@ void AWSInterface::extractTestFailuresFromZippedFolder() { // For a test results zip file called `D:/tt/TestResults--2018-10-02_16-54-11(9426)[DESKTOP-PMKNLSQ].zip` // the folder will be called `TestResults--2018-10-02_16-54-11(9426)[DESKTOP-PMKNLSQ]` // and, this folder will be in the working directory - QStringList parts =_testResults.split('/'); - QString zipFolderName = _snapshotDirectory + "/" + parts[parts.length() - 1].split('.')[0]; + QStringList parts = _testResults.split('/'); + QString zipFolderName = _workingDirectory + "/" + parts[parts.length() - 1].split('.')[0]; if (QDir(zipFolderName).exists()) { QDir dir = zipFolderName; dir.removeRecursively(); } - JlCompress::extractDir(_testResults, _snapshotDirectory); + JlCompress::extractDir(_testResults, _workingDirectory); } void AWSInterface::createHTMLFile() { @@ -60,7 +65,7 @@ void AWSInterface::createHTMLFile() { QString filename = pathComponents[pathComponents.length() - 1]; _resultsFolder = filename.left(filename.length() - 4); - QString resultsPath = _snapshotDirectory + "/" + _resultsFolder + "/"; + QString resultsPath = _workingDirectory + "/" + _resultsFolder + "/"; QDir().mkdir(resultsPath); _htmlFilename = resultsPath + HTML_FILENAME; @@ -98,65 +103,11 @@ void AWSInterface::writeHead(QTextStream& stream) { void AWSInterface::writeBody(QTextStream& stream) { stream << "\t" << "\n"; - writeTitle(stream); - writeTable(stream); - stream << "\t" << "\n"; -} -void AWSInterface::finishHTMLpage(QTextStream& stream) { - stream << "\n"; -} - -void AWSInterface::writeTitle(QTextStream& stream) { - // Separate relevant components from the results name - // The expected format is as follows: `D:/tt/snapshots/TestResults--2018-10-04_11-09-41(PR14128)[DESKTOP-PMKNLSQ].zip` - QStringList tokens = _testResults.split('/'); - - // date_buildorPR_hostName will be 2018-10-03_15-35-28(9433)[DESKTOP-PMKNLSQ] - QString date_buildorPR_hostName = tokens[tokens.length() - 1].split("--")[1].split(".")[0]; - - QString buildorPR = date_buildorPR_hostName.split('(')[1].split(')')[0]; - QString hostName = date_buildorPR_hostName.split('[')[1].split(']')[0]; - - QStringList dateList = date_buildorPR_hostName.split('(')[0].split('_')[0].split('-'); - QString year = dateList[0]; - QString month = dateList[1]; - QString day = dateList[2]; - - QStringList timeList = date_buildorPR_hostName.split('(')[0].split('_')[1].split('-'); - QString hour = timeList[0]; - QString minute = timeList[1]; - QString second = timeList[2]; - - const QString months[] = { "Jan", "Feb", "Mar", "Apr", "May", "Jun", "Jul", "Aug", "Sep", "Oct", "Nov", "Dec" }; - - stream << "\t" << "\t" << "\n"; - stream << "\t" << "\t" << "

Failures for "; - stream << months[month.toInt() - 1] << " " << day << ", " << year << ", "; - stream << hour << ":" << minute << ":" << second << ", "; - - if (buildorPR.left(2) == "PR") { - stream << "PR " << buildorPR.right(buildorPR.length() - 2) << ", "; - } else { - stream << "build " << buildorPR << ", "; - } - - stream << "run on " << hostName << "

\n"; -} - -void AWSInterface::writeTable(QTextStream& stream) { - QString previousTestName{ "" }; - - // Loop over all entries in directory. This is done in stages, as the names are not in the order of the tests - // The first stage reads the directory names into a list - // The second stage renames the tests by removing everything up to "--tests." - // The third stage renames the directories - // The fourth and lasts stage creates the HTML entries - // - // Note that failures are processed first, then successes + // The results are read here as they are used both in the title (for the summary) and for table QStringList originalNamesFailures; QStringList originalNamesSuccesses; - QDirIterator it1(_snapshotDirectory.toStdString().c_str()); + QDirIterator it1(_workingDirectory); while (it1.hasNext()) { QString nextDirectory = it1.next(); @@ -165,7 +116,7 @@ void AWSInterface::writeTable(QTextStream& stream) { continue; } - // Only process failure folders + // Only process result folders if (!nextDirectory.contains("--tests.")) { continue; } @@ -180,6 +131,71 @@ void AWSInterface::writeTable(QTextStream& stream) { } } + writeTitle(stream, originalNamesFailures, originalNamesSuccesses); + writeTable(stream, originalNamesFailures, originalNamesSuccesses); + stream << "\t" << "\n"; +} + +void AWSInterface::finishHTMLpage(QTextStream& stream) { + stream << "\n"; +} + +void AWSInterface::writeTitle(QTextStream& stream, const QStringList& originalNamesFailures, const QStringList& originalNamesSuccesses) { + // Separate relevant components from the results name + // The expected format is as follows: `D:/tt/snapshots/TestResults--2018-10-04_11-09-41(PR14128)[DESKTOP-PMKNLSQ].zip` + QStringList tokens = _testResults.split('/'); + + // date_buildorPR_hostName will be 2018-10-03_15-35-28(9433)[DESKTOP-PMKNLSQ] + QString date_buildorPR_hostName = tokens[tokens.length() - 1].split("--")[1].split(".")[0]; + + QString buildorPR = date_buildorPR_hostName.split('(')[1].split(')')[0]; + QString hostName = date_buildorPR_hostName.split('[')[1].split(']')[0]; + + QStringList dateList = date_buildorPR_hostName.split('(')[0].split('_')[0].split('-'); + QString year = dateList[0]; + QString month = dateList[1]; + QString day = dateList[2]; + + QStringList timeList = date_buildorPR_hostName.split('(')[0].split('_')[1].split('-'); + QString hour = timeList[0]; + QString minute = timeList[1]; + QString second = timeList[2]; + + const QString months[] = { "Jan", "Feb", "Mar", "Apr", "May", "Jun", "Jul", "Aug", "Sep", "Oct", "Nov", "Dec" }; + + stream << "\t" << "\t" << "\n"; + stream << "\t" << "\t" << "

Results for "; + stream << months[month.toInt() - 1] << " " << day << ", " << year << ", "; + stream << hour << ":" << minute << ":" << second << ", "; + + if (buildorPR.left(2) == "PR") { + stream << "PR " << buildorPR.right(buildorPR.length() - 2) << ", "; + } else { + stream << "build " << buildorPR << ", "; + } + + stream << "run on " << hostName << "

\n"; + + int numberOfFailures = originalNamesFailures.length(); + int numberOfSuccesses = originalNamesSuccesses.length(); + + stream << "

" << QString::number(numberOfFailures) << " failed, out of a total of " << QString::number(numberOfSuccesses) << " tests

\n"; + + stream << "\t" << "\t" << "\n"; + stream << "\t" << "\t" << "

The following tests failed:

"; +} + +void AWSInterface::writeTable(QTextStream& stream, const QStringList& originalNamesFailures, const QStringList& originalNamesSuccesses) { + QString previousTestName{ "" }; + + // Loop over all entries in directory. This is done in stages, as the names are not in the order of the tests + // The first stage reads the directory names into a list + // The second stage renames the tests by removing everything up to "--tests." + // The third stage renames the directories + // The fourth and lasts stage creates the HTML entries + // + // Note that failures are processed first, then successes + QStringList newNamesFailures; for (int i = 0; i < originalNamesFailures.length(); ++i) { newNamesFailures.append(originalNamesFailures[i].split("--tests.")[1]); @@ -189,11 +205,11 @@ void AWSInterface::writeTable(QTextStream& stream) { for (int i = 0; i < originalNamesSuccesses.length(); ++i) { newNamesSuccesses.append(originalNamesSuccesses[i].split("--tests.")[1]); } - - _htmlFailuresFolder = _snapshotDirectory + "/" + _resultsFolder + "/" + FAILURES_FOLDER; + + _htmlFailuresFolder = _workingDirectory + "/" + _resultsFolder + "/" + FAILURES_FOLDER; QDir().mkdir(_htmlFailuresFolder); - _htmlSuccessesFolder = _snapshotDirectory + "/" + _resultsFolder + "/" + SUCCESSES_FOLDER; + _htmlSuccessesFolder = _workingDirectory + "/" + _resultsFolder + "/" + SUCCESSES_FOLDER; QDir().mkdir(_htmlSuccessesFolder); for (int i = 0; i < newNamesFailures.length(); ++i) { @@ -204,7 +220,11 @@ void AWSInterface::writeTable(QTextStream& stream) { QDir().rename(originalNamesSuccesses[i], _htmlSuccessesFolder + "/" + newNamesSuccesses[i]); } - QDirIterator it2((_htmlFailuresFolder).toStdString().c_str()); + // Mac does not read folders in lexicographic order, so this step is divided into 2 + // Each test consists of the test name and its index. + QStringList folderNames; + + QDirIterator it2(_htmlFailuresFolder); while (it2.hasNext()) { QString nextDirectory = it2.next(); @@ -214,10 +234,17 @@ void AWSInterface::writeTable(QTextStream& stream) { } QStringList pathComponents = nextDirectory.split('/'); - QString filename = pathComponents[pathComponents.length() - 1]; - int splitIndex = filename.lastIndexOf("."); - QString testName = filename.left(splitIndex).replace(".", " / "); - QString testNumber = filename.right(filename.length() - (splitIndex + 1)); + QString folderName = pathComponents[pathComponents.length() - 1]; + + folderNames << folderName; + } + + folderNames.sort(); + for (const auto& folderName : folderNames) { + int splitIndex = folderName.lastIndexOf("."); + QString testName = folderName.left(splitIndex).replace('.', " / "); + + int testNumber = folderName.right(folderName.length() - (splitIndex + 1)).toInt(); // The failures are ordered lexicographically, so we know that we can rely on the testName changing to create a new table if (testName != previousTestName) { @@ -228,18 +255,20 @@ void AWSInterface::writeTable(QTextStream& stream) { previousTestName = testName; stream << "\t\t

" << testName << "

\n"; - - openTable(stream); + openTable(stream, folderName, true); } - createEntry(testNumber.toInt(), filename, stream, true); + createEntry(testNumber, folderName, stream, true); } closeTable(stream); stream << "\t" << "\t" << "\n"; stream << "\t" << "\t" << "

The following tests passed:

"; - QDirIterator it3((_htmlSuccessesFolder).toStdString().c_str()); + // Now do the same for passes + folderNames.clear(); + + QDirIterator it3(_htmlSuccessesFolder); while (it3.hasNext()) { QString nextDirectory = it3.next(); @@ -249,10 +278,17 @@ void AWSInterface::writeTable(QTextStream& stream) { } QStringList pathComponents = nextDirectory.split('/'); - QString filename = pathComponents[pathComponents.length() - 1]; - int splitIndex = filename.lastIndexOf("."); - QString testName = filename.left(splitIndex).replace(".", " / "); - QString testNumber = filename.right(filename.length() - (splitIndex + 1)); + QString folderName = pathComponents[pathComponents.length() - 1]; + + folderNames << folderName; + } + + folderNames.sort(); + for (const auto& folderName : folderNames) { + int splitIndex = folderName.lastIndexOf("."); + QString testName = folderName.left(splitIndex).replace('.', " / "); + + int testNumber = folderName.right(folderName.length() - (splitIndex + 1)).toInt(); // The failures are ordered lexicographically, so we know that we can rely on the testName changing to create a new table if (testName != previousTestName) { @@ -263,64 +299,153 @@ void AWSInterface::writeTable(QTextStream& stream) { previousTestName = testName; stream << "\t\t

" << testName << "

\n"; - - openTable(stream); + openTable(stream, folderName, false); } - createEntry(testNumber.toInt(), filename, stream, false); + createEntry(testNumber, folderName, stream, false); } closeTable(stream); } -void AWSInterface::openTable(QTextStream& stream) { - stream << "\t\t\n"; - stream << "\t\t\t\n"; - stream << "\t\t\t\t\n"; - stream << "\t\t\t\t\n"; - stream << "\t\t\t\t\n"; - stream << "\t\t\t\t\n"; - stream << "\t\t\t\n"; +void AWSInterface::openTable(QTextStream& stream, const QString& testResult, const bool isFailure) { + QStringList resultNameComponents = testResult.split('/'); + QString resultName = resultNameComponents[resultNameComponents.length() - 1]; + + bool textResultsFileFound; + if (isFailure) { + textResultsFileFound = QFile::exists(_htmlFailuresFolder + "/" + resultName + "/Result.txt"); + } else { + textResultsFileFound = QFile::exists(_htmlSuccessesFolder + "/" + resultName + "/Result.txt"); + } + + if (textResultsFileFound) { + if (isFailure) { + stream << "\t\t

Test

Actual Image

Expected Image

Difference Image

\n"; + stream << "\t\t\t\n"; + stream << "\t\t\t\t\n"; + stream << "\t\t\t\t\n"; + stream << "\t\t\t\t\n"; + stream << "\t\t\t\n"; + } else { + stream << "\t\t

No errors found

\n\n"; + stream << "\t\t

===============

\n\n"; + } + } else { + stream << "\t\t

Element

Actual Value

Expected Value

\n"; + stream << "\t\t\t\n"; + stream << "\t\t\t\t\n"; + stream << "\t\t\t\t\n"; + stream << "\t\t\t\t\n"; + stream << "\t\t\t\t\n"; + stream << "\t\t\t\n"; + } } void AWSInterface::closeTable(QTextStream& stream) { stream << "\t\t

Test

Actual Image

Expected Image

Difference Image

\n"; } -void AWSInterface::createEntry(int index, const QString& testResult, QTextStream& stream, const bool isFailure) { - stream << "\t\t\t\n"; - stream << "\t\t\t\t

" << QString::number(index) << "

\n"; - +void AWSInterface::createEntry(const int index, const QString& testResult, QTextStream& stream, const bool isFailure) { // For a test named `D:/t/fgadhcUDHSFaidsfh3478JJJFSDFIUSOEIrf/Failure_1--tests.engine.interaction.pick.collision.many.00000` // we need `Failure_1--tests.engine.interaction.pick.collision.many.00000` QStringList resultNameComponents = testResult.split('/'); QString resultName = resultNameComponents[resultNameComponents.length() - 1]; + QString textResultFilename; + if (isFailure) { + textResultFilename = _htmlFailuresFolder + "/" + resultName + "/Result.txt"; + } else { + textResultFilename = _htmlSuccessesFolder + "/" + resultName + "/Result.txt"; + } + bool textResultsFileFound{ QFile::exists(textResultFilename) }; + QString folder; bool differenceFileFound; if (isFailure) { folder = FAILURES_FOLDER; differenceFileFound = QFile::exists(_htmlFailuresFolder + "/" + resultName + "/Difference Image.png"); } else { - folder = SUCCESSES_FOLDER; + folder = SUCCESSES_FOLDER; differenceFileFound = QFile::exists(_htmlSuccessesFolder + "/" + resultName + "/Difference Image.png"); } - - stream << "\t\t\t\t\n"; - stream << "\t\t\t\t\n"; + if (textResultsFileFound) { + // Parse the JSON file + QFile file; + file.setFileName(textResultFilename); + if (!file.open(QIODevice::ReadOnly | QIODevice::Text)) { + QMessageBox::critical(0, "Internal error: " + QString(__FILE__) + ":" + QString::number(__LINE__), + "Failed to open file " + textResultFilename); + } - if (differenceFileFound) { - stream << "\t\t\t\t\n"; + QString value = file.readAll(); + file.close(); + + // The Result.txt file is an object containing elements such as the following: + // "angularDamping": { + // "actual": 0.3938899040222168, + // "expected" : 0.3938899, + // "result" : "pass" + // }, + // + // Failures are thos element that have "fail for the result + + QJsonDocument document = QJsonDocument::fromJson(value.toUtf8()); + QJsonObject json = document.object(); + foreach(const QString& key, json.keys()) { + QJsonValue value = json.value(key); + QJsonObject object = value.toObject(); + + QJsonValue actualValue = object.value("actual"); + QString actualValueString; + if (actualValue.isString()) { + actualValueString = actualValue.toString(); + } else if (actualValue.isBool()) { + actualValueString = actualValue.toBool() ? "true" : "false"; + } else if (actualValue.isDouble()) { + actualValueString = QString::number(actualValue.toDouble()); + } + + QJsonValue expectedValue = object.value("expected"); + QString expectedValueString; + if (expectedValue.isString()) { + expectedValueString = expectedValue.toString(); + } else if (expectedValue.isBool()) { + expectedValueString = expectedValue.toBool() ? "true" : "false"; + } else if (expectedValue.isDouble()) { + expectedValueString = QString::number(expectedValue.toDouble()); + } + QString result = object.value("result").toString(); + + if (result == "fail") { + stream << "\t\t\t\n"; + stream << "\t\t\t\t" + key + "\n"; + stream << "\t\t\t\t" + actualValueString + "\n"; + stream << "\t\t\t\t" + expectedValueString + "\n"; + stream << "\t\t\t\n"; + } + } } else { - stream << "\t\t\t\t

No Image Found

\n"; + stream << "\t\t\t\n"; + stream << "\t\t\t\t

" << QString::number(index) << "

\n"; + + stream << "\t\t\t\t\n"; + stream << "\t\t\t\t\n"; + + if (differenceFileFound) { + stream << "\t\t\t\t\n"; + } else { + stream << "\t\t\t\t

No Image Found

\n"; + } + + stream << "\t\t\t\n"; } - stream << "\t\t\t\n"; } void AWSInterface::updateAWS() { - QString filename = _snapshotDirectory + "/updateAWS.py"; + QString filename = _workingDirectory + "/updateAWS.py"; if (QFile::exists(filename)) { QFile::remove(filename); } @@ -337,7 +462,7 @@ void AWSInterface::updateAWS() { stream << "import boto3\n"; stream << "s3 = boto3.resource('s3')\n\n"; - QDirIterator it1(_htmlFailuresFolder.toStdString().c_str()); + QDirIterator it1(_htmlFailuresFolder); while (it1.hasNext()) { QString nextDirectory = it1.next(); @@ -345,26 +470,26 @@ void AWSInterface::updateAWS() { if (nextDirectory.right(1) == ".") { continue; } - + // nextDirectory looks like `D:/t/TestResults--2018-10-02_16-54-11(9426)[DESKTOP-PMKNLSQ]/failures/engine.render.effect.bloom.00000` // We need to concatenate the last 3 components, to get `TestResults--2018-10-02_16-54-11(9426)[DESKTOP-PMKNLSQ]/failures/engine.render.effect.bloom.00000` QStringList parts = nextDirectory.split('/'); QString filename = parts[parts.length() - 3] + "/" + parts[parts.length() - 2] + "/" + parts[parts.length() - 1]; - stream << "data = open('" << _snapshotDirectory << "/" << filename << "/" + stream << "data = open('" << _workingDirectory << "/" << filename << "/" << "Actual Image.png" << "', 'rb')\n"; stream << "s3.Bucket('hifi-content').put_object(Bucket='" << AWS_BUCKET << "', Key='" << filename << "/" << "Actual Image.png" << "', Body=data)\n\n"; - stream << "data = open('" << _snapshotDirectory << "/" << filename << "/" + stream << "data = open('" << _workingDirectory << "/" << filename << "/" << "Expected Image.png" << "', 'rb')\n"; stream << "s3.Bucket('hifi-content').put_object(Bucket='" << AWS_BUCKET << "', Key='" << filename << "/" << "Expected Image.png" << "', Body=data)\n\n"; if (QFile::exists(_htmlFailuresFolder + "/" + parts[parts.length() - 1] + "/Difference Image.png")) { - stream << "data = open('" << _snapshotDirectory << "/" << filename << "/" + stream << "data = open('" << _workingDirectory << "/" << filename << "/" << "Difference Image.png" << "', 'rb')\n"; @@ -372,7 +497,7 @@ void AWSInterface::updateAWS() { } } - QDirIterator it2(_htmlSuccessesFolder.toStdString().c_str()); + QDirIterator it2(_htmlSuccessesFolder); while (it2.hasNext()) { QString nextDirectory = it2.next(); @@ -386,20 +511,20 @@ void AWSInterface::updateAWS() { QStringList parts = nextDirectory.split('/'); QString filename = parts[parts.length() - 3] + "/" + parts[parts.length() - 2] + "/" + parts[parts.length() - 1]; - stream << "data = open('" << _snapshotDirectory << "/" << filename << "/" + stream << "data = open('" << _workingDirectory << "/" << filename << "/" << "Actual Image.png" << "', 'rb')\n"; stream << "s3.Bucket('hifi-content').put_object(Bucket='" << AWS_BUCKET << "', Key='" << filename << "/" << "Actual Image.png" << "', Body=data)\n\n"; - stream << "data = open('" << _snapshotDirectory << "/" << filename << "/" + stream << "data = open('" << _workingDirectory << "/" << filename << "/" << "Expected Image.png" << "', 'rb')\n"; stream << "s3.Bucket('hifi-content').put_object(Bucket='" << AWS_BUCKET << "', Key='" << filename << "/" << "Expected Image.png" << "', Body=data)\n\n"; if (QFile::exists(_htmlSuccessesFolder + "/" + parts[parts.length() - 1] + "/Difference Image.png")) { - stream << "data = open('" << _snapshotDirectory << "/" << filename << "/" + stream << "data = open('" << _workingDirectory << "/" << filename << "/" << "Difference Image.png" << "', 'rb')\n"; @@ -407,7 +532,7 @@ void AWSInterface::updateAWS() { } } - stream << "data = open('" << _snapshotDirectory << "/" << _resultsFolder << "/" << HTML_FILENAME << "', 'rb')\n"; + stream << "data = open('" << _workingDirectory << "/" << _resultsFolder << "/" << HTML_FILENAME << "', 'rb')\n"; stream << "s3.Bucket('hifi-content').put_object(Bucket='" << AWS_BUCKET << "', Key='" << _resultsFolder << "/" << HTML_FILENAME << "', Body=data, ContentType='text/html')\n"; @@ -426,10 +551,10 @@ void AWSInterface::updateAWS() { [=](int exitCode, QProcess::ExitStatus exitStatus) { _busyWindow.hide(); }); #ifdef Q_OS_WIN - QStringList parameters = QStringList() << filename ; + QStringList parameters = QStringList() << filename; process->start(_pythonCommand, parameters); #elif defined Q_OS_MAC - QStringList parameters = QStringList() << "-c" << _pythonCommand + " " + filename; + QStringList parameters = QStringList() << "-c" << _pythonCommand + " " + filename; process->start("sh", parameters); #endif -} +} \ No newline at end of file diff --git a/tools/nitpick/src/AWSInterface.h b/tools/nitpick/src/AWSInterface.h index f4084f1a14..fda250b115 100644 --- a/tools/nitpick/src/AWSInterface.h +++ b/tools/nitpick/src/AWSInterface.h @@ -26,7 +26,7 @@ public: explicit AWSInterface(QObject* parent = 0); void createWebPageFromResults(const QString& testResults, - const QString& snapshotDirectory, + const QString& workingDirectory, QCheckBox* updateAWSCheckBox, QLineEdit* urlLineEdit); @@ -38,18 +38,18 @@ public: void writeBody(QTextStream& stream); void finishHTMLpage(QTextStream& stream); - void writeTitle(QTextStream& stream); - void writeTable(QTextStream& stream); - void openTable(QTextStream& stream); + void writeTitle(QTextStream& stream, const QStringList& originalNamesFailures, const QStringList& originalNamesSuccesses); + void writeTable(QTextStream& stream, const QStringList& originalNamesFailures, const QStringList& originalNamesSuccesses); + void openTable(QTextStream& stream, const QString& testResult, const bool isFailure); void closeTable(QTextStream& stream); - void createEntry(int index, const QString& testResult, QTextStream& stream, const bool isFailure); + void createEntry(const int index, const QString& testResult, QTextStream& stream, const bool isFailure); void updateAWS(); private: QString _testResults; - QString _snapshotDirectory; + QString _workingDirectory; QString _resultsFolder; QString _htmlFailuresFolder; QString _htmlSuccessesFolder; diff --git a/tools/nitpick/src/Test.cpp b/tools/nitpick/src/Test.cpp index e17978d9d0..19c49eac42 100644 --- a/tools/nitpick/src/Test.cpp +++ b/tools/nitpick/src/Test.cpp @@ -105,7 +105,7 @@ int Test::compareImageLists() { ++numberOfFailures; if (!isInteractiveMode) { - appendTestResultsToFile(_testResultsFolderPath, testResult, _mismatchWindow.getComparisonImage(), true); + appendTestResultsToFile(testResult, _mismatchWindow.getComparisonImage(), true); } else { _mismatchWindow.exec(); @@ -113,7 +113,7 @@ int Test::compareImageLists() { case USER_RESPONSE_PASS: break; case USE_RESPONSE_FAIL: - appendTestResultsToFile(_testResultsFolderPath, testResult, _mismatchWindow.getComparisonImage(), true); + appendTestResultsToFile(testResult, _mismatchWindow.getComparisonImage(), true); break; case USER_RESPONSE_ABORT: keepOn = false; @@ -124,7 +124,7 @@ int Test::compareImageLists() { } } } else { - appendTestResultsToFile(_testResultsFolderPath, testResult, _mismatchWindow.getComparisonImage(), false); + appendTestResultsToFile(testResult, _mismatchWindow.getComparisonImage(), false); } _progressBar->setValue(i); @@ -134,12 +134,36 @@ int Test::compareImageLists() { return numberOfFailures; } -void Test::appendTestResultsToFile(const QString& _testResultsFolderPath, TestResult testResult, QPixmap comparisonImage, bool hasFailed) { +int Test::checkTextResults() { + // Create lists of failed and passed tests + QStringList nameFilterFailed; + nameFilterFailed << "*.failed.txt"; + QStringList testsFailed = QDir(_snapshotDirectory).entryList(nameFilterFailed, QDir::Files, QDir::Name); + + QStringList nameFilterPassed; + nameFilterPassed << "*.passed.txt"; + QStringList testsPassed = QDir(_snapshotDirectory).entryList(nameFilterPassed, QDir::Files, QDir::Name); + + // Add results to Test Results folder + foreach(QString currentFilename, testsFailed) { + appendTestResultsToFile(currentFilename, true); + } + + foreach(QString currentFilename, testsPassed) { + appendTestResultsToFile(currentFilename, false); + } + + return testsFailed.length(); +} + +void Test::appendTestResultsToFile(TestResult testResult, QPixmap comparisonImage, bool hasFailed) { + // Critical error if Test Results folder does not exist if (!QDir().exists(_testResultsFolderPath)) { QMessageBox::critical(0, "Internal error: " + QString(__FILE__) + ":" + QString::number(__LINE__), "Folder " + _testResultsFolderPath + " not found"); exit(-1); } + // There are separate subfolders for failures and passes QString resultFolderPath; if (hasFailed) { resultFolderPath = _testResultsFolderPath + "/Failure_" + QString::number(_failureIndex) + "--" + @@ -195,6 +219,33 @@ void Test::appendTestResultsToFile(const QString& _testResultsFolderPath, TestRe comparisonImage.save(resultFolderPath + "/" + "Difference Image.png"); } +void::Test::appendTestResultsToFile(QString testResultFilename, bool hasFailed) { + // The test name includes everything until the penultimate period + QString testNameTemp = testResultFilename.left(testResultFilename.lastIndexOf('.')); + QString testName = testResultFilename.left(testNameTemp.lastIndexOf('.')); + QString resultFolderPath; + if (hasFailed) { + resultFolderPath = _testResultsFolderPath + "/Failure_" + QString::number(_failureIndex) + "--" + testName; + ++_failureIndex; + } else { + resultFolderPath = _testResultsFolderPath + "/Success_" + QString::number(_successIndex) + "--" + testName; + ++_successIndex; + } + + if (!QDir().mkdir(resultFolderPath)) { + QMessageBox::critical(0, "Internal error: " + QString(__FILE__) + ":" + QString::number(__LINE__), + "Failed to create folder " + resultFolderPath); + exit(-1); + } + + QString source = _snapshotDirectory + "/" + testResultFilename; + QString destination = resultFolderPath + "/Result.txt"; + if (!QFile::copy(source, destination)) { + QMessageBox::critical(0, "Internal error: " + QString(__FILE__) + ":" + QString::number(__LINE__), "Failed to copy " + testResultFilename + " to " + resultFolderPath); + exit(-1); + } +} + void Test::startTestsEvaluation(const bool isRunningFromCommandLine, const bool isRunningInAutomaticTestRun, const QString& snapshotDirectory, @@ -211,7 +262,7 @@ void Test::startTestsEvaluation(const bool isRunningFromCommandLine, if (!parent.isNull() && parent.right(1) != "/") { parent += "/"; } - _snapshotDirectory = QFileDialog::getExistingDirectory(nullptr, "Please select folder containing the test images", parent, + _snapshotDirectory = QFileDialog::getExistingDirectory(nullptr, "Please select folder containing the snapshots", parent, QFileDialog::ShowDirsOnly); // If user canceled then restore previous selection and return @@ -270,9 +321,14 @@ void Test::startTestsEvaluation(const bool isRunningFromCommandLine, nitpick->downloadFiles(expectedImagesURLs, _snapshotDirectory, _expectedImagesFilenames, (void *)this); } + void Test::finishTestsEvaluation() { + // First - compare the pairs of images int numberOfFailures = compareImageLists(); - + + // Next - check text results + numberOfFailures += checkTextResults(); + if (!_isRunningFromCommandLine && !_isRunningInAutomaticTestRun) { if (numberOfFailures == 0) { QMessageBox::information(0, "Success", "All images are as expected"); @@ -344,7 +400,7 @@ void Test::createTests() { parent += "/"; } - _snapshotDirectory = QFileDialog::getExistingDirectory(nullptr, "Please select folder containing the test images", parent, + _snapshotDirectory = QFileDialog::getExistingDirectory(nullptr, "Please select folder containing the snapshots", parent, QFileDialog::ShowDirsOnly); // If user canceled then restore previous selection and return @@ -542,7 +598,7 @@ void Test::createAllMDFiles() { createMDFile(_testsRootDirectory); } - QDirIterator it(_testsRootDirectory.toStdString().c_str(), QDirIterator::Subdirectories); + QDirIterator it(_testsRootDirectory, QDirIterator::Subdirectories); while (it.hasNext()) { QString directory = it.next(); @@ -636,7 +692,7 @@ void Test::createAllTestAutoScripts() { createTestAutoScript(_testsRootDirectory); } - QDirIterator it(_testsRootDirectory.toStdString().c_str(), QDirIterator::Subdirectories); + QDirIterator it(_testsRootDirectory, QDirIterator::Subdirectories); while (it.hasNext()) { QString directory = it.next(); @@ -653,7 +709,7 @@ void Test::createAllTestAutoScripts() { } } - QMessageBox::information(0, "Success", "'nitpick.js' scripts have been created"); + QMessageBox::information(0, "Success", "All 'testAuto.js' scripts have been created"); } bool Test::createTestAutoScript(const QString& directory) { @@ -704,7 +760,7 @@ void Test::createAllRecursiveScripts() { createRecursiveScript(_testsRootDirectory, false); - QDirIterator it(_testsRootDirectory.toStdString().c_str(), QDirIterator::Subdirectories); + QDirIterator it(_testsRootDirectory, QDirIterator::Subdirectories); while (it.hasNext()) { QString directory = it.next(); @@ -716,7 +772,7 @@ void Test::createAllRecursiveScripts() { // Only process directories that have sub-directories bool hasNoSubDirectories{ true }; - QDirIterator it2(directory.toStdString().c_str(), QDirIterator::Subdirectories); + QDirIterator it2(directory, QDirIterator::Subdirectories); while (it2.hasNext()) { QString directory2 = it2.next(); @@ -737,16 +793,17 @@ void Test::createAllRecursiveScripts() { } void Test::createRecursiveScript(const QString& topLevelDirectory, bool interactiveMode) { - const QString recursiveTestsFilename("testRecursive.js"); - QFile allTestsFilename(topLevelDirectory + "/" + recursiveTestsFilename); - if (!allTestsFilename.open(QIODevice::WriteOnly | QIODevice::Text)) { + const QString recursiveTestsScriptName("testRecursive.js"); + const QString recursiveTestsFilename(topLevelDirectory + "/" + recursiveTestsScriptName); + QFile recursiveTestsFile(recursiveTestsFilename); + if (!recursiveTestsFile.open(QIODevice::WriteOnly | QIODevice::Text)) { QMessageBox::critical(0, "Internal error: " + QString(__FILE__) + ":" + QString::number(__LINE__), - "Failed to create \"" + recursiveTestsFilename + "\" in directory \"" + topLevelDirectory + "\""); + "Failed to create \"" + recursiveTestsScriptName + "\" in directory \"" + topLevelDirectory + "\""); exit(-1); } - QTextStream textStream(&allTestsFilename); + QTextStream textStream(&recursiveTestsFile); textStream << "// This is an automatically generated file, created by nitpick" << endl; @@ -787,7 +844,7 @@ void Test::createRecursiveScript(const QString& topLevelDirectory, bool interact testFound = true; } - QDirIterator it(topLevelDirectory.toStdString().c_str(), QDirIterator::Subdirectories); + QDirIterator it(topLevelDirectory, QDirIterator::Subdirectories); while (it.hasNext()) { QString directory = it.next(); @@ -809,7 +866,15 @@ void Test::createRecursiveScript(const QString& topLevelDirectory, bool interact if (interactiveMode && !testFound) { QMessageBox::information(0, "Failure", "No \"" + TEST_FILENAME + "\" files found"); - allTestsFilename.close(); + recursiveTestsFile.close(); + return; + } + + // If 'directories' is empty, this means that this recursive script has no tests to call, so it is redundant + // The script will be closed and deleted + if (directories.length() == 0) { + recursiveTestsFile.close(); + QFile::remove(recursiveTestsFilename); return; } @@ -821,7 +886,7 @@ void Test::createRecursiveScript(const QString& topLevelDirectory, bool interact textStream << endl; textStream << "nitpick.runRecursive();" << endl; - allTestsFilename.close(); + recursiveTestsFile.close(); } void Test::createTestsOutline() { @@ -858,7 +923,7 @@ void Test::createTestsOutline() { int rootDepth { _testDirectory.count('/') }; // Each test is shown as the folder name linking to the matching GitHub URL, and the path to the associated test.md file - QDirIterator it(_testDirectory.toStdString().c_str(), QDirIterator::Subdirectories); + QDirIterator it(_testDirectory, QDirIterator::Subdirectories); while (it.hasNext()) { QString directory = it.next(); @@ -1052,11 +1117,11 @@ void Test::createWebPage(QCheckBox* updateAWSCheckBox, QLineEdit* urlLineEdit) { return; } - QString snapshotDirectory = QFileDialog::getExistingDirectory(nullptr, "Please select a folder to store temporary files in", + QString workingDirectory = QFileDialog::getExistingDirectory(nullptr, "Please select a folder to store temporary files in", nullptr, QFileDialog::ShowDirsOnly); - if (snapshotDirectory.isNull()) { + if (workingDirectory.isNull()) { return; } - _awsInterface.createWebPageFromResults(testResults, snapshotDirectory, updateAWSCheckBox, urlLineEdit); + _awsInterface.createWebPageFromResults(testResults, workingDirectory, updateAWSCheckBox, urlLineEdit); } \ No newline at end of file diff --git a/tools/nitpick/src/Test.h b/tools/nitpick/src/Test.h index a79252b92a..9ef7c5627a 100644 --- a/tools/nitpick/src/Test.h +++ b/tools/nitpick/src/Test.h @@ -77,6 +77,7 @@ public: void createRecursiveScript(const QString& topLevelDirectory, bool interactiveMode); int compareImageLists(); + int checkTextResults(); QStringList createListOfAll_imagesInDirectory(const QString& imageFormat, const QString& pathToImageDirectory); @@ -84,7 +85,8 @@ public: void includeTest(QTextStream& textStream, const QString& testPathname); - void appendTestResultsToFile(const QString& testResultsFolderPath, TestResult testResult, QPixmap comparisonImage, bool hasFailed); + void appendTestResultsToFile(TestResult testResult, QPixmap comparisonImage, bool hasFailed); + void appendTestResultsToFile(QString testResultFilename, bool hasFailed); bool createTestResultsFolderPath(const QString& directory); QString zipAndDeleteTestResultsFolder(); diff --git a/tools/nitpick/src/TestRailInterface.cpp b/tools/nitpick/src/TestRailInterface.cpp index a0c0d74526..1d7aa0a32f 100644 --- a/tools/nitpick/src/TestRailInterface.cpp +++ b/tools/nitpick/src/TestRailInterface.cpp @@ -275,7 +275,7 @@ void TestRailInterface::processDirectoryPython(const QString& directory, const QString& userGitHub, const QString& branchGitHub) { // Loop over all entries in directory - QDirIterator it(directory.toStdString().c_str()); + QDirIterator it(directory); while (it.hasNext()) { QString nextDirectory = it.next(); @@ -855,7 +855,7 @@ QDomElement TestRailInterface::processDirectoryXML(const QString& directory, QDomElement result = element; // Loop over all entries in directory - QDirIterator it(directory.toStdString().c_str()); + QDirIterator it(directory); while (it.hasNext()) { QString nextDirectory = it.next(); diff --git a/tools/nitpick/src/TestRunner.cpp b/tools/nitpick/src/TestRunner.cpp index 12bdf87495..9b99e114a7 100644 --- a/tools/nitpick/src/TestRunner.cpp +++ b/tools/nitpick/src/TestRunner.cpp @@ -366,7 +366,7 @@ void TestRunner::createSnapshotFolder() { // Note that we cannot use just a `png` filter, as the filenames include periods // Also, delete any `jpg` and `txt` files // The idea is to leave only previous zipped result folders - QDirIterator it(_snapshotFolder.toStdString().c_str()); + QDirIterator it(_snapshotFolder); while (it.hasNext()) { QString filename = it.next(); if (filename.right(4) == ".png" || filename.right(4) == ".jpg" || filename.right(4) == ".txt") { diff --git a/tools/nitpick/src/ui/Nitpick.cpp b/tools/nitpick/src/ui/Nitpick.cpp index 201d6e562d..cdd2ff89d9 100644 --- a/tools/nitpick/src/ui/Nitpick.cpp +++ b/tools/nitpick/src/ui/Nitpick.cpp @@ -36,7 +36,7 @@ Nitpick::Nitpick(QWidget* parent) : QMainWindow(parent) { _ui.statusLabel->setText(""); _ui.plainTextEdit->setReadOnly(true); - setWindowTitle("Nitpick - v1.1"); + setWindowTitle("Nitpick - v1.2"); // Coming soon to a nitpick near you... //// _helpWindow.textBrowser->setText()