diff --git a/tools/nitpick/README.md b/tools/nitpick/README.md index 7d75d660d7..23105a0e02 100644 --- a/tools/nitpick/README.md +++ b/tools/nitpick/README.md @@ -5,7 +5,7 @@ Nitpick is a stand alone application that provides a mechanism for regression te * The snapshots are compared to a 'canonical' set of images that have been produced beforehand. * The result, if any test failed, is a zipped folder describing the failure. -Nitpick has 5 functions, separated into 4 tabs: +Nitpick has 5 functions, separated into separate tabs: 1. Creating tests, MD files and recursive scripts 1. Windows task bar utility (Windows only) 1. Running tests @@ -22,9 +22,9 @@ Nitpick is built as part of the High Fidelity build. 1. Select all, right-click and select 7-Zip->Add to archive... 1. Set Archive format to 7z 1. Check "Create SFX archive -1. Enter installer name (i.e. `nitpick-installer-v1.1.exe`) +1. Enter installer name (i.e. `nitpick-installer-v1.2.exe`) 1. Click "OK" -1. Copy created installer to https://hifi-qa.s3.amazonaws.com/nitpick/Windows/nitpick-installer-v1.1.exe: aws s3 cp nitpick-installer-v1.1.exe s3://hifi-qa/nitpick/Mac/nitpick-installer-v1.1.exe +1. Copy created installer to https://hifi-qa.s3.amazonaws.com/nitpick/Windows/nitpick-installer-v1.2.exe: aws s3 cp nitpick-installer-v1.2.exe s3://hifi-qa/nitpick/Mac/nitpick-installer-v1.2.exe #### Mac These steps assume the hifi repository has been cloned to `~/hifi`. 1. (first time) Install brew @@ -37,12 +37,12 @@ These steps assume the hifi repository has been cloned to `~/hifi`. 1. Change the loader instruction to find the dynamic library locally In a terminal: `install_name_tool -change ~/hifi/build/ext/Xcode/quazip/project/lib/libquazip5.1.dylib libquazip5.1.dylib nitpick` 1. Delete any existing disk images. In a terminal: `rm *.dmg` -1. Create installer (note final period).In a terminal: `create-dmg --volname nitpick-installer-v1.1 nitpick-installer-v1.1.dmg .` +1. Create installer (note final period).In a terminal: `create-dmg --volname nitpick-installer-v1.2 nitpick-installer-v1.2.dmg .` Make sure to wait for completion. -1. Copy created installer to AWS: `~/Library/Python/3.7/bin/aws s3 cp nitpick-installer-v1.1.dmg s3://hifi-qa/nitpick/Mac/nitpick-installer-v1.1.dmg` +1. Copy created installer to AWS: `~/Library/Python/3.7/bin/aws s3 cp nitpick-installer-v1.2.dmg s3://hifi-qa/nitpick/Mac/nitpick-installer-v1.2.dmg` ### Installation #### Windows -1. (First time) download and install vc_redist.x64.exe (available at https://hifi-qa.s3.amazonaws.com/nitpick/Windows/nitpick-installer-v1.1.exe) +1. (First time) download and install vc_redist.x64.exe (available at https://hifi-qa.s3.amazonaws.com/nitpick/Windows/nitpick-installer-v1.2.exe) 1. (First time) download and install Python 3 from https://hifi-qa.s3.amazonaws.com/nitpick/Windows/python-3.7.0-amd64.exe (also located at https://www.python.org/downloads/) 1. After installation - create an environment variable called PYTHON_PATH and set it to the folder containing the Python executable. 1. (First time) download and install AWS CLI from https://hifi-qa.s3.amazonaws.com/nitpick/Windows/AWSCLI64PY3.msi (also available at https://aws.amazon.com/cli/ @@ -52,7 +52,7 @@ These steps assume the hifi repository has been cloned to `~/hifi`. 1. Leave region name and ouput format as default [None] 1. Install the latest release of Boto3 via pip: `pip install boto3` -1. Download the installer by browsing to [here]() +1. Download the installer by browsing to [here]() 1. Double click on the installer and install to a convenient location ![](./setup_7z.PNG) @@ -76,14 +76,14 @@ In a terminal: `python3 get-pip.py --user` 1. Enter the secret key 1. Leave region name and ouput format as default [None] 1. Install the latest release of Boto3 via pip: pip3 install boto3 -1. Download the installer by browsing to [here](). +1. Download the installer by browsing to [here](). 1. Double-click on the downloaded image to mount it 1. Create a folder for the nitpick files (e.g. ~/nitpick) If this folder exists then delete all it's contents. 1. Copy the downloaded files to the folder In a terminal: `cd ~/nitpick` - `cp -r /Volumes/nitpick-installer-v1.1/* .` + `cp -r /Volumes/nitpick-installer-v1.2/* .` 1. __To run nitpick, cd to the folder that you copied to and run `./nitpick`__ # Usage diff --git a/tools/nitpick/src/AWSInterface.cpp b/tools/nitpick/src/AWSInterface.cpp index 2e9ba7ad6d..0b93ce44e5 100644 --- a/tools/nitpick/src/AWSInterface.cpp +++ b/tools/nitpick/src/AWSInterface.cpp @@ -10,6 +10,8 @@ #include "AWSInterface.h" #include +#include +#include #include #include @@ -101,8 +103,36 @@ void AWSInterface::writeHead(QTextStream& stream) { void AWSInterface::writeBody(QTextStream& stream) { stream << "\t" << "\n"; - writeTitle(stream); - writeTable(stream); + + // The results are read here as they are used both in the title (for the summary) and for table + QStringList originalNamesFailures; + QStringList originalNamesSuccesses; + QDirIterator it1(_workingDirectory); + while (it1.hasNext()) { + QString nextDirectory = it1.next(); + + // Skip `.` and `..` directories + if (nextDirectory.right(1) == ".") { + continue; + } + + // Only process result folders + if (!nextDirectory.contains("--tests.")) { + continue; + } + + // Look at the filename at the end of the path + QStringList parts = nextDirectory.split('/'); + QString name = parts[parts.length() - 1]; + if (name.left(7) == "Failure") { + originalNamesFailures.append(nextDirectory); + } else { + originalNamesSuccesses.append(nextDirectory); + } + } + + writeTitle(stream, originalNamesFailures, originalNamesSuccesses); + writeTable(stream, originalNamesFailures, originalNamesSuccesses); stream << "\t" << "\n"; } @@ -110,7 +140,7 @@ void AWSInterface::finishHTMLpage(QTextStream& stream) { stream << "\n"; } -void AWSInterface::writeTitle(QTextStream& stream) { +void AWSInterface::writeTitle(QTextStream& stream, const QStringList& originalNamesFailures, const QStringList& originalNamesSuccesses) { // Separate relevant components from the results name // The expected format is as follows: `D:/tt/snapshots/TestResults--2018-10-04_11-09-41(PR14128)[DESKTOP-PMKNLSQ].zip` QStringList tokens = _testResults.split('/'); @@ -133,8 +163,8 @@ void AWSInterface::writeTitle(QTextStream& stream) { const QString months[] = { "Jan", "Feb", "Mar", "Apr", "May", "Jun", "Jul", "Aug", "Sep", "Oct", "Nov", "Dec" }; - stream << "\t" << "\t" << "\n"; - stream << "\t" << "\t" << "

Failures for "; + stream << "\t" << "\t" << "\n"; + stream << "\t" << "\t" << "

Results for "; stream << months[month.toInt() - 1] << " " << day << ", " << year << ", "; stream << hour << ":" << minute << ":" << second << ", "; @@ -145,9 +175,17 @@ void AWSInterface::writeTitle(QTextStream& stream) { } stream << "run on " << hostName << "

\n"; + + int numberOfFailures = originalNamesFailures.length(); + int numberOfSuccesses = originalNamesSuccesses.length(); + + stream << "

" << QString::number(numberOfFailures) << " failed, out of a total of " << QString::number(numberOfSuccesses) << " tests

\n"; + + stream << "\t" << "\t" << "\n"; + stream << "\t" << "\t" << "

The following tests failed:

"; } -void AWSInterface::writeTable(QTextStream& stream) { +void AWSInterface::writeTable(QTextStream& stream, const QStringList& originalNamesFailures, const QStringList& originalNamesSuccesses) { QString previousTestName{ "" }; // Loop over all entries in directory. This is done in stages, as the names are not in the order of the tests @@ -157,31 +195,6 @@ void AWSInterface::writeTable(QTextStream& stream) { // The fourth and lasts stage creates the HTML entries // // Note that failures are processed first, then successes - QStringList originalNamesFailures; - QStringList originalNamesSuccesses; - QDirIterator it1(_workingDirectory); - while (it1.hasNext()) { - QString nextDirectory = it1.next(); - - // Skip `.` and `..` directories - if (nextDirectory.right(1) == ".") { - continue; - } - - // Only process failure folders - if (!nextDirectory.contains("--tests.")) { - continue; - } - - // Look at the filename at the end of the path - QStringList parts = nextDirectory.split('/'); - QString name = parts[parts.length() - 1]; - if (name.left(7) == "Failure") { - originalNamesFailures.append(nextDirectory); - } else { - originalNamesSuccesses.append(nextDirectory); - } - } QStringList newNamesFailures; for (int i = 0; i < originalNamesFailures.length(); ++i) { @@ -209,9 +222,9 @@ void AWSInterface::writeTable(QTextStream& stream) { // Mac does not read folders in lexicographic order, so this step is divided into 2 // Each test consists of the test name and its index. - QDirIterator it2(_htmlFailuresFolder); QStringList folderNames; + QDirIterator it2(_htmlFailuresFolder); while (it2.hasNext()) { QString nextDirectory = it2.next(); @@ -242,8 +255,7 @@ void AWSInterface::writeTable(QTextStream& stream) { previousTestName = testName; stream << "\t\t

" << testName << "

\n"; - - openTable(stream); + openTable(stream, folderName, true); } createEntry(testNumber, folderName, stream, true); @@ -253,6 +265,9 @@ void AWSInterface::writeTable(QTextStream& stream) { stream << "\t" << "\t" << "\n"; stream << "\t" << "\t" << "

The following tests passed:

"; + // Now do the same for passes + folderNames.clear(); + QDirIterator it3(_htmlSuccessesFolder); while (it3.hasNext()) { QString nextDirectory = it3.next(); @@ -263,10 +278,17 @@ void AWSInterface::writeTable(QTextStream& stream) { } QStringList pathComponents = nextDirectory.split('/'); - QString filename = pathComponents[pathComponents.length() - 1]; - int splitIndex = filename.lastIndexOf("."); - QString testName = filename.left(splitIndex).replace(".", " / "); - QString testNumber = filename.right(filename.length() - (splitIndex + 1)); + QString folderName = pathComponents[pathComponents.length() - 1]; + + folderNames << folderName; + } + + folderNames.sort(); + for (const auto& folderName : folderNames) { + int splitIndex = folderName.lastIndexOf("."); + QString testName = folderName.left(splitIndex).replace('.', " / "); + + int testNumber = folderName.right(folderName.length() - (splitIndex + 1)).toInt(); // The failures are ordered lexicographically, so we know that we can rely on the testName changing to create a new table if (testName != previousTestName) { @@ -277,39 +299,67 @@ void AWSInterface::writeTable(QTextStream& stream) { previousTestName = testName; stream << "\t\t

" << testName << "

\n"; - - openTable(stream); + openTable(stream, folderName, false); } - createEntry(testNumber.toInt(), filename, stream, false); + createEntry(testNumber, folderName, stream, false); } closeTable(stream); } -void AWSInterface::openTable(QTextStream& stream) { - stream << "\t\t\n"; - stream << "\t\t\t\n"; - stream << "\t\t\t\t\n"; - stream << "\t\t\t\t\n"; - stream << "\t\t\t\t\n"; - stream << "\t\t\t\t\n"; - stream << "\t\t\t\n"; +void AWSInterface::openTable(QTextStream& stream, const QString& testResult, const bool isFailure) { + QStringList resultNameComponents = testResult.split('/'); + QString resultName = resultNameComponents[resultNameComponents.length() - 1]; + + bool textResultsFileFound; + if (isFailure) { + textResultsFileFound = QFile::exists(_htmlFailuresFolder + "/" + resultName + "/Result.txt"); + } else { + textResultsFileFound = QFile::exists(_htmlSuccessesFolder + "/" + resultName + "/Result.txt"); + } + + if (textResultsFileFound) { + if (isFailure) { + stream << "\t\t

Test

Actual Image

Expected Image

Difference Image

\n"; + stream << "\t\t\t\n"; + stream << "\t\t\t\t\n"; + stream << "\t\t\t\t\n"; + stream << "\t\t\t\t\n"; + stream << "\t\t\t\n"; + } else { + stream << "\t\t

No errors found

\n\n"; + stream << "\t\t

===============

\n\n"; + } + } else { + stream << "\t\t

Element

Actual Value

Expected Value

\n"; + stream << "\t\t\t\n"; + stream << "\t\t\t\t\n"; + stream << "\t\t\t\t\n"; + stream << "\t\t\t\t\n"; + stream << "\t\t\t\t\n"; + stream << "\t\t\t\n"; + } } void AWSInterface::closeTable(QTextStream& stream) { stream << "\t\t

Test

Actual Image

Expected Image

Difference Image

\n"; } -void AWSInterface::createEntry(int index, const QString& testResult, QTextStream& stream, const bool isFailure) { - stream << "\t\t\t\n"; - stream << "\t\t\t\t

" << QString::number(index) << "

\n"; - +void AWSInterface::createEntry(const int index, const QString& testResult, QTextStream& stream, const bool isFailure) { // For a test named `D:/t/fgadhcUDHSFaidsfh3478JJJFSDFIUSOEIrf/Failure_1--tests.engine.interaction.pick.collision.many.00000` // we need `Failure_1--tests.engine.interaction.pick.collision.many.00000` QStringList resultNameComponents = testResult.split('/'); QString resultName = resultNameComponents[resultNameComponents.length() - 1]; + QString textResultFilename; + if (isFailure) { + textResultFilename = _htmlFailuresFolder + "/" + resultName + "/Result.txt"; + } else { + textResultFilename = _htmlSuccessesFolder + "/" + resultName + "/Result.txt"; + } + bool textResultsFileFound{ QFile::exists(textResultFilename) }; + QString folder; bool differenceFileFound; if (isFailure) { @@ -320,17 +370,78 @@ void AWSInterface::createEntry(int index, const QString& testResult, QTextStream differenceFileFound = QFile::exists(_htmlSuccessesFolder + "/" + resultName + "/Difference Image.png"); } + if (textResultsFileFound) { + // Parse the JSON file + QFile file; + file.setFileName(textResultFilename); + if (!file.open(QIODevice::ReadOnly | QIODevice::Text)) { + QMessageBox::critical(0, "Internal error: " + QString(__FILE__) + ":" + QString::number(__LINE__), + "Failed to open file " + textResultFilename); + } - stream << "\t\t\t\t\n"; - stream << "\t\t\t\t\n"; + QString value = file.readAll(); + file.close(); - if (differenceFileFound) { - stream << "\t\t\t\t\n"; + // The Result.txt file is an object containing elements such as the following: + // "angularDamping": { + // "actual": 0.3938899040222168, + // "expected" : 0.3938899, + // "result" : "pass" + // }, + // + // Failures are thos element that have "fail for the result + + QJsonDocument document = QJsonDocument::fromJson(value.toUtf8()); + QJsonObject json = document.object(); + foreach(const QString& key, json.keys()) { + QJsonValue value = json.value(key); + QJsonObject object = value.toObject(); + + QJsonValue actualValue = object.value("actual"); + QString actualValueString; + if (actualValue.isString()) { + actualValueString = actualValue.toString(); + } else if (actualValue.isBool()) { + actualValueString = actualValue.toBool() ? "true" : "false"; + } else if (actualValue.isDouble()) { + actualValueString = QString::number(actualValue.toDouble()); + } + + QJsonValue expectedValue = object.value("expected"); + QString expectedValueString; + if (expectedValue.isString()) { + expectedValueString = expectedValue.toString(); + } else if (expectedValue.isBool()) { + expectedValueString = expectedValue.toBool() ? "true" : "false"; + } else if (expectedValue.isDouble()) { + expectedValueString = QString::number(expectedValue.toDouble()); + } + QString result = object.value("result").toString(); + + if (result == "fail") { + stream << "\t\t\t\n"; + stream << "\t\t\t\t" + key + "\n"; + stream << "\t\t\t\t" + actualValueString + "\n"; + stream << "\t\t\t\t" + expectedValueString + "\n"; + stream << "\t\t\t\n"; + } + } } else { - stream << "\t\t\t\t

No Image Found

\n"; + stream << "\t\t\t\n"; + stream << "\t\t\t\t

" << QString::number(index) << "

\n"; + + stream << "\t\t\t\t\n"; + stream << "\t\t\t\t\n"; + + if (differenceFileFound) { + stream << "\t\t\t\t\n"; + } else { + stream << "\t\t\t\t

No Image Found

\n"; + } + + stream << "\t\t\t\n"; } - stream << "\t\t\t\n"; } void AWSInterface::updateAWS() { diff --git a/tools/nitpick/src/AWSInterface.h b/tools/nitpick/src/AWSInterface.h index c5be5f35bb..fda250b115 100644 --- a/tools/nitpick/src/AWSInterface.h +++ b/tools/nitpick/src/AWSInterface.h @@ -38,12 +38,12 @@ public: void writeBody(QTextStream& stream); void finishHTMLpage(QTextStream& stream); - void writeTitle(QTextStream& stream); - void writeTable(QTextStream& stream); - void openTable(QTextStream& stream); + void writeTitle(QTextStream& stream, const QStringList& originalNamesFailures, const QStringList& originalNamesSuccesses); + void writeTable(QTextStream& stream, const QStringList& originalNamesFailures, const QStringList& originalNamesSuccesses); + void openTable(QTextStream& stream, const QString& testResult, const bool isFailure); void closeTable(QTextStream& stream); - void createEntry(int index, const QString& testResult, QTextStream& stream, const bool isFailure); + void createEntry(const int index, const QString& testResult, QTextStream& stream, const bool isFailure); void updateAWS(); diff --git a/tools/nitpick/src/Test.cpp b/tools/nitpick/src/Test.cpp index 477ced068c..19c49eac42 100644 --- a/tools/nitpick/src/Test.cpp +++ b/tools/nitpick/src/Test.cpp @@ -146,6 +146,11 @@ int Test::checkTextResults() { // Add results to Test Results folder foreach(QString currentFilename, testsFailed) { + appendTestResultsToFile(currentFilename, true); + } + + foreach(QString currentFilename, testsPassed) { + appendTestResultsToFile(currentFilename, false); } return testsFailed.length(); @@ -215,16 +220,27 @@ void Test::appendTestResultsToFile(TestResult testResult, QPixmap comparisonImag } void::Test::appendTestResultsToFile(QString testResultFilename, bool hasFailed) { - QString resultFolderPath { _testResultsFolderPath }; + // The test name includes everything until the penultimate period + QString testNameTemp = testResultFilename.left(testResultFilename.lastIndexOf('.')); + QString testName = testResultFilename.left(testNameTemp.lastIndexOf('.')); + QString resultFolderPath; if (hasFailed) { - resultFolderPath += "/Failure_"; + resultFolderPath = _testResultsFolderPath + "/Failure_" + QString::number(_failureIndex) + "--" + testName; ++_failureIndex; } else { - resultFolderPath += "/Success_"; + resultFolderPath = _testResultsFolderPath + "/Success_" + QString::number(_successIndex) + "--" + testName; ++_successIndex; } - if (!QFile::copy(testResultFilename, resultFolderPath)) { + if (!QDir().mkdir(resultFolderPath)) { + QMessageBox::critical(0, "Internal error: " + QString(__FILE__) + ":" + QString::number(__LINE__), + "Failed to create folder " + resultFolderPath); + exit(-1); + } + + QString source = _snapshotDirectory + "/" + testResultFilename; + QString destination = resultFolderPath + "/Result.txt"; + if (!QFile::copy(source, destination)) { QMessageBox::critical(0, "Internal error: " + QString(__FILE__) + ":" + QString::number(__LINE__), "Failed to copy " + testResultFilename + " to " + resultFolderPath); exit(-1); } @@ -246,7 +262,7 @@ void Test::startTestsEvaluation(const bool isRunningFromCommandLine, if (!parent.isNull() && parent.right(1) != "/") { parent += "/"; } - _snapshotDirectory = QFileDialog::getExistingDirectory(nullptr, "Please select folder containing the test images", parent, + _snapshotDirectory = QFileDialog::getExistingDirectory(nullptr, "Please select folder containing the snapshots", parent, QFileDialog::ShowDirsOnly); // If user canceled then restore previous selection and return @@ -384,7 +400,7 @@ void Test::createTests() { parent += "/"; } - _snapshotDirectory = QFileDialog::getExistingDirectory(nullptr, "Please select folder containing the test images", parent, + _snapshotDirectory = QFileDialog::getExistingDirectory(nullptr, "Please select folder containing the snapshots", parent, QFileDialog::ShowDirsOnly); // If user canceled then restore previous selection and return