Merge branch '20116_nitpick_v1.2' of github.com:NissimHadar/hifi into 20116_nitpick_v1.2

tools/nitpick/README.md

@@ -5,7 +5,7 @@ Nitpick is a stand alone application that provides a mechanism for regression te
 * The snapshots are compared to a 'canonical' set of images that have been produced beforehand.
 * The result, if any test failed, is a zipped folder describing the failure.
 
-Nitpick has 5 functions, separated into 4 tabs:
+Nitpick has 5 functions, separated into separate tabs:
 1. Creating tests, MD files and recursive scripts
 1. Windows task bar utility (Windows only)
 1. Running tests
@@ -22,9 +22,9 @@ Nitpick is built as part of the High Fidelity build.
 1.  Select all, right-click and select 7-Zip->Add to archive...
 1.  Set Archive format to 7z
 1.  Check "Create SFX archive
-1.  Enter installer name (i.e. `nitpick-installer-v1.1.exe`)
+1.  Enter installer name (i.e. `nitpick-installer-v1.2.exe`)
 1.  Click "OK"
-1.  Copy created installer to https://hifi-qa.s3.amazonaws.com/nitpick/Windows/nitpick-installer-v1.1.exe: aws s3 cp nitpick-installer-v1.1.exe s3://hifi-qa/nitpick/Mac/nitpick-installer-v1.1.exe
+1.  Copy created installer to https://hifi-qa.s3.amazonaws.com/nitpick/Windows/nitpick-installer-v1.2.exe: aws s3 cp nitpick-installer-v1.2.exe s3://hifi-qa/nitpick/Mac/nitpick-installer-v1.2.exe
 #### Mac
 These steps assume the hifi repository has been cloned to `~/hifi`.
 1.  (first time) Install brew
@@ -37,12 +37,12 @@ These steps assume the hifi repository has been cloned to `~/hifi`.
 1.  Change the loader instruction to find the dynamic library locally
     In a terminal: `install_name_tool -change ~/hifi/build/ext/Xcode/quazip/project/lib/libquazip5.1.dylib libquazip5.1.dylib nitpick`
 1.  Delete any existing disk images. In a terminal: `rm *.dmg`
-1.  Create installer (note final period).In a terminal: `create-dmg --volname nitpick-installer-v1.1 nitpick-installer-v1.1.dmg .`  
+1.  Create installer (note final period).In a terminal: `create-dmg --volname nitpick-installer-v1.2 nitpick-installer-v1.2.dmg .`  
     Make sure to wait for completion.
-1.  Copy created installer to AWS: `~/Library/Python/3.7/bin/aws s3 cp nitpick-installer-v1.1.dmg s3://hifi-qa/nitpick/Mac/nitpick-installer-v1.1.dmg`
+1.  Copy created installer to AWS: `~/Library/Python/3.7/bin/aws s3 cp nitpick-installer-v1.2.dmg s3://hifi-qa/nitpick/Mac/nitpick-installer-v1.2.dmg`
 ### Installation
 #### Windows
-1.  (First time) download and install vc_redist.x64.exe (available at https://hifi-qa.s3.amazonaws.com/nitpick/Windows/nitpick-installer-v1.1.exe)
+1.  (First time) download and install vc_redist.x64.exe (available at https://hifi-qa.s3.amazonaws.com/nitpick/Windows/nitpick-installer-v1.2.exe)
 1.  (First time) download and install Python 3 from https://hifi-qa.s3.amazonaws.com/nitpick/Windows/python-3.7.0-amd64.exe (also located at https://www.python.org/downloads/)
     1. After installation - create an environment variable called PYTHON_PATH and set it to the folder containing the Python executable.
 1.  (First time) download and install AWS CLI from https://hifi-qa.s3.amazonaws.com/nitpick/Windows/AWSCLI64PY3.msi (also available at https://aws.amazon.com/cli/
@@ -52,7 +52,7 @@ These steps assume the hifi repository has been cloned to `~/hifi`.
     1.  Leave region name and ouput format as default [None]
     1.  Install the latest release of Boto3 via pip:  `pip install boto3`
 
-1. Download the installer by browsing to [here](<https://hifi-qa.s3.amazonaws.com/nitpick/Windows/nitpick-installer-v1.1.exe>)
+1. Download the installer by browsing to [here](<https://hifi-qa.s3.amazonaws.com/nitpick/Windows/nitpick-installer-v1.2.exe>)
 1. Double click on the installer and install to a convenient location  
 ![](./setup_7z.PNG)
 
@@ -76,14 +76,14 @@ In a terminal: `python3 get-pip.py --user`
     1.  Enter the secret key
     1.  Leave region name and ouput format as default [None]
     1.  Install the latest release of Boto3 via pip:  pip3 install boto3
-1.  Download the installer by browsing to [here](<https://hifi-qa.s3.amazonaws.com/nitpick/Mac/nitpick-installer-v1.1.dmg>).
+1.  Download the installer by browsing to [here](<https://hifi-qa.s3.amazonaws.com/nitpick/Mac/nitpick-installer-v1.2.dmg>).
 1.  Double-click on the downloaded image to mount it
 1. Create a folder for the nitpick files (e.g. ~/nitpick)
    If this folder exists then delete all it's contents.
 1. Copy the downloaded files to the folder  
    In a terminal:  
      `cd ~/nitpick`  
-     `cp -r /Volumes/nitpick-installer-v1.1/* .`
+     `cp -r /Volumes/nitpick-installer-v1.2/* .`
 
 1. __To run nitpick, cd to the folder that you copied to and run `./nitpick`__  
 # Usage

tools/nitpick/src/AWSInterface.cpp

@@ -10,6 +10,8 @@
 #include "AWSInterface.h"
 
 #include <QDirIterator>
+#include <QJsonDocument>
+#include <QJsonObject>
 #include <QMessageBox>
 #include <QProcess>
 
@@ -101,8 +103,36 @@ void AWSInterface::writeHead(QTextStream& stream) {
 
 void AWSInterface::writeBody(QTextStream& stream) {
     stream << "\t" << "<body>\n";
-    writeTitle(stream);
+
-    writeTable(stream);
+    // The results are read here as they are used both in the title (for the summary) and for table
+    QStringList originalNamesFailures;
+    QStringList originalNamesSuccesses;
+    QDirIterator it1(_workingDirectory);
+    while (it1.hasNext()) {
+        QString nextDirectory = it1.next();
+
+        // Skip `.` and `..` directories
+        if (nextDirectory.right(1) == ".") {
+            continue;
+        }
+
+        // Only process result folders
+        if (!nextDirectory.contains("--tests.")) {
+            continue;
+        }
+
+        // Look at the filename at the end of the path
+        QStringList parts = nextDirectory.split('/');
+        QString name = parts[parts.length() - 1];
+        if (name.left(7) == "Failure") {
+            originalNamesFailures.append(nextDirectory);
+        } else {
+            originalNamesSuccesses.append(nextDirectory);
+        }
+    }
+
+    writeTitle(stream, originalNamesFailures, originalNamesSuccesses);
+    writeTable(stream, originalNamesFailures, originalNamesSuccesses);
     stream << "\t" << "</body>\n";
 }
 
@@ -110,7 +140,7 @@ void AWSInterface::finishHTMLpage(QTextStream& stream) {
     stream << "</html>\n";
 }
 
-void AWSInterface::writeTitle(QTextStream& stream) {
+void AWSInterface::writeTitle(QTextStream& stream, const QStringList& originalNamesFailures, const QStringList& originalNamesSuccesses) {
     // Separate relevant components from the results name
     // The expected format is as follows: `D:/tt/snapshots/TestResults--2018-10-04_11-09-41(PR14128)[DESKTOP-PMKNLSQ].zip`
     QStringList tokens = _testResults.split('/');
@@ -133,8 +163,8 @@ void AWSInterface::writeTitle(QTextStream& stream) {
 
     const QString months[] = { "Jan", "Feb", "Mar", "Apr", "May", "Jun", "Jul", "Aug", "Sep", "Oct", "Nov", "Dec" };
 
-    stream << "\t" << "\t" << "<font color=\"red\">\n";
+    stream << "\t" << "\t" << "<font color=\"Green\">\n";
-    stream << "\t" << "\t" << "<h1>Failures for ";
+    stream << "\t" << "\t" << "<h1>Results for ";
     stream << months[month.toInt() - 1] << " " << day << ", " << year << ", ";
     stream << hour << ":" << minute << ":" << second << ", ";
 
@@ -145,9 +175,17 @@ void AWSInterface::writeTitle(QTextStream& stream) {
     }
 
     stream << "run on " << hostName << "</h1>\n";
+
+    int numberOfFailures = originalNamesFailures.length();
+    int numberOfSuccesses = originalNamesSuccesses.length();
+
+    stream << "<h2>" << QString::number(numberOfFailures) << " failed, out of a total of " << QString::number(numberOfSuccesses) << " tests</h2>\n";
+
+    stream << "\t" << "\t" << "<font color=\"red\">\n";
+    stream << "\t" << "\t" << "<h1>The following tests failed:</h1>";
 }
 
-void AWSInterface::writeTable(QTextStream& stream) {
+void AWSInterface::writeTable(QTextStream& stream, const QStringList& originalNamesFailures, const QStringList& originalNamesSuccesses) {
     QString previousTestName{ "" };
 
     // Loop over all entries in directory.  This is done in stages, as the names are not in the order of the tests
@@ -157,31 +195,6 @@ void AWSInterface::writeTable(QTextStream& stream) {
     //      The fourth and lasts stage creates the HTML entries
     //
     // Note that failures are processed first, then successes
-    QStringList originalNamesFailures;
-    QStringList originalNamesSuccesses;
-    QDirIterator it1(_workingDirectory);
-    while (it1.hasNext()) {
-        QString nextDirectory = it1.next();
-
-        // Skip `.` and `..` directories
-        if (nextDirectory.right(1) == ".") {
-            continue;
-        }
-
-        // Only process failure folders
-        if (!nextDirectory.contains("--tests.")) {
-            continue;
-        }
-
-        // Look at the filename at the end of the path
-        QStringList parts = nextDirectory.split('/');
-        QString name = parts[parts.length() - 1];
-        if (name.left(7) == "Failure") {
-            originalNamesFailures.append(nextDirectory);
-        } else {
-            originalNamesSuccesses.append(nextDirectory);
-        }
-    }
 
     QStringList newNamesFailures;
     for (int i = 0; i < originalNamesFailures.length(); ++i) {
@@ -209,9 +222,9 @@ void AWSInterface::writeTable(QTextStream& stream) {
 
     // Mac does not read folders in lexicographic order, so this step is divided into 2
     // Each test consists of the test name and its index.
-    QDirIterator it2(_htmlFailuresFolder);
     QStringList folderNames;
 
+    QDirIterator it2(_htmlFailuresFolder);
     while (it2.hasNext()) {
         QString nextDirectory = it2.next();
 
@@ -242,8 +255,7 @@ void AWSInterface::writeTable(QTextStream& stream) {
             previousTestName = testName;
 
             stream << "\t\t<h2>" << testName << "</h2>\n";
-
+            openTable(stream, folderName, true);
-            openTable(stream);
         }
 
         createEntry(testNumber, folderName, stream, true);
@@ -253,6 +265,9 @@ void AWSInterface::writeTable(QTextStream& stream) {
     stream << "\t" << "\t" << "<font color=\"blue\">\n";
     stream << "\t" << "\t" << "<h1>The following tests passed:</h1>";
 
+    // Now do the same for passes
+    folderNames.clear();
+
     QDirIterator it3(_htmlSuccessesFolder);
     while (it3.hasNext()) {
         QString nextDirectory = it3.next();
@@ -263,10 +278,17 @@ void AWSInterface::writeTable(QTextStream& stream) {
         }
 
         QStringList pathComponents = nextDirectory.split('/');
-        QString filename = pathComponents[pathComponents.length() - 1];
+        QString folderName = pathComponents[pathComponents.length() - 1];
-        int splitIndex = filename.lastIndexOf(".");
+
-        QString testName = filename.left(splitIndex).replace(".", " / ");
+        folderNames << folderName;
-        QString testNumber = filename.right(filename.length() - (splitIndex + 1));
+    }
+
+    folderNames.sort();
+    for (const auto& folderName : folderNames) {
+        int splitIndex = folderName.lastIndexOf(".");
+        QString testName = folderName.left(splitIndex).replace('.', " / ");
+
+        int testNumber = folderName.right(folderName.length() - (splitIndex + 1)).toInt();
 
         // The failures are ordered lexicographically, so we know that we can rely on the testName changing to create a new table
         if (testName != previousTestName) {
@@ -277,39 +299,67 @@ void AWSInterface::writeTable(QTextStream& stream) {
             previousTestName = testName;
 
             stream << "\t\t<h2>" << testName << "</h2>\n";
-
+            openTable(stream, folderName, false);
-            openTable(stream);
         }
 
-        createEntry(testNumber.toInt(), filename, stream, false);
+        createEntry(testNumber, folderName, stream, false);
     }
 
     closeTable(stream);
 }
 
-void AWSInterface::openTable(QTextStream& stream) {
+void AWSInterface::openTable(QTextStream& stream, const QString& testResult, const bool isFailure) {
-    stream << "\t\t<table>\n";
+    QStringList resultNameComponents = testResult.split('/');
-    stream << "\t\t\t<tr>\n";
+    QString resultName = resultNameComponents[resultNameComponents.length() - 1];
-    stream << "\t\t\t\t<th><h1>Test</h1></th>\n";
+
-    stream << "\t\t\t\t<th><h1>Actual Image</h1></th>\n";
+    bool textResultsFileFound;
-    stream << "\t\t\t\t<th><h1>Expected Image</h1></th>\n";
+    if (isFailure) {
-    stream << "\t\t\t\t<th><h1>Difference Image</h1></th>\n";
+        textResultsFileFound = QFile::exists(_htmlFailuresFolder + "/" + resultName + "/Result.txt");
-    stream << "\t\t\t</tr>\n";
+    } else {
+        textResultsFileFound = QFile::exists(_htmlSuccessesFolder + "/" + resultName + "/Result.txt");
+    }
+
+    if (textResultsFileFound) {
+        if (isFailure) {
+            stream << "\t\t<table>\n";
+            stream << "\t\t\t<tr>\n";
+            stream << "\t\t\t\t<th><h1>Element</h1></th>\n";
+            stream << "\t\t\t\t<th><h1>Actual Value</h1></th>\n";
+            stream << "\t\t\t\t<th><h1>Expected Value</h1></th>\n";
+            stream << "\t\t\t</tr>\n";
+        } else {
+            stream << "\t\t<h3>No errors found</h3>\n\n";
+            stream << "\t\t<h3>===============</h3>\n\n";
+        }
+    } else {
+        stream << "\t\t<table>\n";
+        stream << "\t\t\t<tr>\n";
+        stream << "\t\t\t\t<th><h1>Test</h1></th>\n";
+        stream << "\t\t\t\t<th><h1>Actual Image</h1></th>\n";
+        stream << "\t\t\t\t<th><h1>Expected Image</h1></th>\n";
+        stream << "\t\t\t\t<th><h1>Difference Image</h1></th>\n";
+        stream << "\t\t\t</tr>\n";
+    }
 }
 
 void AWSInterface::closeTable(QTextStream& stream) {
     stream << "\t\t</table>\n";
 }
 
-void AWSInterface::createEntry(int index, const QString& testResult, QTextStream& stream, const bool isFailure) {
+void AWSInterface::createEntry(const int index, const QString& testResult, QTextStream& stream, const bool isFailure) {
-    stream << "\t\t\t<tr>\n";
-    stream << "\t\t\t\t<td><h1>" << QString::number(index) << "</h1></td>\n";
-
     // For a test named `D:/t/fgadhcUDHSFaidsfh3478JJJFSDFIUSOEIrf/Failure_1--tests.engine.interaction.pick.collision.many.00000`
     // we need `Failure_1--tests.engine.interaction.pick.collision.many.00000`
     QStringList resultNameComponents = testResult.split('/');
     QString resultName = resultNameComponents[resultNameComponents.length() - 1];
 
+    QString textResultFilename;
+    if (isFailure) {
+        textResultFilename = _htmlFailuresFolder + "/" + resultName + "/Result.txt";
+    } else {
+        textResultFilename = _htmlSuccessesFolder + "/" + resultName + "/Result.txt";
+    }
+    bool textResultsFileFound{ QFile::exists(textResultFilename) };
+
     QString folder;
     bool differenceFileFound;
     if (isFailure) {
@@ -320,17 +370,78 @@ void AWSInterface::createEntry(int index, const QString& testResult, QTextStream
         differenceFileFound = QFile::exists(_htmlSuccessesFolder + "/" + resultName + "/Difference Image.png");
     }
 
+    if (textResultsFileFound) {
+        // Parse the JSON file
+        QFile file;
+        file.setFileName(textResultFilename);
+        if (!file.open(QIODevice::ReadOnly | QIODevice::Text)) {
+            QMessageBox::critical(0, "Internal error: " + QString(__FILE__) + ":" + QString::number(__LINE__),
+                "Failed to open file " + textResultFilename);
+        }
 
-    stream << "\t\t\t\t<td><img src=\"./" << folder << "/" << resultName << "/Actual Image.png\" width = \"576\" height = \"324\" ></td>\n";
+        QString value = file.readAll();
-    stream << "\t\t\t\t<td><img src=\"./" << folder << "/" << resultName << "/Expected Image.png\" width = \"576\" height = \"324\" ></td>\n";
+        file.close();
 
-    if (differenceFileFound) {
+        // The Result.txt file is an object containing elements such as the following:
-        stream << "\t\t\t\t<td><img src=\"./" << folder << "/" << resultName << "/Difference Image.png\" width = \"576\" height = \"324\" ></td>\n";
+        //    "angularDamping": {
+        //        "actual": 0.3938899040222168,
+        //        "expected" : 0.3938899,
+        //        "result" : "pass"
+        //    },
+        //
+        // Failures are thos element that have "fail for the result
+
+        QJsonDocument document = QJsonDocument::fromJson(value.toUtf8());
+        QJsonObject json = document.object();
+        foreach(const QString& key, json.keys()) {
+            QJsonValue value = json.value(key);
+            QJsonObject object = value.toObject();
+
+            QJsonValue actualValue = object.value("actual");
+            QString actualValueString;
+            if (actualValue.isString()) {
+                actualValueString = actualValue.toString();
+            } else if (actualValue.isBool()) {
+                actualValueString = actualValue.toBool() ? "true" : "false";
+            } else if (actualValue.isDouble()) {
+                actualValueString = QString::number(actualValue.toDouble());
+            }
+
+            QJsonValue expectedValue = object.value("expected");
+            QString expectedValueString;
+            if (expectedValue.isString()) {
+                expectedValueString = expectedValue.toString();
+            } else if (expectedValue.isBool()) {
+                expectedValueString = expectedValue.toBool() ? "true" : "false";
+            } else if (expectedValue.isDouble()) {
+                expectedValueString = QString::number(expectedValue.toDouble());
+            }
+            QString result = object.value("result").toString();
+           
+            if (result == "fail") {
+                stream << "\t\t\t<tr>\n";
+                stream << "\t\t\t\t<td><font size=\"6\">" + key + "</td>\n";
+                stream << "\t\t\t\t<td><font size=\"6\">" + actualValueString + "</td>\n";
+                stream << "\t\t\t\t<td><font size=\"6\">" + expectedValueString + "</td>\n";
+                stream << "\t\t\t</tr>\n";
+            }
+        }
     } else {
-        stream << "\t\t\t\t<td><h2>No Image Found</h2>\n";
+        stream << "\t\t\t<tr>\n";
+        stream << "\t\t\t\t<td><h1>" << QString::number(index) << "</h1></td>\n";
+
+        stream << "\t\t\t\t<td><img src=\"./" << folder << "/" << resultName << "/Actual Image.png\" width = \"576\" height = \"324\" ></td>\n";
+        stream << "\t\t\t\t<td><img src=\"./" << folder << "/" << resultName << "/Expected Image.png\" width = \"576\" height = \"324\" ></td>\n";
+
+        if (differenceFileFound) {
+            stream << "\t\t\t\t<td><img src=\"./" << folder << "/" << resultName << "/Difference Image.png\" width = \"576\" height = \"324\" ></td>\n";
+        } else {
+            stream << "\t\t\t\t<td><h2>No Image Found</h2>\n";
+        }
+
+        stream << "\t\t\t</tr>\n";
     }
 
-    stream << "\t\t\t</tr>\n";
 }
 
 void AWSInterface::updateAWS() {

tools/nitpick/src/AWSInterface.h

@@ -38,12 +38,12 @@ public:
     void writeBody(QTextStream& stream);
     void finishHTMLpage(QTextStream& stream);
 
-    void writeTitle(QTextStream& stream);
+    void writeTitle(QTextStream& stream, const QStringList& originalNamesFailures, const QStringList& originalNamesSuccesses);
-    void writeTable(QTextStream& stream);
+    void writeTable(QTextStream& stream, const QStringList& originalNamesFailures, const QStringList& originalNamesSuccesses);
-    void openTable(QTextStream& stream);
+    void openTable(QTextStream& stream, const QString& testResult, const bool isFailure);
     void closeTable(QTextStream& stream);
 
-    void createEntry(int index, const QString& testResult, QTextStream& stream, const bool isFailure);
+    void createEntry(const int index, const QString& testResult, QTextStream& stream, const bool isFailure);
 
     void updateAWS();
 

tools/nitpick/src/Test.cpp

@@ -146,6 +146,11 @@ int Test::checkTextResults() {
 
     // Add results to Test Results folder
     foreach(QString currentFilename, testsFailed) {
+        appendTestResultsToFile(currentFilename, true);
+    }
+
+    foreach(QString currentFilename, testsPassed) {
+        appendTestResultsToFile(currentFilename, false);
     }
 
     return testsFailed.length();
@@ -215,16 +220,27 @@ void Test::appendTestResultsToFile(TestResult testResult, QPixmap comparisonImag
 }
 
 void::Test::appendTestResultsToFile(QString testResultFilename, bool hasFailed) {
-    QString resultFolderPath { _testResultsFolderPath };
+    // The test name includes everything until the penultimate period
+    QString testNameTemp = testResultFilename.left(testResultFilename.lastIndexOf('.'));
+    QString testName = testResultFilename.left(testNameTemp.lastIndexOf('.'));
+    QString resultFolderPath;
     if (hasFailed) {
-        resultFolderPath += "/Failure_";
+        resultFolderPath = _testResultsFolderPath + "/Failure_" + QString::number(_failureIndex) + "--" + testName;
         ++_failureIndex;
     } else {
-        resultFolderPath += "/Success_";
+        resultFolderPath = _testResultsFolderPath + "/Success_" + QString::number(_successIndex) + "--" + testName;
         ++_successIndex;
     }
 
-    if (!QFile::copy(testResultFilename, resultFolderPath)) {
+    if (!QDir().mkdir(resultFolderPath)) {
+        QMessageBox::critical(0, "Internal error: " + QString(__FILE__) + ":" + QString::number(__LINE__),
+            "Failed to create folder " + resultFolderPath);
+        exit(-1);
+    }
+
+    QString source = _snapshotDirectory + "/" + testResultFilename;
+    QString destination = resultFolderPath + "/Result.txt";
+    if (!QFile::copy(source, destination)) {
         QMessageBox::critical(0, "Internal error: " + QString(__FILE__) + ":" + QString::number(__LINE__), "Failed to copy " + testResultFilename + " to " + resultFolderPath);
         exit(-1);
     }
@@ -246,7 +262,7 @@ void Test::startTestsEvaluation(const bool isRunningFromCommandLine,
         if (!parent.isNull() && parent.right(1) != "/") {
             parent += "/";
         }
-        _snapshotDirectory = QFileDialog::getExistingDirectory(nullptr, "Please select folder containing the test images", parent,
+        _snapshotDirectory = QFileDialog::getExistingDirectory(nullptr, "Please select folder containing the snapshots", parent,
             QFileDialog::ShowDirsOnly);
 
         // If user canceled then restore previous selection and return
@@ -384,7 +400,7 @@ void Test::createTests() {
         parent += "/";
     }
 
-    _snapshotDirectory = QFileDialog::getExistingDirectory(nullptr, "Please select folder containing the test images", parent,
+    _snapshotDirectory = QFileDialog::getExistingDirectory(nullptr, "Please select folder containing the snapshots", parent,
                                                           QFileDialog::ShowDirsOnly);
 
     // If user canceled then restore previous selection and return