Merge pull request #15140 from NissimHadar/21616-addPerTileThreshold

Case 21616: Add per tile threshold

tools/nitpick/src/ImageComparer.cpp

@@ -43,6 +43,8 @@ void ImageComparer::compareImages(const QImage& resultImage, const QImage& expec
 
     int windowCounter{ 0 };
     double ssim{ 0.0 };
+    double worstTileValue{ 1.0 };
+
     double min { 1.0 };
     double max { -1.0 };
 
@@ -108,6 +110,10 @@ void ImageComparer::compareImages(const QImage& resultImage, const QImage& expec
             if (value < min) min = value;
             if (value > max) max = value;
 
+            if (value < worstTileValue) {
+                worstTileValue = value;
+            }
+
             ++windowCounter;
 
             y += WIN_SIZE;
@@ -122,12 +128,17 @@ void ImageComparer::compareImages(const QImage& resultImage, const QImage& expec
     _ssimResults.min = min;
     _ssimResults.max = max;
     _ssimResults.ssim = ssim / windowCounter;
+    _ssimResults.worstTileValue = worstTileValue;
 };
 
 double ImageComparer::getSSIMValue() {
     return _ssimResults.ssim;
 }
 
+double ImageComparer::getWorstTileValue() {
+    return _ssimResults.worstTileValue;
+}
+
 SSIMResults ImageComparer::getSSIMResults() {
     return _ssimResults;
 }

tools/nitpick/src/ImageComparer.h

@@ -18,7 +18,9 @@
 class ImageComparer {
 public:
     void compareImages(const QImage& resultImage, const QImage& expectedImage);
+
     double getSSIMValue();
+    double getWorstTileValue();
 
     SSIMResults getSSIMResults();
 

tools/nitpick/src/MismatchWindow.cpp

@@ -61,7 +61,7 @@ QPixmap MismatchWindow::computeDiffPixmap(const QImage& expectedImage, const QIm
 }
 
 void MismatchWindow::setTestResult(const TestResult& testResult) {
-    errorLabel->setText("Similarity: " + QString::number(testResult._error));
+    errorLabel->setText("Similarity: " + QString::number(testResult._errorGlobal) + "  (worst tile: " + QString::number(testResult._errorLocal) + ")");
 
     imagePath->setText("Path to test: " + testResult._pathname);
 

tools/nitpick/src/Nitpick.cpp

@@ -38,7 +38,7 @@ Nitpick::Nitpick(QWidget* parent) : QMainWindow(parent) {
  
     _ui.plainTextEdit->setReadOnly(true);
 
-    setWindowTitle("Nitpick - v3.1.2");
+    setWindowTitle("Nitpick - v3.1.3");
 
     clientProfiles << "VR-High" << "Desktop-High" << "Desktop-Low" << "Mobile-Touch" << "VR-Standalone";
     _ui.clientProfileComboBox->insertItems(0, clientProfiles);

tools/nitpick/src/TestCreator.cpp

@@ -83,6 +83,7 @@ int TestCreator::compareImageLists() {
         QImage expectedImage(_expectedImagesFullFilenames[i]);
 
         double similarityIndex;  // in [-1.0 .. 1.0], where 1.0 means images are identical
+        double worstTileValue;   // in [-1.0 .. 1.0], where 1.0 means images are identical
 
         bool isInteractiveMode = (!_isRunningFromCommandLine && _checkBoxInteractiveMode->isChecked() && !_isRunningInAutomaticTestRun);
                                  
@@ -90,13 +91,16 @@ int TestCreator::compareImageLists() {
         if (isInteractiveMode && (resultImage.width() != expectedImage.width() || resultImage.height() != expectedImage.height())) {
             QMessageBox::critical(0, "Internal error: " + QString(__FILE__) + ":" + QString::number(__LINE__), "Images are not the same size");
             similarityIndex = -100.0;
+            worstTileValue = 0.0;
         } else {
             _imageComparer.compareImages(resultImage, expectedImage);
             similarityIndex = _imageComparer.getSSIMValue();
+            worstTileValue = _imageComparer.getWorstTileValue();
         }
 
         TestResult testResult = TestResult{
-            (float)similarityIndex,
+            similarityIndex,
+            worstTileValue,
             _expectedImagesFullFilenames[i].left(_expectedImagesFullFilenames[i].lastIndexOf("/") + 1),  // path to the test (including trailing /)
             QFileInfo(_expectedImagesFullFilenames[i].toStdString().c_str()).fileName(),                 // filename of expected image
             QFileInfo(_resultImagesFullFilenames[i].toStdString().c_str()).fileName(),                   // filename of result image
@@ -105,10 +109,9 @@ int TestCreator::compareImageLists() {
 
         _mismatchWindow.setTestResult(testResult);
         
-        if (similarityIndex < THRESHOLD) {
+        if (similarityIndex < THRESHOLD_GLOBAL || worstTileValue < THRESHOLD_LOCAL) {
-            ++numberOfFailures;
-
             if (!isInteractiveMode) {
+                ++numberOfFailures;
                 appendTestResultsToFile(testResult, _mismatchWindow.getComparisonImage(), _mismatchWindow.getSSIMResultsImage(testResult._ssimResults), true);
             } else {
                 _mismatchWindow.exec();
@@ -117,6 +120,7 @@ int TestCreator::compareImageLists() {
                     case USER_RESPONSE_PASS:
                         break;
                     case USE_RESPONSE_FAIL:
+                        ++numberOfFailures;
                         appendTestResultsToFile(testResult, _mismatchWindow.getComparisonImage(), _mismatchWindow.getSSIMResultsImage(testResult._ssimResults), true);
                         break;
                     case USER_RESPONSE_ABORT:
@@ -198,7 +202,8 @@ void TestCreator::appendTestResultsToFile(const TestResult& testResult, const QP
     stream << "TestCreator in folder " << testResult._pathname.left(testResult._pathname.length() - 1) << endl; // remove trailing '/'
     stream << "Expected image was    " << testResult._expectedImageFilename << endl;
     stream << "Actual image was      " << testResult._actualImageFilename << endl;
-    stream << "Similarity index was  " << testResult._error << endl;
+    stream << "Similarity index was  " << testResult._errorGlobal << endl;
+    stream << "Worst tile was  " << testResult._errorLocal << endl;
 
     descriptionFile.close();
 
@@ -819,6 +824,10 @@ void TestCreator::createRecursiveScript(const QString& directory, bool interacti
 
     // If 'directories' is empty, this means that this recursive script has no tests to call, so it is redundant
     if (directories.length() == 0) {
+        QString testRecursivePathname = directory + "/" + TEST_RECURSIVE_FILENAME;
+        if (QFile::exists(testRecursivePathname)) {
+            QFile::remove(testRecursivePathname);
+        }
         return;
     }
 
@@ -851,10 +860,7 @@ void TestCreator::createRecursiveScript(const QString& directory, bool interacti
     textStream << "   nitpick = createNitpick(Script.resolvePath(\".\"));" << endl;
     textStream << "   testsRootPath = nitpick.getTestsRootPath();" << endl << endl;
     textStream << "   nitpick.enableRecursive();" << endl;
-    textStream << "   nitpick.enableAuto();" << endl << endl;
+    textStream << "   nitpick.enableAuto();" << endl;
-    textStream << "   if (typeof Test !== 'undefined') {" << endl;
-    textStream << "       Test.wait(10000);" << endl;
-    textStream << "   }" << endl;
     textStream << "} else {" << endl;
     textStream << "   depth++" << endl;
     textStream << "}" << endl << endl;

tools/nitpick/src/TestCreator.h

@@ -121,7 +121,8 @@ private:
     const QString TEST_RESULTS_FOLDER { "TestResults" };
     const QString TEST_RESULTS_FILENAME { "TestResults.txt" };
 
-    const double THRESHOLD{ 0.9999 };
+    const double THRESHOLD_GLOBAL{ 0.9995 };
+    const double THRESHOLD_LOCAL { 0.6 };
 
     QDir _imageDirectory;
 

tools/nitpick/src/TestRunnerMobile.cpp

@@ -43,7 +43,7 @@ TestRunnerMobile::TestRunnerMobile(
     _installAPKPushbutton = installAPKPushbutton;
     _runInterfacePushbutton = runInterfacePushbutton;
 
-    folderLineEdit->setText("/sdcard/DCIM/TEST");
+    folderLineEdit->setText("/sdcard/snapshots");
 
     modelNames["SM_G955U1"] = "Samsung S8+ unlocked";
     modelNames["SM_N960U1"] = "Samsung Note 9 unlocked";
@@ -60,6 +60,7 @@ void TestRunnerMobile::setWorkingFolderAndEnableControls() {
     setWorkingFolder(_workingFolderLabel);
 
     _connectDeviceButton->setEnabled(true);
+    _downloadAPKPushbutton->setEnabled(true);
 }
 
 void TestRunnerMobile::connectDevice() {
@@ -154,8 +155,6 @@ void TestRunnerMobile::downloadComplete() {
     } else {
         _statusLabel->setText("Installer download complete");
     }
-
-    _installAPKPushbutton->setEnabled(true);
 }
 
 void TestRunnerMobile::installAPK() {
@@ -164,22 +163,16 @@ void TestRunnerMobile::installAPK() {
         _adbInterface = new AdbInterface();
     }
 
-    if (_installerFilename.isNull()) {
+    QString installerPathname = QFileDialog::getOpenFileName(nullptr, "Please select the APK", _workingFolder,
-        QString installerPathname = QFileDialog::getOpenFileName(nullptr, "Please select the APK", _workingFolder,
+        "Available APKs (*.apk)"
-            "Available APKs (*.apk)"
+    );
-        );
 
-        if (installerPathname.isNull()) {
+    if (installerPathname.isNull()) {
-            return;
+        return;
-        }
-
-        // Remove the path
-        QStringList parts = installerPathname.split('/');
-        _installerFilename = parts[parts.length() - 1];
     }
 
     _statusLabel->setText("Installing");
-    QString command = _adbInterface->getAdbCommand() + " install -r -d " + _workingFolder + "/" + _installerFilename + " >" + _workingFolder  + "/installOutput.txt";
+    QString command = _adbInterface->getAdbCommand() + " install -r -d " + installerPathname + " >" + _workingFolder  + "/installOutput.txt";
     appendLog(command);
     system(command.toStdString().c_str());
     _statusLabel->setText("Installation complete");

tools/nitpick/src/common.h

@@ -18,7 +18,9 @@ public:
     int width;
     int height;
     std::vector<double> results;
+ 
     double ssim;
+    double worstTileValue;
 
     // Used for scaling
     double min;
@@ -27,15 +29,17 @@ public:
 
 class TestResult {
 public:
-    TestResult(float error, const QString& pathname, const QString& expectedImageFilename, const QString& actualImageFilename, const SSIMResults& ssimResults) :
+    TestResult(double errorGlobal, double errorLocal, const QString& pathname, const QString& expectedImageFilename, const QString& actualImageFilename, const SSIMResults& ssimResults) :
-        _error(error),
+        _errorGlobal(errorGlobal),
+        _errorLocal(errorLocal),
         _pathname(pathname),
         _expectedImageFilename(expectedImageFilename),
         _actualImageFilename(actualImageFilename),
         _ssimResults(ssimResults)
     {}
 
-    double _error;
+    double _errorGlobal;
+    double _errorLocal;
 
     QString _pathname;
     QString _expectedImageFilename;

tools/nitpick/ui/MismatchWindow.ui

@@ -45,7 +45,7 @@
   <widget class="QLabel" name="diffImage">
    <property name="geometry">
     <rect>
-     <x>540</x>
+     <x>900</x>
      <y>480</y>
      <width>800</width>
      <height>450</height>
@@ -78,7 +78,7 @@
     <rect>
      <x>60</x>
      <y>630</y>
-     <width>480</width>
+     <width>540</width>
      <height>28</height>
     </rect>
    </property>
@@ -145,7 +145,7 @@
     </rect>
    </property>
    <property name="text">
-    <string>Abort current test</string>
+    <string>Abort evaluation</string>
    </property>
   </widget>
   <widget class="QLabel" name="errorLabel">

tools/nitpick/ui/Nitpick.ui

@@ -46,7 +46,7 @@
      </rect>
     </property>
     <property name="currentIndex">
-     <number>5</number>
+     <number>0</number>
     </property>
     <widget class="QWidget" name="tab_1">
      <attribute name="title">