Merge pull request #15313 from NissimHadar/22021-specificThresholdsPerGPUVendor-v3.2.0

Case 22021: specific thresholds per GPU vendor v3.2.0

tools/nitpick/src/AWSInterface.cpp

@@ -480,7 +480,7 @@ void AWSInterface::createEntry(const int index, const QString& testResult, QText
         if (differenceFileFound) {
             stream << "\t\t\t\t<td><img src=\"./" << folder << "/" << resultName << "/" << _comparisonImageFilename << "\" width = \"576\" height = \"324\" ></td>\n";
         } else {
-            stream << "\t\t\t\t<td><h2>No Image Found</h2>\n";
+            stream << "\t\t\t\t<td><h2>Image size mismatch</h2>\n";
         }
 
         stream << "\t\t\t</tr>\n";

tools/nitpick/src/Nitpick.cpp

@@ -9,6 +9,7 @@
 //  See the accompanying file LICENSE or http://www.apache.org/licenses/LICENSE-2.0.html
 //
 #include "Nitpick.h"
+#include "Platform.h"
 
 #ifdef Q_OS_WIN
 #include <windows.h>
@@ -40,9 +41,15 @@ Nitpick::Nitpick(QWidget* parent) : QMainWindow(parent) {
 
     setWindowTitle("Nitpick - " + nitpickVersion);
 
-    clientProfiles << "VR-High" << "Desktop-High" << "Desktop-Low" << "Mobile-Touch" << "VR-Standalone";
-    _ui.clientProfileComboBox->insertItems(0, clientProfiles);
+    _GPUVendors << "Nvidia" << "AMD";
+    _ui.gpuVendorComboBox->insertItems(0, _GPUVendors);
 
+    QString gpuVendor = Platform::getGraphicsCardType().toUpper();
+    if (gpuVendor.contains("NVIDIA")) {
+        _ui.gpuVendorComboBox->setCurrentIndex(0);
+    } else {
+        _ui.gpuVendorComboBox->setCurrentIndex(1);
+    }
 }
 
 Nitpick::~Nitpick() {
@@ -126,13 +133,14 @@ void Nitpick::setup() {
     );
 }
 
-void Nitpick::startTestsEvaluation(const bool isRunningFromCommandLine,
-                                      const bool isRunningInAutomaticTestRun,
-                                      const QString& snapshotDirectory,
-                                      const QString& branch,
-                                      const QString& user
+void Nitpick::startTestsEvaluation(
+    const bool isRunningFromCommandLine,
+    const bool isRunningInAutomaticTestRun,
+    const QString& snapshotDirectory,
+    const QString& branch,
+    const QString& user
 ) {
-    _testCreator->startTestsEvaluation(isRunningFromCommandLine, isRunningInAutomaticTestRun, snapshotDirectory, branch, user);
+    _testCreator->startTestsEvaluation(_ui.gpuVendorComboBox, isRunningFromCommandLine, isRunningInAutomaticTestRun, snapshotDirectory, branch, user);
 }
 
 void Nitpick::on_tabWidget_currentChanged(int index) {
@@ -144,9 +152,11 @@ void Nitpick::on_tabWidget_currentChanged(int index) {
 #endif
         _ui.userLineEdit->setDisabled(false);
         _ui.branchLineEdit->setDisabled(false);
+        _ui.gpuVendorComboBox->setDisabled(false);
     } else {
         _ui.userLineEdit->setDisabled(true);
         _ui.branchLineEdit->setDisabled(true);
+        _ui.gpuVendorComboBox->setDisabled(true);
     }
 }
 
@@ -159,7 +169,7 @@ void Nitpick::on_createAllRecursiveScriptsPushbutton_clicked() {
 }
 
 void Nitpick::on_createTestsPushbutton_clicked() {
-    _testCreator->createTests(_ui.clientProfileComboBox->currentText());
+    _testCreator->createTests(_ui.gpuVendorComboBox->currentText());
 }
 
 void Nitpick::on_createMDFilePushbutton_clicked() {
@@ -250,7 +260,7 @@ void Nitpick::on_showTaskbarPushbutton_clicked() {
 }
 
 void Nitpick::on_evaluateTestsPushbutton_clicked() {
-    _testCreator->startTestsEvaluation(false, false);
+    _testCreator->startTestsEvaluation(_ui.gpuVendorComboBox, false, false);
 }
 
 void Nitpick::on_closePushbutton_clicked() {

tools/nitpick/src/Nitpick.h

@@ -28,11 +28,13 @@ public:
 
     void setup();
 
-    void startTestsEvaluation(const bool isRunningFromCommandLine,
-                              const bool isRunningInAutomaticTestRun,
-                              const QString& snapshotDirectory,
-                              const QString& branch,
-                              const QString& user);
+    void startTestsEvaluation(
+        const bool isRunningFromCommandLine,
+        const bool isRunningInAutomaticTestRun,
+        const QString& snapshotDirectory,
+        const QString& branch,
+        const QString& user
+    );
 
     void automaticTestRunEvaluationComplete(QString zippedFolderName, int numberOfFailures);
 
@@ -111,7 +113,7 @@ private:
 
     bool _isRunningFromCommandline{ false };
 
-    QStringList clientProfiles;
+    QStringList _GPUVendors;
 };
 
 #endif  // hifi_Nitpick_h

tools/nitpick/src/Platform.cpp

@@ -0,0 +1,121 @@
+//
+//  Platform.h
+//
+//  Created by Nissim Hadar on 2 Apr 2019.
+//  Copyright 2013 High Fidelity, Inc.
+//
+//  Distributed under the Apache License, Version 2.0.
+//  See the accompanying file LICENSE or http://www.apache.org/licenses/LICENSE-2.0.html
+//
+#include "Platform.h"
+
+#include <vector>
+
+#ifdef Q_OS_WIN
+#include <dxgi1_3.h>
+#pragma comment(lib, "dxgi.lib")
+#elif defined Q_OS_MAC
+#include <OpenGL/OpenGL.h>
+#include <sstream>
+#include <QStringList>
+#endif
+
+QString Platform::getGraphicsCardType() {
+#ifdef Q_OS_WIN
+    IDXGIFactory1* pFactory = nullptr;
+    HRESULT hr = CreateDXGIFactory1(__uuidof(IDXGIFactory1), (void**)(&pFactory));
+    if (hr != S_OK || pFactory == nullptr) {
+        return "Unable to create DXGI";
+    }
+    std::vector<int> validAdapterList;
+
+    using AdapterEntry = std::pair<std::pair<DXGI_ADAPTER_DESC1, LARGE_INTEGER>, std::vector<DXGI_OUTPUT_DESC>>;
+    std::vector<AdapterEntry> adapterToOutputs;
+    // Enumerate adapters and outputs
+    {
+        UINT adapterNum = 0;
+        IDXGIAdapter1* pAdapter = nullptr;
+        while (pFactory->EnumAdapters1(adapterNum, &pAdapter) != DXGI_ERROR_NOT_FOUND) {
+
+            // Found an adapter, get descriptor
+            DXGI_ADAPTER_DESC1 adapterDesc;
+            pAdapter->GetDesc1(&adapterDesc);
+
+            LARGE_INTEGER version;
+            hr = pAdapter->CheckInterfaceSupport(__uuidof(IDXGIDevice), &version);
+
+            std::wstring wDescription (adapterDesc.Description);
+            std::string description(wDescription.begin(), wDescription.end());
+
+            AdapterEntry adapterEntry;
+            adapterEntry.first.first = adapterDesc;
+            adapterEntry.first.second = version;
+
+
+
+            UINT outputNum = 0;
+            IDXGIOutput * pOutput;
+            bool hasOutputConnectedToDesktop = false;
+            while (pAdapter->EnumOutputs(outputNum, &pOutput) != DXGI_ERROR_NOT_FOUND) {
+
+                // FOund an output attached to the adapter, get descriptor
+                DXGI_OUTPUT_DESC outputDesc;
+                pOutput->GetDesc(&outputDesc);
+
+                adapterEntry.second.push_back(outputDesc);
+
+                std::wstring wDeviceName(outputDesc.DeviceName);
+                std::string deviceName(wDeviceName.begin(), wDeviceName.end());
+
+                hasOutputConnectedToDesktop |= (bool)outputDesc.AttachedToDesktop;
+
+                pOutput->Release();
+                outputNum++;
+            }
+
+            adapterToOutputs.push_back(adapterEntry);
+
+            // add this adapter to the valid list if has output
+            if (hasOutputConnectedToDesktop && !adapterEntry.second.empty()) {
+                validAdapterList.push_back(adapterNum);
+            }
+
+            pAdapter->Release();
+            adapterNum++;
+        }
+    }
+    pFactory->Release();
+
+    if (!validAdapterList.empty()) {
+        auto& adapterEntry = adapterToOutputs[validAdapterList.front()];
+
+        std::wstring wDescription(adapterEntry.first.first.Description);
+        std::string description(wDescription.begin(), wDescription.end());
+        return QString(description.c_str());
+    }
+
+#elif defined Q_OS_MAC
+    FILE* stream = popen("system_profiler SPDisplaysDataType | grep Chipset", "r");
+
+    std::ostringstream hostStream;
+    while (!feof(stream) && !ferror(stream)) {
+        char buf[128];
+        int bytesRead = fread(buf, 1, 128, stream);
+        hostStream.write(buf, bytesRead);
+    }
+
+    QString result = QString::fromStdString(hostStream.str());
+    QStringList parts = result.split('\n');
+    for (int i = 0; i < parts.size(); ++i) {
+        if (parts[i].toLower().contains("radeon") || parts[i].toLower().contains("nvidia")) {
+            return parts[i];
+        }
+    }
+
+    // unkown graphics card
+    return "UNKNOWN";
+#else
+    return QString("NO IMPLEMENTED");
+#endif
+    return "";
+}

tools/nitpick/src/Platform.h

@@ -0,0 +1,20 @@
+//
+//  Platform.h
+//
+//  Created by Nissim Hadar on 2 Apr 2019.
+//  Copyright 2013 High Fidelity, Inc.
+//
+//  Distributed under the Apache License, Version 2.0.
+//  See the accompanying file LICENSE or http://www.apache.org/licenses/LICENSE-2.0.html
+//
+#ifndef hifi_Platform_h
+#define hifi_Platform_h
+
+#include <QString>
+
+class Platform {
+public:
+    static QString getGraphicsCardType();
+};
+
+#endif

tools/nitpick/src/TestCreator.cpp

@@ -67,7 +67,7 @@ QString TestCreator::zipAndDeleteTestResultsFolder() {
     return zippedResultsFileName;
 }
 
-int TestCreator::compareImageLists() {
+int TestCreator::compareImageLists(const QString& gpuVendor) {
     _progressBar->setMinimum(0);
     _progressBar->setMaximum(_expectedImagesFullFilenames.length() - 1);
     _progressBar->setValue(0);
@@ -108,8 +108,16 @@ int TestCreator::compareImageLists() {
         };
 
         _mismatchWindow.setTestResult(testResult);
-        
-        if (similarityIndex < THRESHOLD_GLOBAL || worstTileValue < THRESHOLD_LOCAL) {
+
+        // Lower threshold for non-Nvidia GPUs
+        double thresholdGlobal{ 0.9995 };
+        double thresholdLocal{ 0.6 };
+        if (gpuVendor != "Nvidia") {
+            thresholdGlobal =  0.97;
+            thresholdLocal = 0.2;
+        }
+
+        if (similarityIndex < thresholdGlobal || worstTileValue < thresholdLocal) {
             if (!isInteractiveMode) {
                 ++numberOfFailures;
                 appendTestResultsToFile(testResult, _mismatchWindow.getComparisonImage(), _mismatchWindow.getSSIMResultsImage(testResult._ssimResults), true);
@@ -258,11 +266,13 @@ void::TestCreator::appendTestResultsToFile(QString testResultFilename, bool hasF
     }
 }
 
-void TestCreator::startTestsEvaluation(const bool isRunningFromCommandLine,
-                                const bool isRunningInAutomaticTestRun, 
-                                const QString& snapshotDirectory,
-                                const QString& branchFromCommandLine,
-                                const QString& userFromCommandLine
+void TestCreator::startTestsEvaluation(
+    QComboBox *gpuVendor, 
+    const bool isRunningFromCommandLine,
+    const bool isRunningInAutomaticTestRun, 
+    const QString& snapshotDirectory,
+    const QString& branchFromCommandLine,
+    const QString& userFromCommandLine
 ) {
     _isRunningFromCommandLine = isRunningFromCommandLine;
     _isRunningInAutomaticTestRun = isRunningInAutomaticTestRun;
@@ -332,12 +342,12 @@ void TestCreator::startTestsEvaluation(const bool isRunningFromCommandLine,
     }
 
     _downloader->downloadFiles(expectedImagesURLs, _snapshotDirectory, _expectedImagesFilenames, (void *)this);
-    finishTestsEvaluation();
+    finishTestsEvaluation(gpuVendor->currentText());
 }
 
-void TestCreator::finishTestsEvaluation() {
+void TestCreator::finishTestsEvaluation(const QString& gpuVendor) {
     // First - compare the pairs of images
-    int numberOfFailures = compareImageLists();
+    int numberOfFailures = compareImageLists(gpuVendor);
  
     // Next - check text results
     numberOfFailures += checkTextResults();
@@ -430,8 +440,9 @@ void TestCreator::createTests(const QString& clientProfile) {
         parent += "/";
     }
 
-    _testsRootDirectory = QFileDialog::getExistingDirectory(nullptr, "Please select test root folder", parent,
-                                                              QFileDialog::ShowDirsOnly);
+    if (!createAllFilesSetup()) {
+        return;
+    }
 
     // If user canceled then restore previous selection and return
     if (_testsRootDirectory == "") {
@@ -881,23 +892,12 @@ void TestCreator::createRecursiveScript(const QString& directory, bool interacti
 }
 
 void TestCreator::createTestsOutline() {
-    QString previousSelection = _testDirectory;
-    QString parent = previousSelection.left(previousSelection.lastIndexOf('/'));
-    if (!parent.isNull() && parent.right(1) != "/") {
-        parent += "/";
-    }
-
-    _testDirectory =
-        QFileDialog::getExistingDirectory(nullptr, "Please select the tests root folder", parent, QFileDialog::ShowDirsOnly);
-
-    // If user canceled then restore previous selection and return
-    if (_testDirectory == "") {
-        _testDirectory = previousSelection;
+    if (!createAllFilesSetup()) {
         return;
     }
 
     const QString testsOutlineFilename { "testsOutline.md" };
-    QString mdFilename(_testDirectory + "/" + testsOutlineFilename);
+    QString mdFilename(_testsRootDirectory + "/" + testsOutlineFilename);
     QFile mdFile(mdFilename);
     if (!mdFile.open(QIODevice::WriteOnly)) {
         QMessageBox::critical(0, "Internal error: " + QString(__FILE__) + ":" + QString::number(__LINE__), "Failed to create file " + mdFilename);
@@ -911,10 +911,10 @@ void TestCreator::createTestsOutline() {
     stream << "Directories with an appended (*) have an automatic test\n\n";
 
     // We need to know our current depth, as this isn't given by QDirIterator
-    int rootDepth { _testDirectory.count('/') };
+    int rootDepth { _testsRootDirectory.count('/') };
 
     // Each test is shown as the folder name linking to the matching GitHub URL, and the path to the associated test.md file
-    QDirIterator it(_testDirectory, QDirIterator::Subdirectories);
+    QDirIterator it(_testsRootDirectory, QDirIterator::Subdirectories);
     while (it.hasNext()) {
         QString directory = it.next();
 

tools/nitpick/src/TestCreator.h

@@ -45,13 +45,16 @@ class TestCreator {
 public: 
     TestCreator(QProgressBar* progressBar, QCheckBox* checkBoxInteractiveMode);
 
-    void startTestsEvaluation(const bool isRunningFromCommandLine,
-                              const bool isRunningInAutomaticTestRun, 
-                              const QString& snapshotDirectory = QString(),
-                              const QString& branchFromCommandLine = QString(),
-                              const QString& userFromCommandLine = QString());
+    void startTestsEvaluation(
+        QComboBox *gpuVendor,
+        const bool isRunningFromCommandLine,
+        const bool isRunningInAutomaticTestRun, 
+        const QString& snapshotDirectory = QString(),
+        const QString& branchFromCommandLine = QString(),
+        const QString& userFromCommandLine = QString()
+    );
 
-    void finishTestsEvaluation();
+    void finishTestsEvaluation(const QString& gpuVendor);
 
     void createTests(const QString& clientProfile);
 
@@ -79,7 +82,7 @@ public:
     void createRecursiveScript();
     void createRecursiveScript(const QString& directory, bool interactiveMode);
 
-    int compareImageLists();
+    int compareImageLists(const QString& gpuVendor);
     int checkTextResults();
 
     QStringList createListOfAll_imagesInDirectory(const QString& imageFormat, const QString& pathToImageDirectory);
@@ -124,9 +127,6 @@ private:
     const QString TEST_RESULTS_FOLDER { "TestResults" };
     const QString TEST_RESULTS_FILENAME { "TestResults.txt" };
 
-    const double THRESHOLD_GLOBAL{ 0.9995 };
-    const double THRESHOLD_LOCAL { 0.6 };
-
     QDir _imageDirectory;
 
     MismatchWindow _mismatchWindow;

tools/nitpick/src/common.h

@@ -60,5 +60,5 @@ const double R_Y = 0.212655f;
 const double G_Y = 0.715158f;
 const double B_Y = 0.072187f;
 
-const QString nitpickVersion{ "v3.1.5" }; 
+const QString nitpickVersion{ "v3.2.0" }; 
 #endif // hifi_common_h

tools/nitpick/ui/Nitpick.ui

@@ -56,7 +56,7 @@
       <property name="geometry">
        <rect>
         <x>70</x>
-        <y>40</y>
+        <y>120</y>
         <width>220</width>
         <height>40</height>
        </rect>
@@ -69,7 +69,7 @@
       <property name="geometry">
        <rect>
         <x>70</x>
-        <y>180</y>
+        <y>200</y>
         <width>220</width>
         <height>40</height>
        </rect>
@@ -82,7 +82,7 @@
       <property name="geometry">
        <rect>
         <x>320</x>
-        <y>180</y>
+        <y>200</y>
         <width>220</width>
         <height>40</height>
        </rect>
@@ -94,7 +94,7 @@
      <widget class="QPushButton" name="createTestsOutlinePushbutton">
       <property name="geometry">
        <rect>
-        <x>210</x>
+        <x>320</x>
         <y>120</y>
         <width>220</width>
         <height>40</height>
@@ -108,7 +108,7 @@
       <property name="geometry">
        <rect>
         <x>70</x>
-        <y>300</y>
+        <y>360</y>
         <width>220</width>
         <height>40</height>
        </rect>
@@ -121,7 +121,7 @@
       <property name="geometry">
        <rect>
         <x>320</x>
-        <y>300</y>
+        <y>360</y>
         <width>220</width>
         <height>40</height>
        </rect>
@@ -134,7 +134,7 @@
       <property name="geometry">
        <rect>
         <x>70</x>
-        <y>240</y>
+        <y>280</y>
         <width>220</width>
         <height>40</height>
        </rect>
@@ -147,7 +147,7 @@
       <property name="geometry">
        <rect>
         <x>320</x>
-        <y>240</y>
+        <y>280</y>
         <width>220</width>
         <height>40</height>
        </rect>
@@ -156,16 +156,6 @@
        <string>Create all testAuto scripts</string>
       </property>
      </widget>
-     <widget class="QComboBox" name="clientProfileComboBox">
-      <property name="geometry">
-       <rect>
-        <x>320</x>
-        <y>40</y>
-        <width>120</width>
-        <height>40</height>
-       </rect>
-      </property>
-     </widget>
     </widget>
     <widget class="QWidget" name="tab_4">
      <attribute name="title">
@@ -927,8 +917,8 @@
        <rect>
         <x>330</x>
         <y>190</y>
-        <width>181</width>
-        <height>51</height>
+        <width>180</width>
+        <height>50</height>
        </rect>
       </property>
       <property name="text">
@@ -1186,6 +1176,39 @@
      </rect>
     </property>
    </widget>
+   <widget class="QComboBox" name="gpuVendorComboBox">
+    <property name="geometry">
+     <rect>
+      <x>450</x>
+      <y>60</y>
+      <width>180</width>
+      <height>40</height>
+     </rect>
+    </property>
+    <property name="font">
+     <font>
+      <pointsize>14</pointsize>
+     </font>
+    </property>
+   </widget>
+   <widget class="QLabel" name="label_5">
+    <property name="geometry">
+     <rect>
+      <x>505</x>
+      <y>40</y>
+      <width>81</width>
+      <height>16</height>
+     </rect>
+    </property>
+    <property name="font">
+     <font>
+      <pointsize>10</pointsize>
+     </font>
+    </property>
+    <property name="text">
+     <string>GPU Vendor</string>
+    </property>
+   </widget>
   </widget>
   <widget class="QMenuBar" name="menuBar">
    <property name="geometry">