Merge pull request #13247 from NissimHadar/DailyTests

Daily tests

interface/src/scripting/TestScriptingInterface.cpp

@@ -14,6 +14,7 @@
 #include <shared/FileUtils.h>
 #include <shared/QtHelpers.h>
 #include <DependencyManager.h>
+#include <MainWindow.h>
 #include <OffscreenUi.h>
 #include <StatTracker.h>
 #include <Trace.h>
@@ -186,3 +187,7 @@ void TestScriptingInterface::saveObject(QVariant variant, const QString& filenam
     file.write(jsonData);
     file.close();
 }
+
+void TestScriptingInterface::showMaximized() {
+    qApp->getWindow()->showMaximized();
+}

interface/src/scripting/TestScriptingInterface.h

@@ -27,70 +27,128 @@ public slots:
 
     /**jsdoc
      * Exits the application
+     * @function Test.quit
      */
     void quit();
 
     /**jsdoc
     * Waits for all texture transfers to be complete
+    * @function Test.waitForTextureIdle
     */
     void waitForTextureIdle();
 
     /**jsdoc
     * Waits for all pending downloads to be complete
+    * @function Test.waitForDownloadIdle
     */
     void waitForDownloadIdle();
 
     /**jsdoc
     * Waits for all file parsing operations to be complete
+    * @function Test.waitForProcessingIdle
     */
     void waitForProcessingIdle();
 
     /**jsdoc
     * Waits for all pending downloads, parsing and texture transfers to be complete
+    * @function Test.waitIdle
     */
     void waitIdle();
 
+    /**jsdoc
+    * Waits for establishment of connection to server
+    * @function Test.waitForConnection
+    * @param {int} maxWaitMs [default=10000] - Number of milliseconds to wait
+    */
     bool waitForConnection(qint64 maxWaitMs = 10000);
 
+    /**jsdoc
+    * Waits a specific number of milliseconds
+    * @function Test.wait
+    * @param {int} milliseconds - Number of milliseconds to wait
+    */
     void wait(int milliseconds);
 
+    /**jsdoc
+    * Waits for all pending downloads, parsing and texture transfers to be complete
+    * @function Test.loadTestScene
+    * @param {string} sceneFile - URL of scene to load
+    */
     bool loadTestScene(QString sceneFile);
 
+    /**jsdoc
+    * Clears all caches
+    * @function Test.clear
+    */
     void clear();
 
     /**jsdoc
     * Start recording Chrome compatible tracing events
     * logRules can be used to specify a set of logging category rules to limit what gets captured
+    * @function Test.startTracing
+    * @param {string} logrules [defaultValue=""] - See implementation for explanation
     */
     bool startTracing(QString logrules = "");
 
     /**jsdoc
     * Stop recording Chrome compatible tracing events and serialize recorded events to a file
     * Using a filename with a .gz extension will automatically compress the output file
+    * @function Test.stopTracing
+    * @param {string} filename - Name of file to save to
+    * @returns {bool} True if successful.   
     */
     bool stopTracing(QString filename);
 
+    /**jsdoc
+    * Starts a specific trace event
+    * @function Test.startTraceEvent
+    * @param {string} name - Name of event
+    */
     void startTraceEvent(QString name);
 
+    /**jsdoc
+    * Stop a specific name event
+    * Using a filename with a .gz extension will automatically compress the output file
+    * @function Test.endTraceEvent
+    * @param {string} filename - Name of event
+    */
     void endTraceEvent(QString name);
 
     /**jsdoc
      * Write detailed timing stats of next physics stepSimulation() to filename
+     * @function Test.savePhysicsSimulationStats
+     * @param {string} filename - Name of file to save to
      */
     void savePhysicsSimulationStats(QString filename);
 
+    /**jsdoc
+    * Profiles a specific function
+    * @function Test.savePhysicsSimulationStats
+    * @param {string} name - Name used to reference the function
+    * @param {function} function - Function to profile
+    */
     Q_INVOKABLE void profileRange(const QString& name, QScriptValue function);
 
     /**jsdoc
     * Clear all caches (menu command Reload Content)
+    * @function Test.clearCaches
     */
     void clearCaches();
 
     /**jsdoc
     * Save a JSON object to a file in the test results location
+    * @function Test.saveObject
+    * @param {string} name - Name of the object
+    * @param {string} filename - Name of file to save to
     */
     void saveObject(QVariant v, const QString& filename);
 
+    /**jsdoc
+    * Maximizes the window
+    * @function Test.showMaximized
+    */
+    void showMaximized();
+
 private:
     bool waitForCondition(qint64 maxWaitMs, std::function<bool()> condition);
     QString _testResultsLocation;

tools/auto-tester/src/ImageComparer.cpp

@@ -15,6 +15,12 @@
 // Computes SSIM - see https://en.wikipedia.org/wiki/Structural_similarity
 // The value is computed for the luminance component and the average value is returned
 double ImageComparer::compareImages(QImage resultImage, QImage expectedImage) const {
+    // Make sure the image is 8 bits per colour
+    QImage::Format format = expectedImage.format();
+    if (format != QImage::Format::Format_ARGB32) {
+        throw -1;
+    }
+
     const int L = 255; // (2^number of bits per pixel) - 1
     const double K1 { 0.01 };
     const double K2 { 0.03 };
@@ -54,8 +60,8 @@ double ImageComparer::compareImages(QImage resultImage, QImage expectedImage) co
 
             // Collect pixels into linear arrays
             int i{ 0 };
-            for (int xx = 0; xx < WIN_SIZE; ++xx) {
-                for (int yy = 0; yy < WIN_SIZE; ++yy) {
+            for (int xx = 0; xx < WIN_SIZE && x + xx < expectedImage.width(); ++xx) {
+                for (int yy = 0; yy < WIN_SIZE && y + yy < expectedImage.height(); ++yy) {
                     // Get pixels
                     QRgb pixelP = expectedImage.pixel(QPoint(x + xx, y + yy));
                     QRgb pixelQ = resultImage.pixel(QPoint(x + xx, y + yy));

tools/auto-tester/src/Test.cpp

@@ -63,7 +63,7 @@ bool Test::compareImageLists(bool isInteractiveMode, QProgressBar* progressBar)
 
     // Loop over both lists and compare each pair of images
     // Quit loop if user has aborted due to a failed test.
-    const double THRESHOLD { 0.99 };
+    const double THRESHOLD { 0.9995 };
     bool success{ true };
     bool keepOn{ true };
     for (int i = 0; keepOn && i < expectedImagesFullFilenames.length(); ++i) {
@@ -131,7 +131,9 @@ void Test::appendTestResultsToFile(const QString& testResultsFolderPath, TestFai
         exit(-1);
     }
 
-    QString failureFolderPath { testResultsFolderPath + "/" + "Failure_" + QString::number(index) + "--" + testFailure._actualImageFilename.left(testFailure._actualImageFilename.length() - 4) };
+    QString err = QString::number(testFailure._error).left(6);
+
+    QString failureFolderPath { testResultsFolderPath + "/" + err +  "-Failure_" + QString::number(index) + "--" + testFailure._actualImageFilename.left(testFailure._actualImageFilename.length() - 4) };
     if (!QDir().mkdir(failureFolderPath)) {
         QMessageBox::critical(0, "Internal error: " + QString(__FILE__) + ":" + QString::number(__LINE__), "Failed to create folder " + failureFolderPath);
         exit(-1);
@@ -395,9 +397,9 @@ void Test::createRecursiveScript(const QString& topLevelDirectory, bool interact
     const QString DATE_TIME_FORMAT("MMM d yyyy, h:mm");
     textStream << "// This is an automatically generated file, created by auto-tester on " << QDateTime::currentDateTime().toString(DATE_TIME_FORMAT) << endl << endl;
 
-    textStream << "user = \"" + GIT_HUB_USER + "/\"" << endl;
-    textStream << "repository = \"" + GIT_HUB_REPOSITORY + "/\"" << endl;
-    textStream << "branch = \"" + GIT_HUB_BRANCH + "/\"" << endl << endl;
+    textStream << "user = \"" + GIT_HUB_USER + "/\";" << endl;
+    textStream << "repository = \"" + GIT_HUB_REPOSITORY + "/\";" << endl;
+    textStream << "branch = \"" + GIT_HUB_BRANCH + "/\";" << endl << endl;
 
     textStream << "var autoTester = Script.require(\"https://github.com/" + GIT_HUB_USER + "/hifi_tests/blob/" 
         + GIT_HUB_BRANCH + "/tests/utils/autoTester.js?raw=true\");" << endl << endl;

tools/auto-tester/src/ui/AutoTester.cpp

@@ -10,6 +10,11 @@
 //
 #include "AutoTester.h"
 
+#ifdef Q_OS_WIN
+#include <windows.h>
+#include <shellapi.h>
+#endif
+
 AutoTester::AutoTester(QWidget *parent) : QMainWindow(parent) {
     ui.setupUi(this);
     ui.checkBoxInteractiveMode->setChecked(true);
@@ -20,6 +25,11 @@ AutoTester::AutoTester(QWidget *parent) : QMainWindow(parent) {
     connect(ui.actionClose, &QAction::triggered, this, &AutoTester::on_closeButton_clicked);
     connect(ui.actionAbout, &QAction::triggered, this, &AutoTester::about);
 
+#ifndef Q_OS_WIN
+    ui.hideTaskbarButton->setVisible(false);
+    ui.showTaskbarButton->setVisible(false);
+#endif
+
     test = new Test();
 }
 
@@ -56,6 +66,30 @@ void AutoTester::on_createTestsOutlineButton_clicked() {
     test->createTestsOutline();
 }
 
+// To toggle between show and hide
+//   if (uState & ABS_AUTOHIDE) on_showTaskbarButton_clicked();
+//   else on_hideTaskbarButton_clicked();
+//
+void AutoTester::on_hideTaskbarButton_clicked() {
+#ifdef Q_OS_WIN
+    APPBARDATA abd = { sizeof abd };
+    UINT uState = (UINT)SHAppBarMessage(ABM_GETSTATE, &abd);
+    LPARAM param = uState & ABS_ALWAYSONTOP;
+    abd.lParam = ABS_AUTOHIDE | param;
+    SHAppBarMessage(ABM_SETSTATE, &abd);
+#endif
+}
+
+void AutoTester::on_showTaskbarButton_clicked() {
+#ifdef Q_OS_WIN
+    APPBARDATA abd = { sizeof abd };
+    UINT uState = (UINT)SHAppBarMessage(ABM_GETSTATE, &abd);
+    LPARAM param = uState & ABS_ALWAYSONTOP;
+    abd.lParam = param;
+    SHAppBarMessage(ABM_SETSTATE, &abd);
+#endif
+}
+
 void AutoTester::on_closeButton_clicked() {
     exit(0);
 }
@@ -95,7 +129,7 @@ void AutoTester::saveImage(int index) {
     pixmap.loadFromData(downloaders[index]->downloadedData());
 
     QImage image = pixmap.toImage();
-    image = image.convertToFormat(QImage::Format_RGB32);
+    image = image.convertToFormat(QImage::Format_ARGB32);
 
     QString fullPathname = _directoryName + "/" + _filenames[index];
     if (!image.save(fullPathname, 0, 100)) {

tools/auto-tester/src/ui/AutoTester.h

@@ -36,6 +36,10 @@ private slots:
     void on_createMDFileButton_clicked();
     void on_createAllMDFilesButton_clicked();
     void on_createTestsOutlineButton_clicked();
+
+    void on_hideTaskbarButton_clicked();
+    void on_showTaskbarButton_clicked();
+
     void on_closeButton_clicked();
 
     void saveImage(int index);

tools/auto-tester/src/ui/AutoTester.ui

@@ -147,6 +147,32 @@
      <string>Create Tests Outline</string>
     </property>
    </widget>
+   <widget class="QPushButton" name="showTaskbarButton">
+    <property name="geometry">
+     <rect>
+      <x>490</x>
+      <y>280</y>
+      <width>91</width>
+      <height>40</height>
+     </rect>
+    </property>
+    <property name="text">
+     <string>Show Taskbar</string>
+    </property>
+   </widget>
+   <widget class="QPushButton" name="hideTaskbarButton">
+    <property name="geometry">
+     <rect>
+      <x>490</x>
+      <y>230</y>
+      <width>91</width>
+      <height>40</height>
+     </rect>
+    </property>
+    <property name="text">
+     <string>Hide Taskbar</string>
+    </property>
+   </widget>
   </widget>
   <widget class="QMenuBar" name="menuBar">
    <property name="geometry">