Merge pull request #13046 from NissimHadar/commandLineParametersForAutotester

Command line parameters for autotester

interface/src/Application.cpp

@@ -743,6 +743,11 @@ extern DisplayPluginList getDisplayPlugins();
 extern InputPluginList getInputPlugins();
 extern void saveInputPluginSettings(const InputPluginList& plugins);
 
+// Parameters used for running tests from teh command line
+const QString TEST_SCRIPT_COMMAND { "--testScript" };
+const QString TEST_QUIT_WHEN_FINISHED_OPTION { "quitWhenFinished" };
+const QString TEST_SNAPSHOT_LOCATION_COMMAND { "--testSnapshotLocation" };
+
 bool setupEssentials(int& argc, char** argv, bool runningMarkerExisted) {
     const char** constArgv = const_cast<const char**>(argv);
 
@@ -777,7 +782,22 @@ bool setupEssentials(int& argc, char** argv, bool runningMarkerExisted) {
 
     static const auto SUPPRESS_SETTINGS_RESET = "--suppress-settings-reset";
     bool suppressPrompt = cmdOptionExists(argc, const_cast<const char**>(argv), SUPPRESS_SETTINGS_RESET);
-    bool previousSessionCrashed = CrashHandler::checkForResetSettings(runningMarkerExisted, suppressPrompt);
+
+    // Ignore any previous crashes if running from command line with a test script.  
+    bool inTestMode { false };
+    for (int i = 0; i < argc; ++i) {
+        QString parameter(argv[i]);
+        if (parameter == TEST_SCRIPT_COMMAND) {
+            inTestMode = true;
+            break;
+        }
+    }
+
+    bool previousSessionCrashed { false };
+    if (!inTestMode) {
+        previousSessionCrashed = CrashHandler::checkForResetSettings(runningMarkerExisted, suppressPrompt);
+    }
+
     // get dir to use for cache
     static const auto CACHE_SWITCH = "--cache";
     QString cacheDir = getCmdOption(argc, const_cast<const char**>(argv), CACHE_SWITCH);
@@ -996,13 +1016,30 @@ Application::Application(int& argc, char** argv, QElapsedTimer& startupTimer, bo
     setProperty(hifi::properties::STEAM, (steamClient && steamClient->isRunning()));
     setProperty(hifi::properties::CRASHED, _previousSessionCrashed);
     {
-        const QString TEST_SCRIPT = "--testScript";
         const QStringList args = arguments();
+
         for (int i = 0; i < args.size() - 1; ++i) {
-            if (args.at(i) == TEST_SCRIPT) {
+            if (args.at(i) == TEST_SCRIPT_COMMAND && (i + 1) < args.size()) {
                 QString testScriptPath = args.at(i + 1);
-                if (QFileInfo(testScriptPath).exists()) {
+
+                // If the URL scheme is http(s) or ftp, then use as is, else - treat it as a local file
+                // This is done so as not break previous command line scripts
+                if (testScriptPath.left(URL_SCHEME_HTTP.length()) == URL_SCHEME_HTTP || testScriptPath.left(URL_SCHEME_FTP.length()) == URL_SCHEME_FTP) {
+                    setProperty(hifi::properties::TEST, QUrl::fromUserInput(testScriptPath));
+                } else if (QFileInfo(testScriptPath).exists()) {
                     setProperty(hifi::properties::TEST, QUrl::fromLocalFile(testScriptPath));
+                }
+
+				// quite when finished parameter must directly follow the test script
+                if ((i + 2) < args.size() && args.at(i + 2) == TEST_QUIT_WHEN_FINISHED_OPTION) {
+                    quitWhenFinished = true;
+                }
+            } else if (args.at(i) == TEST_SNAPSHOT_LOCATION_COMMAND) {
+                // Set test snapshot location only if it is a writeable directory
+                QString pathname(args.at(i + 1));
+                QFileInfo fileInfo(pathname);
+                if (fileInfo.isDir() && fileInfo.isWritable()) {
+                    testSnapshotLocation = pathname;
                 }
             }
         }
@@ -2135,7 +2172,17 @@ Application::Application(int& argc, char** argv, QElapsedTimer& startupTimer, bo
     if (testProperty.isValid()) {
         auto scriptEngines = DependencyManager::get<ScriptEngines>();
         const auto testScript = property(hifi::properties::TEST).toUrl();
-        scriptEngines->loadScript(testScript, false);
+       
+        // Set last parameter to exit interface when the test script finishes, if so requested
+        scriptEngines->loadScript(testScript, false, false, false, false, quitWhenFinished);
+
+        // This is done so we don't get a "connection time-out" message when we haven't passed in a URL.
+        if (arguments().contains("--url")) {
+            auto reply = SandboxUtils::getStatus();
+            connect(reply, &QNetworkReply::finished, this, [=] {
+                handleSandboxStatus(reply);
+            });
+        }
     } else {
         PROFILE_RANGE(render, "GetSandboxStatus");
         auto reply = SandboxUtils::getStatus();
@@ -7465,7 +7512,7 @@ void Application::loadAvatarBrowser() const {
 void Application::takeSnapshot(bool notify, bool includeAnimated, float aspectRatio, const QString& filename) {
     postLambdaEvent([notify, includeAnimated, aspectRatio, filename, this] {
         // Get a screenshot and save it
-        QString path = Snapshot::saveSnapshot(getActiveDisplayPlugin()->getScreenshot(aspectRatio), filename);
+        QString path = Snapshot::saveSnapshot(getActiveDisplayPlugin()->getScreenshot(aspectRatio), filename, testSnapshotLocation);
         // If we're not doing an animated snapshot as well...
         if (!includeAnimated) {
             // Tell the dependency manager that the capture of the still snapshot has taken place.
@@ -7479,7 +7526,7 @@ void Application::takeSnapshot(bool notify, bool includeAnimated, float aspectRa
 
 void Application::takeSecondaryCameraSnapshot(const QString& filename) {
     postLambdaEvent([filename, this] {
-        QString snapshotPath = Snapshot::saveSnapshot(getActiveDisplayPlugin()->getSecondaryCameraScreenshot(), filename);
+        QString snapshotPath = Snapshot::saveSnapshot(getActiveDisplayPlugin()->getSecondaryCameraScreenshot(), filename, testSnapshotLocation);
         emit DependencyManager::get<WindowScriptingInterface>()->stillSnapshotTaken(snapshotPath, true);
     });
 }

interface/src/Application.h

@@ -750,5 +750,8 @@ private:
 
     std::atomic<bool> _pendingIdleEvent { true };
     std::atomic<bool> _pendingRenderEvent { true };
+
+    QString testSnapshotLocation;
+    bool quitWhenFinished { false };
 };
 #endif // hifi_Application_h

interface/src/ui/Snapshot.cpp

@@ -74,9 +74,9 @@ SnapshotMetaData* Snapshot::parseSnapshotData(QString snapshotPath) {
     return data;
 }
 
-QString Snapshot::saveSnapshot(QImage image, const QString& filename) {
+QString Snapshot::saveSnapshot(QImage image, const QString& filename, const QString& pathname) {
 
-    QFile* snapshotFile = savedFileForSnapshot(image, false, filename);
+    QFile* snapshotFile = savedFileForSnapshot(image, false, filename, pathname);
 
     // we don't need the snapshot file, so close it, grab its filename and delete it
     snapshotFile->close();
@@ -93,7 +93,7 @@ QTemporaryFile* Snapshot::saveTempSnapshot(QImage image) {
     return static_cast<QTemporaryFile*>(savedFileForSnapshot(image, true));
 }
 
-QFile* Snapshot::savedFileForSnapshot(QImage & shot, bool isTemporary, const QString& userSelectedFilename) {
+QFile* Snapshot::savedFileForSnapshot(QImage & shot, bool isTemporary, const QString& userSelectedFilename, const QString& userSelectedPathname) {
 
     // adding URL to snapshot
     QUrl currentURL = DependencyManager::get<AddressManager>()->currentPublicAddress();
@@ -118,7 +118,13 @@ QFile* Snapshot::savedFileForSnapshot(QImage & shot, bool isTemporary, const QSt
     const int IMAGE_QUALITY = 100;
 
     if (!isTemporary) {
-        QString snapshotFullPath = snapshotsLocation.get();
+        // If user has requested specific path then use it, else use the application value
+        QString snapshotFullPath;
+        if (!userSelectedPathname.isNull()) {
+            snapshotFullPath = userSelectedPathname;
+        } else {
+            snapshotFullPath = snapshotsLocation.get();
+        }
 
         if (snapshotFullPath.isEmpty()) {
             snapshotFullPath = OffscreenUi::getExistingDirectory(nullptr, "Choose Snapshots Directory", QStandardPaths::writableLocation(QStandardPaths::DesktopLocation));

interface/src/ui/Snapshot.h

@@ -38,7 +38,7 @@ class Snapshot : public QObject, public Dependency {
     Q_OBJECT
     SINGLETON_DEPENDENCY
 public:
-    static QString saveSnapshot(QImage image, const QString& filename);
+    static QString saveSnapshot(QImage image, const QString& filename, const QString& pathname = QString());
     static QTemporaryFile* saveTempSnapshot(QImage image);
     static SnapshotMetaData* parseSnapshotData(QString snapshotPath);
 
@@ -52,7 +52,10 @@ public slots:
     Q_INVOKABLE QString getSnapshotsLocation();
     Q_INVOKABLE void setSnapshotsLocation(const QString& location);
 private:
-    static QFile* savedFileForSnapshot(QImage & image, bool isTemporary, const QString& userSelectedFilename = QString());
+    static QFile* savedFileForSnapshot(QImage& image,
+                                       bool isTemporary,
+                                       const QString& userSelectedFilename = QString(),
+                                       const QString& userSelectedPathname = QString());
 };
 
 #endif // hifi_Snapshot_h

libraries/script-engine/src/ScriptEngine.h

@@ -526,6 +526,9 @@ public:
     void setUserLoaded(bool isUserLoaded) { _isUserLoaded = isUserLoaded; }
     bool isUserLoaded() const { return _isUserLoaded; }
 
+    void setQuitWhenFinished(const bool quitWhenFinished) { _quitWhenFinished = quitWhenFinished; }
+    bool isQuitWhenFinished() const { return _quitWhenFinished; }
+
     // NOTE - this is used by the TypedArray implementation. we need to review this for thread safety
     ArrayBufferClass* getArrayBufferClass() { return _arrayBufferClass; }
 
@@ -768,6 +771,8 @@ protected:
     std::atomic<bool> _isUserLoaded { false };
     bool _isReloading { false };
 
+    std::atomic<bool> _quitWhenFinished;
+
     ArrayBufferClass* _arrayBufferClass;
 
     AssetScriptingInterface* _assetScriptingInterface;

libraries/script-engine/src/ScriptEngines.cpp

@@ -347,7 +347,8 @@ void ScriptEngines::saveScripts() {
     {
         QReadLocker lock(&_scriptEnginesHashLock);
         for (auto it = _scriptEnginesHash.begin(); it != _scriptEnginesHash.end(); ++it) {
-            if (it.value() && it.value()->isUserLoaded()) {
+            // Save user-loaded scripts, only if they are set to quit when finished
+            if (it.value() && it.value()->isUserLoaded()  && !it.value()->isQuitWhenFinished()) {
                 auto normalizedUrl = normalizeScriptURL(it.key());
                 list.append(normalizedUrl.toString());
             }
@@ -456,7 +457,7 @@ void ScriptEngines::reloadAllScripts() {
 }
 
 ScriptEnginePointer ScriptEngines::loadScript(const QUrl& scriptFilename, bool isUserLoaded, bool loadScriptFromEditor,
-                                        bool activateMainWindow, bool reload) {
+                                              bool activateMainWindow, bool reload, bool quitWhenFinished) {
     if (thread() != QThread::currentThread()) {
         ScriptEnginePointer result { nullptr };
         BLOCKING_INVOKE_METHOD(this, "loadScript", Q_RETURN_ARG(ScriptEnginePointer, result),
@@ -488,6 +489,7 @@ ScriptEnginePointer ScriptEngines::loadScript(const QUrl& scriptFilename, bool i
     scriptEngine = ScriptEnginePointer(new ScriptEngine(_context, NO_SCRIPT, "about:" + scriptFilename.fileName()));
     addScriptEngine(scriptEngine);
     scriptEngine->setUserLoaded(isUserLoaded);
+    scriptEngine->setQuitWhenFinished(quitWhenFinished);
 
     if (scriptFilename.isEmpty() || !scriptUrl.isValid()) {
         launchScriptEngine(scriptEngine);
@@ -496,6 +498,11 @@ ScriptEnginePointer ScriptEngines::loadScript(const QUrl& scriptFilename, bool i
         connect(scriptEngine.data(), &ScriptEngine::scriptLoaded, this, &ScriptEngines::onScriptEngineLoaded);
         connect(scriptEngine.data(), &ScriptEngine::errorLoadingScript, this, &ScriptEngines::onScriptEngineError);
 
+        // Shutdown Interface when script finishes, if requested
+        if (quitWhenFinished) {
+            connect(scriptEngine.data(), &ScriptEngine::finished, this, &ScriptEngines::quitWhenFinished);
+        }
+
         // get the script engine object to load the script at the designated script URL
         scriptEngine->loadURL(scriptUrl, reload);
     }
@@ -536,6 +543,10 @@ void ScriptEngines::onScriptEngineLoaded(const QString& rawScriptURL) {
     emit scriptCountChanged();
 }
 
+void ScriptEngines::quitWhenFinished() {
+    qApp->quit();
+}
+
 int ScriptEngines::runScriptInitializers(ScriptEnginePointer scriptEngine) {
     int ii=0;
     for (auto initializer : _scriptInitializers) {

libraries/script-engine/src/ScriptEngines.h

@@ -88,10 +88,11 @@ public:
      * @param {boolean} [loadScriptFromEditor=false]
      * @param {boolean} [activateMainWindow=false]
      * @param {boolean} [reload=false]
+     * @param {boolean} [quitWhenFinished=false]
      * @returns {boolean}
      */
     Q_INVOKABLE ScriptEnginePointer loadScript(const QUrl& scriptFilename = QString(),
-        bool isUserLoaded = true, bool loadScriptFromEditor = false, bool activateMainWindow = false, bool reload = false);
+        bool isUserLoaded = true, bool loadScriptFromEditor = false, bool activateMainWindow = false, bool reload = false, bool quitWhenFinished = false);
 
     /**jsdoc
      * @function ScriptDiscoveryService.stopScript
@@ -266,6 +267,7 @@ protected:
     ScriptEnginePointer reloadScript(const QString& scriptName, bool isUserLoaded = true) { return loadScript(scriptName, isUserLoaded, false, false, true); }
     void removeScriptEngine(ScriptEnginePointer);
     void onScriptEngineLoaded(const QString& scriptFilename);
+    void quitWhenFinished();
     void onScriptEngineError(const QString& scriptFilename);
     void launchScriptEngine(ScriptEnginePointer);
 

tools/auto-tester/src/Downloader.cpp

@@ -9,6 +9,8 @@
 //
 #include "Downloader.h"
 
+#include <QtWidgets/QMessageBox>
+
 Downloader::Downloader(QUrl imageUrl, QObject *parent) : QObject(parent) {
     connect(
         &_networkAccessManager, SIGNAL (finished(QNetworkReply*)),
@@ -20,6 +22,12 @@ Downloader::Downloader(QUrl imageUrl, QObject *parent) : QObject(parent) {
 }
 
 void Downloader::fileDownloaded(QNetworkReply* reply) {
+    QNetworkReply::NetworkError error = reply->error();
+    if (error != QNetworkReply::NetworkError::NoError) {
+        QMessageBox::information(0, "Test Aborted", "Failed to download image: " + reply->errorString());
+        return;
+    }
+
     _downloadedData = reply->readAll();
 
     //emit a signal

tools/auto-tester/src/Test.cpp

@@ -27,7 +27,7 @@ Test::Test() {
     mismatchWindow.setModal(true);
 }
 
-bool Test::createTestResultsFolderPath(QString directory) {
+bool Test::createTestResultsFolderPath(const QString& directory) {
     QDateTime now = QDateTime::currentDateTime();
     testResultsFolderPath =  directory + "/" + TEST_RESULTS_FOLDER + "--" + now.toString(DATETIME_FORMAT);
     QDir testResultsFolder(testResultsFolderPath);
@@ -72,7 +72,7 @@ bool Test::compareImageLists(bool isInteractiveMode, QProgressBar* progressBar)
         QImage expectedImage(expectedImagesFullFilenames[i]);
 
         if (resultImage.width() != expectedImage.width() || resultImage.height() != expectedImage.height()) {
-            messageBox.critical(0, "Internal error: " + QString(__FILE__) + ":" + QString::number(__LINE__), "Images are not the same size");
+            QMessageBox::critical(0, "Internal error: " + QString(__FILE__) + ":" + QString::number(__LINE__), "Images are not the same size");
             exit(-1);
         }
 
@@ -80,7 +80,7 @@ bool Test::compareImageLists(bool isInteractiveMode, QProgressBar* progressBar)
         try {
             similarityIndex = imageComparer.compareImages(resultImage, expectedImage);
         } catch (...) {
-            messageBox.critical(0, "Internal error: " + QString(__FILE__) + ":" + QString::number(__LINE__), "Image not in expected format");
+            QMessageBox::critical(0, "Internal error: " + QString(__FILE__) + ":" + QString::number(__LINE__), "Image not in expected format");
             exit(-1);
         }
 
@@ -125,22 +125,22 @@ bool Test::compareImageLists(bool isInteractiveMode, QProgressBar* progressBar)
     return success;
 }
 
-void Test::appendTestResultsToFile(QString testResultsFolderPath, TestFailure testFailure, QPixmap comparisonImage) {
+void Test::appendTestResultsToFile(const QString& testResultsFolderPath, TestFailure testFailure, QPixmap comparisonImage) {
     if (!QDir().exists(testResultsFolderPath)) {
-        messageBox.critical(0, "Internal error: " + QString(__FILE__) + ":" + QString::number(__LINE__), "Folder " + testResultsFolderPath + " not found");
+        QMessageBox::critical(0, "Internal error: " + QString(__FILE__) + ":" + QString::number(__LINE__), "Folder " + testResultsFolderPath + " not found");
         exit(-1);
     }
 
     QString failureFolderPath { testResultsFolderPath + "/" + "Failure_" + QString::number(index) };
     if (!QDir().mkdir(failureFolderPath)) {
-        messageBox.critical(0, "Internal error: " + QString(__FILE__) + ":" + QString::number(__LINE__), "Failed to create folder " + failureFolderPath);
+        QMessageBox::critical(0, "Internal error: " + QString(__FILE__) + ":" + QString::number(__LINE__), "Failed to create folder " + failureFolderPath);
         exit(-1);
     }
     ++index;
 
     QFile descriptionFile(failureFolderPath + "/" + TEST_RESULTS_FILENAME);
     if (!descriptionFile.open(QIODevice::ReadWrite)) {
-        messageBox.critical(0, "Internal error: " + QString(__FILE__) + ":" + QString::number(__LINE__), "Failed to create file " + TEST_RESULTS_FILENAME);
+        QMessageBox::critical(0, "Internal error: " + QString(__FILE__) + ":" + QString::number(__LINE__), "Failed to create file " + TEST_RESULTS_FILENAME);
         exit(-1);
     }
 
@@ -160,24 +160,30 @@ void Test::appendTestResultsToFile(QString testResultsFolderPath, TestFailure te
     sourceFile = testFailure._pathname + testFailure._expectedImageFilename;
     destinationFile = failureFolderPath + "/" + "Expected Image.jpg";
     if (!QFile::copy(sourceFile, destinationFile)) {
-        messageBox.critical(0, "Internal error: " + QString(__FILE__) + ":" + QString::number(__LINE__), "Failed to copy " + sourceFile + " to " + destinationFile);
+        QMessageBox::critical(0, "Internal error: " + QString(__FILE__) + ":" + QString::number(__LINE__), "Failed to copy " + sourceFile + " to " + destinationFile);
         exit(-1);
     }
 
     sourceFile = testFailure._pathname + testFailure._actualImageFilename;
     destinationFile = failureFolderPath + "/" + "Actual Image.jpg";
     if (!QFile::copy(sourceFile, destinationFile)) {
-        messageBox.critical(0, "Internal error: " + QString(__FILE__) + ":" + QString::number(__LINE__), "Failed to copy " + sourceFile + " to " + destinationFile);
+        QMessageBox::critical(0, "Internal error: " + QString(__FILE__) + ":" + QString::number(__LINE__), "Failed to copy " + sourceFile + " to " + destinationFile);
         exit(-1);
     }
 
     comparisonImage.save(failureFolderPath + "/" + "Difference Image.jpg");
 }
 
-void Test::startTestsEvaluation() {
+void Test::startTestsEvaluation(const QString& testFolder) {
-    // Get list of JPEG images in folder, sorted by name
+    QString pathToTestResultsDirectory;
-    pathToTestResultsDirectory = QFileDialog::getExistingDirectory(nullptr, "Please select folder containing the test images", ".", QFileDialog::ShowDirsOnly);
+    if (testFolder.isNull()) {
-    if (pathToTestResultsDirectory == "") {
+        // Get list of JPEG images in folder, sorted by name
+        pathToTestResultsDirectory = QFileDialog::getExistingDirectory(nullptr, "Please select folder containing the test images", ".", QFileDialog::ShowDirsOnly);
+    } else {
+        pathToTestResultsDirectory = testFolder;
+    }
+
+    if (pathToTestResultsDirectory == QString()) {
         return;
     }
 
@@ -221,8 +227,8 @@ void Test::startTestsEvaluation() {
             QString expectedImageFilenameTail = currentFilename.left(currentFilename.length() - 4).right(NUM_DIGITS);
             QString expectedImageStoredFilename = EXPECTED_IMAGE_PREFIX + expectedImageFilenameTail + ".png";
 
-            QString imageURLString("https://github.com/" + githubUser + "/hifi_tests/blob/" + gitHubBranch + "/" + 
+            QString imageURLString("https://raw.githubusercontent.com/" + githubUser + "/hifi_tests/" + gitHubBranch + "/" + 
-                expectedImagePartialSourceDirectory + "/" + expectedImageStoredFilename + "?raw=true");
+                expectedImagePartialSourceDirectory + "/" + expectedImageStoredFilename);
 
             expectedImagesURLs << imageURLString;
 
@@ -237,19 +243,21 @@ void Test::startTestsEvaluation() {
     autoTester->downloadImages(expectedImagesURLs, pathToTestResultsDirectory, expectedImagesFilenames);
 }
 
-void Test::finishTestsEvaluation(bool interactiveMode, QProgressBar* progressBar) {
+void Test::finishTestsEvaluation(bool isRunningFromCommandline, bool interactiveMode, QProgressBar* progressBar) {
-    bool success = compareImageLists(interactiveMode, progressBar);
+    bool success = compareImageLists((!isRunningFromCommandline && interactiveMode), progressBar);
     
-    if (success) {
+    if (!isRunningFromCommandline) {
-        messageBox.information(0, "Success", "All images are as expected");
+        if (success) {
-    } else {
+            QMessageBox::information(0, "Success", "All images are as expected");
-        messageBox.information(0, "Failure", "One or more images are not as expected");
+        } else {
+            QMessageBox::information(0, "Failure", "One or more images are not as expected");
+        }
     }
 
     zipAndDeleteTestResultsFolder();
 }
 
-bool Test::isAValidDirectory(QString pathname) {
+bool Test::isAValidDirectory(const QString& pathname) {
     // Only process directories
     QDir dir(pathname);
     if (!dir.exists()) {
@@ -264,7 +272,7 @@ bool Test::isAValidDirectory(QString pathname) {
     return true;
 }
 
-QString Test::extractPathFromTestsDown(QString fullPath) {
+QString Test::extractPathFromTestsDown(const QString& fullPath) {
     // `fullPath` includes the full path to the test.  We need the portion below (and including) `tests`
     QStringList pathParts = fullPath.split('/');
     int i{ 0 };
@@ -273,7 +281,7 @@ QString Test::extractPathFromTestsDown(QString fullPath) {
     }
 
     if (i == pathParts.length()) {
-        messageBox.critical(0, "Internal error: " + QString(__FILE__) + ":" + QString::number(__LINE__), "Bad testPathname");
+        QMessageBox::critical(0, "Internal error: " + QString(__FILE__) + ":" + QString::number(__LINE__), "Bad testPathname");
         exit(-1);
     }
 
@@ -342,14 +350,14 @@ void Test::createAllRecursiveScripts() {
         }
     }
 
-    messageBox.information(0, "Success", "Scripts have been created");
+    QMessageBox::information(0, "Success", "Scripts have been created");
 }
 
-void Test::createRecursiveScript(QString topLevelDirectory, bool interactiveMode) {
+void Test::createRecursiveScript(const QString& topLevelDirectory, bool interactiveMode) {
     const QString recursiveTestsFilename("testRecursive.js");
     QFile allTestsFilename(topLevelDirectory + "/" + recursiveTestsFilename);
     if (!allTestsFilename.open(QIODevice::WriteOnly | QIODevice::Text)) {
-        messageBox.critical(0,
+        QMessageBox::critical(0,
             "Internal error: " + QString(__FILE__) + ":" + QString::number(__LINE__),
             "Failed to create \"" + recursiveTestsFilename + "\" in directory \"" + topLevelDirectory + "\""
         );
@@ -358,12 +366,15 @@ void Test::createRecursiveScript(QString topLevelDirectory, bool interactiveMode
     }
 
     QTextStream textStream(&allTestsFilename);
-    textStream << "// This is an automatically generated file, created by auto-tester" << endl << endl;
+
+    const QString DATE_TIME_FORMAT("MMM d yyyy, h:mm");
+    textStream << "// This is an automatically generated file, created by auto-tester on " << QDateTime::currentDateTime().toString(DATE_TIME_FORMAT) << endl << endl;
 
     textStream << "var autoTester = Script.require(\"https://github.com/" + githubUser + "/hifi_tests/blob/" 
-        + gitHubBranch + "/tests/utils/autoTester.js?raw=true\");" << endl;
+        + gitHubBranch + "/tests/utils/autoTester.js?raw=true\");" << endl << endl;
 
-    textStream << "autoTester.enableRecursive();" << endl << endl;
+    textStream << "autoTester.enableRecursive();" << endl;
+    textStream << "autoTester.enableAuto();" << endl << endl;
 
     QVector<QString> testPathnames;
 
@@ -398,7 +409,7 @@ void Test::createRecursiveScript(QString topLevelDirectory, bool interactiveMode
     }
 
     if (interactiveMode && testPathnames.length() <= 0) {
-        messageBox.information(0, "Failure", "No \"" + TEST_FILENAME + "\" files found");
+        QMessageBox::information(0, "Failure", "No \"" + TEST_FILENAME + "\" files found");
         allTestsFilename.close();
         return;
     }
@@ -409,7 +420,7 @@ void Test::createRecursiveScript(QString topLevelDirectory, bool interactiveMode
     allTestsFilename.close();
     
     if (interactiveMode) {
-        messageBox.information(0, "Success", "Script has been created");
+        QMessageBox::information(0, "Success", "Script has been created");
     }
 }
 
@@ -434,7 +445,7 @@ void Test::createTest() {
         QString fullCurrentFilename = imageSourceDirectory + "/" + currentFilename;
         if (isInSnapshotFilenameFormat("jpg", currentFilename)) {
             if (i >= maxImages) {
-                messageBox.critical(0, "Error", "More than " + QString::number(maxImages) + " images not supported");
+                QMessageBox::critical(0, "Error", "More than " + QString::number(maxImages) + " images not supported");
                 exit(-1);
             }
             QString newFilename = "ExpectedImage_" + QString::number(i - 1).rightJustified(5, '0') + ".png";
@@ -443,14 +454,14 @@ void Test::createTest() {
             try {
                 copyJPGtoPNG(fullCurrentFilename, fullNewFileName);
             } catch (...) {
-                messageBox.critical(0, "Error", "Could not delete existing file: " + currentFilename + "\nTest creation aborted");
+                QMessageBox::critical(0, "Error", "Could not delete existing file: " + currentFilename + "\nTest creation aborted");
                 exit(-1);
             }
             ++i;
         }
     }
 
-    messageBox.information(0, "Success", "Test images have been created");
+    QMessageBox::information(0, "Success", "Test images have been created");
 }
 
 ExtractedText Test::getTestScriptLines(QString testFileName) {
@@ -459,7 +470,7 @@ ExtractedText Test::getTestScriptLines(QString testFileName) {
     QFile inputFile(testFileName);
     inputFile.open(QIODevice::ReadOnly);
     if (!inputFile.isOpen()) {
-        messageBox.critical(0,
+        QMessageBox::critical(0,
             "Internal error: " + QString(__FILE__) + ":" + QString::number(__LINE__),
             "Failed to open \"" + testFileName
         );
@@ -498,6 +509,12 @@ ExtractedText Test::getTestScriptLines(QString testFileName) {
     const QString regexAssertGPU(ws + functionAssertGPU + ws + "\\(" + ws + quotedString + ".*");
     const QRegularExpression lineAssertGPU = QRegularExpression(regexAssertGPU);
 
+    // Assert the correct amount of memory
+    const QString functionAssertPhysicalMemoryGB(ws + "autoTester" + ws + "\\." + ws + "assertPhysicalMemoryGB");
+    const QString regexAssertPhysicalMemoryGB(ws + functionAssertPhysicalMemoryGB + ws + "\\(" + ws + quotedString + ".*");
+    const QRegularExpression lineAssertPhysicalMemoryGB = QRegularExpression(regexAssertPhysicalMemoryGB);
+
+
     // Each step is either of the following forms:
     //        autoTester.addStepSnapshot("Take snapshot"...
     //        autoTester.addStep("Clean up after test"...
@@ -514,18 +531,7 @@ ExtractedText Test::getTestScriptLines(QString testFileName) {
         if (lineContainingTitle.match(line).hasMatch()) {
             QStringList tokens = line.split('"');
             relevantTextFromTest.title = tokens[1];
-        } else if (lineAssertPlatform.match(line).hasMatch()) {
+
-            QStringList platforms = line.split('"');
-            relevantTextFromTest.platform = platforms[1];
-        } else if (lineAssertDisplay.match(line).hasMatch()) {
-            QStringList displays = line.split('"');
-            relevantTextFromTest.display = displays[1];
-        } else if (lineAssertCPU.match(line).hasMatch()) {
-            QStringList cpus = line.split('"');
-            relevantTextFromTest.cpu = cpus[1];
-        } else if (lineAssertGPU.match(line).hasMatch()) {
-            QStringList gpus = line.split('"');
-            relevantTextFromTest.gpu = gpus[1];
         } else if (lineStepSnapshot.match(line).hasMatch()) {
             QStringList tokens = line.split('"');
             QString nameOfStep = tokens[1];
@@ -534,6 +540,7 @@ ExtractedText Test::getTestScriptLines(QString testFileName) {
             step->text = nameOfStep;
             step->takeSnapshot = true;
             relevantTextFromTest.stepList.emplace_back(step);
+
         } else if (lineStep.match(line).hasMatch()) {
             QStringList tokens = line.split('"');
             QString nameOfStep = tokens[1];
@@ -593,15 +600,15 @@ void Test::createAllMDFiles() {
         }
     }
 
-    messageBox.information(0, "Success", "MD files have been created");
+    QMessageBox::information(0, "Success", "MD files have been created");
 }
 
-void Test::createMDFile(QString testDirectory) {
+void Test::createMDFile(const QString& testDirectory) {
     // Verify folder contains test.js file
     QString testFileName(testDirectory + "/" + TEST_FILENAME);
     QFileInfo testFileInfo(testFileName);
     if (!testFileInfo.exists()) {
-        messageBox.critical(0, "Error", "Could not find file: " + TEST_FILENAME);
+        QMessageBox::critical(0, "Error", "Could not find file: " + TEST_FILENAME);
         return;
     }
 
@@ -610,7 +617,7 @@ void Test::createMDFile(QString testDirectory) {
     QString mdFilename(testDirectory + "/" + "test.md");
     QFile mdFile(mdFilename);
     if (!mdFile.open(QIODevice::WriteOnly)) {
-        messageBox.critical(0, "Internal error: " + QString(__FILE__) + ":" + QString::number(__LINE__), "Failed to create file " + mdFilename);
+        QMessageBox::critical(0, "Internal error: " + QString(__FILE__) + ":" + QString::number(__LINE__), "Failed to create file " + mdFilename);
         exit(-1);
     }
 
@@ -628,64 +635,24 @@ void Test::createMDFile(QString testDirectory) {
     stream << "## Preconditions" << "\n";
     stream << "- In an empty region of a domain with editing rights." << "\n\n";
 
-    // Platform
-    QStringList  platforms = testScriptLines.platform.split(" ");;
-    stream << "## Platforms\n";
-    stream << "Run the test on each of the following platforms\n";
-    for (int i = 0; i < platforms.size(); ++i) {
-        // Note that the platforms parameter may include extra spaces, these appear as empty strings in the list
-        if (platforms[i] != QString()) {
-            stream << " - " << platforms[i] << "\n";
-        }
-    }
-
-    // Display
-    QStringList  displays = testScriptLines.display.split(" ");
-    stream << "## Displays\n";
-    stream << "Run the test on each of the following displays\n";
-    for (int i = 0; i < displays.size(); ++i) {
-        // Note that the displays parameter may include extra spaces, these appear as empty strings in the list
-        if (displays[i] != QString()) {
-            stream << " - " << displays[i] << "\n";
-        }
-    }
-
-    // CPU
-    QStringList  cpus = testScriptLines.cpu.split(" ");
-    stream << "## Processors\n";
-    stream << "Run the test on each of the following processors\n";
-    for (int i = 0; i < cpus.size(); ++i) {
-        // Note that the cpus parameter may include extra spaces, these appear as empty strings in the list
-        if (cpus[i] != QString()) {
-            stream << " - " << cpus[i] << "\n";
-        }
-    }
-
-    // GPU
-    QStringList  gpus = testScriptLines.gpu.split(" ");
-    stream << "## Graphics Cards\n";
-    stream << "Run the test on graphics cards from each of the following vendors\n";
-    for (int i = 0; i < gpus.size(); ++i) {
-        // Note that the gpus parameter may include extra spaces, these appear as empty strings in the list
-        if (gpus[i] != QString()) {
-            stream << " - " << gpus[i] << "\n";
-        }
-    }
-
     stream << "## Steps\n";
     stream << "Press space bar to advance step by step\n\n";
 
+    // Note that snapshots of step n are taken in step n+1
+    // (this implies that if the LAST step requests a snapshot then this will not work - caveat emptor)
     int snapShotIndex { 0 };
     for (size_t i = 0; i < testScriptLines.stepList.size(); ++i) {
         stream << "### Step " << QString::number(i + 1) << "\n";
         stream << "- " << testScriptLines.stepList[i]->text << "\n";
-        if (testScriptLines.stepList[i]->takeSnapshot) {
+        if ((i + 1 < testScriptLines.stepList.size()) && testScriptLines.stepList[i + 1]->takeSnapshot) {
             stream << "- ![](./ExpectedImage_" << QString::number(snapShotIndex).rightJustified(5, '0') << ".png)\n";
             ++snapShotIndex;
         }
     }
 
     mdFile.close();
+
+    QMessageBox::information(0, "Success", "Test MD file " + mdFilename + " has been created");
 }
 
 void Test::createTestsOutline() {
@@ -698,7 +665,7 @@ void Test::createTestsOutline() {
     QString mdFilename(testsRootDirectory + "/" + testsOutlineFilename);
     QFile mdFile(mdFilename);
     if (!mdFile.open(QIODevice::WriteOnly)) {
-        messageBox.critical(0, "Internal error: " + QString(__FILE__) + ":" + QString::number(__LINE__), "Failed to create file " + mdFilename);
+        QMessageBox::critical(0, "Internal error: " + QString(__FILE__) + ":" + QString::number(__LINE__), "Failed to create file " + mdFilename);
         exit(-1);
     }
 
@@ -756,10 +723,10 @@ void Test::createTestsOutline() {
 
     mdFile.close();
 
-    messageBox.information(0, "Success", "Test outline file " + testsOutlineFilename + " has been created");
+    QMessageBox::information(0, "Success", "Test outline file " + testsOutlineFilename + " has been created");
 }
 
-void Test::copyJPGtoPNG(QString sourceJPGFullFilename, QString destinationPNGFullFilename) {
+void Test::copyJPGtoPNG(const QString& sourceJPGFullFilename, const QString& destinationPNGFullFilename) {
     QFile::remove(destinationPNGFullFilename);
 
     QImageReader reader;
@@ -772,7 +739,7 @@ void Test::copyJPGtoPNG(QString sourceJPGFullFilename, QString destinationPNGFul
     writer.write(image);
 }
 
-QStringList Test::createListOfAll_imagesInDirectory(QString imageFormat, QString pathToImageDirectory) {
+QStringList Test::createListOfAll_imagesInDirectory(const QString& imageFormat, const QString& pathToImageDirectory) {
     imageDirectory = QDir(pathToImageDirectory);
     QStringList nameFilters;
     nameFilters << "*." + imageFormat;
@@ -785,7 +752,7 @@ QStringList Test::createListOfAll_imagesInDirectory(QString imageFormat, QString
 //      Filename (i.e. without extension) contains _tests_ (this is based on all test scripts being within the tests folder
 //      Last 5 characters in filename are digits
 //      Extension is jpg
-bool Test::isInSnapshotFilenameFormat(QString imageFormat, QString filename) {
+bool Test::isInSnapshotFilenameFormat(const QString& imageFormat, const QString& filename) {
     QStringList filenameParts = filename.split(".");
 
     bool filnameHasNoPeriods = (filenameParts.size() == 2);
@@ -802,7 +769,7 @@ bool Test::isInSnapshotFilenameFormat(QString imageFormat, QString filename) {
 // For a file named "D_GitHub_hifi-tests_tests_content_entity_zone_create_0.jpg", the test directory is
 // D:/GitHub/hifi-tests/tests/content/entity/zone/create
 // This method assumes the filename is in the correct format
-QString Test::getExpectedImageDestinationDirectory(QString filename) {
+QString Test::getExpectedImageDestinationDirectory(const QString& filename) {
     QString filenameWithoutExtension = filename.split(".")[0];
     QStringList filenameParts = filenameWithoutExtension.split("_");
 
@@ -819,7 +786,7 @@ QString Test::getExpectedImageDestinationDirectory(QString filename) {
 // is ...tests/content/entity/zone/create
 // This is used to create the full URL
 // This method assumes the filename is in the correct format
-QString Test::getExpectedImagePartialSourceDirectory(QString filename) {
+QString Test::getExpectedImagePartialSourceDirectory(const QString& filename) {
     QString filenameWithoutExtension = filename.split(".")[0];
     QStringList filenameParts = filenameWithoutExtension.split("_");
 
@@ -831,7 +798,7 @@ QString Test::getExpectedImagePartialSourceDirectory(QString filename) {
     }
 
     if (i < 0) {
-        messageBox.critical(0, "Internal error: " + QString(__FILE__) + ":" + QString::number(__LINE__), "Bad filename");
+        QMessageBox::critical(0, "Internal error: " + QString(__FILE__) + ":" + QString::number(__LINE__), "Bad filename");
         exit(-1);
     }
 

tools/auto-tester/src/Test.h

@@ -30,10 +30,6 @@ using StepList = std::vector<Step*>;
 class ExtractedText {
 public:
     QString title;
-    QString platform;
-    QString display;
-    QString cpu;
-    QString gpu;
     StepList stepList;
 };
 
@@ -41,61 +37,58 @@ class Test {
 public: 
     Test();
 
-    void startTestsEvaluation();
+    void startTestsEvaluation(const QString& testFolder = QString());
-    void finishTestsEvaluation(bool interactiveMode, QProgressBar* progressBar);
+    void finishTestsEvaluation(bool isRunningFromCommandline, bool interactiveMode, QProgressBar* progressBar);
 
     void createRecursiveScript();
     void createAllRecursiveScripts();
-    void createRecursiveScript(QString topLevelDirectory, bool interactiveMode);
+    void createRecursiveScript(const QString& topLevelDirectory, bool interactiveMode);
 
     void createTest();
     void createMDFile();
     void createAllMDFiles();
-    void createMDFile(QString topLevelDirectory);
+    void createMDFile(const QString& topLevelDirectory);
 
     void createTestsOutline();
 
     bool compareImageLists(bool isInteractiveMode, QProgressBar* progressBar);
 
-    QStringList createListOfAll_imagesInDirectory(QString imageFormat, QString pathToImageDirectory);
+    QStringList createListOfAll_imagesInDirectory(const QString& imageFormat, const QString& pathToImageDirectory);
 
-    bool isInSnapshotFilenameFormat(QString imageFormat, QString filename);
+    bool isInSnapshotFilenameFormat(const QString& imageFormat, const QString& filename);
 
     void importTest(QTextStream& textStream, const QString& testPathname);
 
-    void appendTestResultsToFile(QString testResultsFolderPath, TestFailure testFailure, QPixmap comparisonImage);
+    void appendTestResultsToFile(const QString& testResultsFolderPath, TestFailure testFailure, QPixmap comparisonImage);
 
-    bool createTestResultsFolderPath(QString directory);
+    bool createTestResultsFolderPath(const QString& directory);
     void zipAndDeleteTestResultsFolder();
 
-    bool isAValidDirectory(QString pathname);
+    bool isAValidDirectory(const QString& pathname);
-	QString extractPathFromTestsDown(QString fullPath);
+	QString extractPathFromTestsDown(const QString& fullPath);
-    QString getExpectedImageDestinationDirectory(QString filename);
+    QString getExpectedImageDestinationDirectory(const QString& filename);
-    QString getExpectedImagePartialSourceDirectory(QString filename);
+    QString getExpectedImagePartialSourceDirectory(const QString& filename);
 
-    void copyJPGtoPNG(QString sourceJPGFullFilename, QString destinationPNGFullFilename);
+    void copyJPGtoPNG(const QString& sourceJPGFullFilename, const QString& destinationPNGFullFilename);
 
 private:
     const QString TEST_FILENAME { "test.js" };
     const QString TEST_RESULTS_FOLDER { "TestResults" };
     const QString TEST_RESULTS_FILENAME { "TestResults.txt" };
 
-    QMessageBox messageBox;
-
     QDir imageDirectory;
 
     MismatchWindow mismatchWindow;
 
     ImageComparer imageComparer;
 
-    QString testResultsFolderPath { "" };
+    QString testResultsFolderPath;
     int index { 1 };
 
     // Expected images are in the format ExpectedImage_dddd.jpg (d == decimal digit)
     const int NUM_DIGITS { 5 };
     const QString EXPECTED_IMAGE_PREFIX { "ExpectedImage_" };
 
-    QString pathToTestResultsDirectory;
     QStringList expectedImagesFilenames;
     QStringList expectedImagesFullFilenames;
     QStringList resultImagesFullFilenames;

tools/auto-tester/src/main.cpp

@@ -13,10 +13,23 @@
 AutoTester* autoTester;
 
 int main(int argc, char *argv[]) {
+    // Only parameter is "--testFolder"
+    QString testFolder;
+    if (argc == 3) {
+        if (QString(argv[1]) == "--testFolder") {
+            testFolder = QString(argv[2]);
+        }
+    }
+
     QApplication application(argc, argv);
 
     autoTester = new AutoTester();
-    autoTester->show();
+
+    if (!testFolder.isNull()) {
+        autoTester->runFromCommandLine(testFolder);
+    } else {
+        autoTester->show();
+    }
 
     return application.exec();
 }

tools/auto-tester/src/ui/AutoTester.cpp

@@ -15,9 +15,17 @@ AutoTester::AutoTester(QWidget *parent) : QMainWindow(parent) {
     ui.checkBoxInteractiveMode->setChecked(true);
     ui.progressBar->setVisible(false);
 
-    test = new Test();
-
     signalMapper = new QSignalMapper();
+
+    connect(ui.actionClose, &QAction::triggered, this, &AutoTester::on_closeButton_clicked);
+    connect(ui.actionAbout, &QAction::triggered, this, &AutoTester::about);
+
+    test = new Test();
+}
+
+void AutoTester::runFromCommandLine(const QString& testFolder) {
+    isRunningFromCommandline = true;
+    test->startTestsEvaluation(testFolder);
 }
 
 void AutoTester::on_evaluateTestsButton_clicked() {
@@ -90,13 +98,21 @@ void AutoTester::saveImage(int index) {
     image = image.convertToFormat(QImage::Format_ARGB32);
 
     QString fullPathname = _directoryName + "/" + _filenames[index];
-    image.save(fullPathname, 0, 100);
+    if (!image.save(fullPathname, 0, 100)) {
+        QMessageBox::information(0, "Test Aborted", "Failed to save image: " + _filenames[index]);
+        ui.progressBar->setVisible(false);
+        return;
+    }
 
     ++_numberOfImagesDownloaded;
 
     if (_numberOfImagesDownloaded == _numberOfImagesToDownload) {
-        test->finishTestsEvaluation(ui.checkBoxInteractiveMode->isChecked(), ui.progressBar);
+        test->finishTestsEvaluation(isRunningFromCommandline, ui.checkBoxInteractiveMode->isChecked(), ui.progressBar);
     } else {
         ui.progressBar->setValue(_numberOfImagesDownloaded);
     }
 }
+
+void AutoTester::about() {
+    QMessageBox::information(0, "About", QString("Built ") + __DATE__ + " : " + __TIME__);
+}

tools/auto-tester/src/ui/AutoTester.h

@@ -22,6 +22,9 @@ class AutoTester : public QMainWindow {
 
 public:
     AutoTester(QWidget *parent = Q_NULLPTR);
+
+    void runFromCommandLine(const QString& testFolder);
+
     void downloadImage(const QUrl& url);
     void downloadImages(const QStringList& URLs, const QString& directoryName, const QStringList& filenames);
 
@@ -37,6 +40,8 @@ private slots:
 
     void saveImage(int index);
 
+    void about();
+
 private:
     Ui::AutoTesterClass ui;
     Test* test;
@@ -50,9 +55,11 @@ private:
     // Used to enable passing a parameter to slots
     QSignalMapper* signalMapper;
 
-    int _numberOfImagesToDownload;
+    int _numberOfImagesToDownload { 0 };
-    int _numberOfImagesDownloaded;
+    int _numberOfImagesDownloaded { 0 };
-    int _index;
+    int _index { 0 };
+
+    bool isRunningFromCommandline { false };
 };
 
 #endif // hifi_AutoTester_h

tools/auto-tester/src/ui/AutoTester.ui

@@ -95,7 +95,7 @@
      <number>24</number>
     </property>
    </widget>
-   <widget class="QPushButton" name="createRecursiveScriptsRecursivelyButton">
+   <widget class="QPushButton" name="createAllRecursiveScriptsButton">
     <property name="geometry">
      <rect>
       <x>360</x>
@@ -157,6 +157,20 @@
      <height>21</height>
     </rect>
    </property>
+   <widget class="QMenu" name="menuFile">
+    <property name="title">
+     <string>File</string>
+    </property>
+    <addaction name="actionClose"/>
+   </widget>
+   <widget class="QMenu" name="menuHelp">
+    <property name="title">
+     <string>Help</string>
+    </property>
+    <addaction name="actionAbout"/>
+   </widget>
+   <addaction name="menuFile"/>
+   <addaction name="menuHelp"/>
   </widget>
   <widget class="QToolBar" name="mainToolBar">
    <attribute name="toolBarArea">
@@ -167,6 +181,16 @@
    </attribute>
   </widget>
   <widget class="QStatusBar" name="statusBar"/>
+  <action name="actionClose">
+   <property name="text">
+    <string>Close</string>
+   </property>
+  </action>
+  <action name="actionAbout">
+   <property name="text">
+    <string>About</string>
+   </property>
+  </action>
  </widget>
  <layoutdefault spacing="6" margin="11"/>
  <resources/>