diff --git a/interface/src/Application.cpp b/interface/src/Application.cpp
index e1bdbfb76c..69c835248d 100644
--- a/interface/src/Application.cpp
+++ b/interface/src/Application.cpp
@@ -743,6 +743,11 @@ extern DisplayPluginList getDisplayPlugins();
extern InputPluginList getInputPlugins();
extern void saveInputPluginSettings(const InputPluginList& plugins);
+// Parameters used for running tests from teh command line
+const QString TEST_SCRIPT_COMMAND { "--testScript" };
+const QString TEST_QUIT_WHEN_FINISHED_OPTION { "quitWhenFinished" };
+const QString TEST_SNAPSHOT_LOCATION_COMMAND { "--testSnapshotLocation" };
+
bool setupEssentials(int& argc, char** argv, bool runningMarkerExisted) {
const char** constArgv = const_cast<const char**>(argv);
@@ -777,7 +782,22 @@ bool setupEssentials(int& argc, char** argv, bool runningMarkerExisted) {
static const auto SUPPRESS_SETTINGS_RESET = "--suppress-settings-reset";
bool suppressPrompt = cmdOptionExists(argc, const_cast<const char**>(argv), SUPPRESS_SETTINGS_RESET);
- bool previousSessionCrashed = CrashHandler::checkForResetSettings(runningMarkerExisted, suppressPrompt);
+
+ // Ignore any previous crashes if running from command line with a test script.
+ bool inTestMode { false };
+ for (int i = 0; i < argc; ++i) {
+ QString parameter(argv[i]);
+ if (parameter == TEST_SCRIPT_COMMAND) {
+ inTestMode = true;
+ break;
+ }
+ }
+
+ bool previousSessionCrashed { false };
+ if (!inTestMode) {
+ previousSessionCrashed = CrashHandler::checkForResetSettings(runningMarkerExisted, suppressPrompt);
+ }
+
// get dir to use for cache
static const auto CACHE_SWITCH = "--cache";
QString cacheDir = getCmdOption(argc, const_cast<const char**>(argv), CACHE_SWITCH);
@@ -996,13 +1016,30 @@ Application::Application(int& argc, char** argv, QElapsedTimer& startupTimer, bo
setProperty(hifi::properties::STEAM, (steamClient && steamClient->isRunning()));
setProperty(hifi::properties::CRASHED, _previousSessionCrashed);
{
- const QString TEST_SCRIPT = "--testScript";
const QStringList args = arguments();
+
for (int i = 0; i < args.size() - 1; ++i) {
- if (args.at(i) == TEST_SCRIPT) {
+ if (args.at(i) == TEST_SCRIPT_COMMAND && (i + 1) < args.size()) {
QString testScriptPath = args.at(i + 1);
- if (QFileInfo(testScriptPath).exists()) {
+
+ // If the URL scheme is http(s) or ftp, then use as is, else - treat it as a local file
+ // This is done so as not break previous command line scripts
+ if (testScriptPath.left(URL_SCHEME_HTTP.length()) == URL_SCHEME_HTTP || testScriptPath.left(URL_SCHEME_FTP.length()) == URL_SCHEME_FTP) {
+ setProperty(hifi::properties::TEST, QUrl::fromUserInput(testScriptPath));
+ } else if (QFileInfo(testScriptPath).exists()) {
setProperty(hifi::properties::TEST, QUrl::fromLocalFile(testScriptPath));
+ }
+
+ // quite when finished parameter must directly follow the test script
+ if ((i + 2) < args.size() && args.at(i + 2) == TEST_QUIT_WHEN_FINISHED_OPTION) {
+ quitWhenFinished = true;
+ }
+ } else if (args.at(i) == TEST_SNAPSHOT_LOCATION_COMMAND) {
+ // Set test snapshot location only if it is a writeable directory
+ QString pathname(args.at(i + 1));
+ QFileInfo fileInfo(pathname);
+ if (fileInfo.isDir() && fileInfo.isWritable()) {
+ testSnapshotLocation = pathname;
}
}
}
@@ -2135,7 +2172,17 @@ Application::Application(int& argc, char** argv, QElapsedTimer& startupTimer, bo
if (testProperty.isValid()) {
auto scriptEngines = DependencyManager::get<ScriptEngines>();
const auto testScript = property(hifi::properties::TEST).toUrl();
- scriptEngines->loadScript(testScript, false);
+
+ // Set last parameter to exit interface when the test script finishes, if so requested
+ scriptEngines->loadScript(testScript, false, false, false, false, quitWhenFinished);
+
+ // This is done so we don't get a "connection time-out" message when we haven't passed in a URL.
+ if (arguments().contains("--url")) {
+ auto reply = SandboxUtils::getStatus();
+ connect(reply, &QNetworkReply::finished, this, [=] {
+ handleSandboxStatus(reply);
+ });
+ }
} else {
PROFILE_RANGE(render, "GetSandboxStatus");
auto reply = SandboxUtils::getStatus();
@@ -7465,7 +7512,7 @@ void Application::loadAvatarBrowser() const {
void Application::takeSnapshot(bool notify, bool includeAnimated, float aspectRatio, const QString& filename) {
postLambdaEvent([notify, includeAnimated, aspectRatio, filename, this] {
// Get a screenshot and save it
- QString path = Snapshot::saveSnapshot(getActiveDisplayPlugin()->getScreenshot(aspectRatio), filename);
+ QString path = Snapshot::saveSnapshot(getActiveDisplayPlugin()->getScreenshot(aspectRatio), filename, testSnapshotLocation);
// If we're not doing an animated snapshot as well...
if (!includeAnimated) {
// Tell the dependency manager that the capture of the still snapshot has taken place.
@@ -7479,7 +7526,7 @@ void Application::takeSnapshot(bool notify, bool includeAnimated, float aspectRa
void Application::takeSecondaryCameraSnapshot(const QString& filename) {
postLambdaEvent([filename, this] {
- QString snapshotPath = Snapshot::saveSnapshot(getActiveDisplayPlugin()->getSecondaryCameraScreenshot(), filename);
+ QString snapshotPath = Snapshot::saveSnapshot(getActiveDisplayPlugin()->getSecondaryCameraScreenshot(), filename, testSnapshotLocation);
emit DependencyManager::get<WindowScriptingInterface>()->stillSnapshotTaken(snapshotPath, true);
});
}
diff --git a/interface/src/Application.h b/interface/src/Application.h
index 654b5c797b..aa6469c592 100644
--- a/interface/src/Application.h
+++ b/interface/src/Application.h
@@ -750,5 +750,8 @@ private:
std::atomic<bool> _pendingIdleEvent { true };
std::atomic<bool> _pendingRenderEvent { true };
+
+ QString testSnapshotLocation;
+ bool quitWhenFinished { false };
};
#endif // hifi_Application_h
diff --git a/interface/src/ui/Snapshot.cpp b/interface/src/ui/Snapshot.cpp
index c6750ad424..39fef1d742 100644
--- a/interface/src/ui/Snapshot.cpp
+++ b/interface/src/ui/Snapshot.cpp
@@ -74,9 +74,9 @@ SnapshotMetaData* Snapshot::parseSnapshotData(QString snapshotPath) {
return data;
}
-QString Snapshot::saveSnapshot(QImage image, const QString& filename) {
+QString Snapshot::saveSnapshot(QImage image, const QString& filename, const QString& pathname) {
- QFile* snapshotFile = savedFileForSnapshot(image, false, filename);
+ QFile* snapshotFile = savedFileForSnapshot(image, false, filename, pathname);
// we don't need the snapshot file, so close it, grab its filename and delete it
snapshotFile->close();
@@ -93,7 +93,7 @@ QTemporaryFile* Snapshot::saveTempSnapshot(QImage image) {
return static_cast<QTemporaryFile*>(savedFileForSnapshot(image, true));
}
-QFile* Snapshot::savedFileForSnapshot(QImage & shot, bool isTemporary, const QString& userSelectedFilename) {
+QFile* Snapshot::savedFileForSnapshot(QImage & shot, bool isTemporary, const QString& userSelectedFilename, const QString& userSelectedPathname) {
// adding URL to snapshot
QUrl currentURL = DependencyManager::get<AddressManager>()->currentPublicAddress();
@@ -118,7 +118,13 @@ QFile* Snapshot::savedFileForSnapshot(QImage & shot, bool isTemporary, const QSt
const int IMAGE_QUALITY = 100;
if (!isTemporary) {
- QString snapshotFullPath = snapshotsLocation.get();
+ // If user has requested specific path then use it, else use the application value
+ QString snapshotFullPath;
+ if (!userSelectedPathname.isNull()) {
+ snapshotFullPath = userSelectedPathname;
+ } else {
+ snapshotFullPath = snapshotsLocation.get();
+ }
if (snapshotFullPath.isEmpty()) {
snapshotFullPath = OffscreenUi::getExistingDirectory(nullptr, "Choose Snapshots Directory", QStandardPaths::writableLocation(QStandardPaths::DesktopLocation));
diff --git a/interface/src/ui/Snapshot.h b/interface/src/ui/Snapshot.h
index 93aaed8aa4..606313f3c3 100644
--- a/interface/src/ui/Snapshot.h
+++ b/interface/src/ui/Snapshot.h
@@ -38,7 +38,7 @@ class Snapshot : public QObject, public Dependency {
Q_OBJECT
SINGLETON_DEPENDENCY
public:
- static QString saveSnapshot(QImage image, const QString& filename);
+ static QString saveSnapshot(QImage image, const QString& filename, const QString& pathname = QString());
static QTemporaryFile* saveTempSnapshot(QImage image);
static SnapshotMetaData* parseSnapshotData(QString snapshotPath);
@@ -52,7 +52,10 @@ public slots:
Q_INVOKABLE QString getSnapshotsLocation();
Q_INVOKABLE void setSnapshotsLocation(const QString& location);
private:
- static QFile* savedFileForSnapshot(QImage & image, bool isTemporary, const QString& userSelectedFilename = QString());
+ static QFile* savedFileForSnapshot(QImage& image,
+ bool isTemporary,
+ const QString& userSelectedFilename = QString(),
+ const QString& userSelectedPathname = QString());
};
#endif // hifi_Snapshot_h
diff --git a/libraries/script-engine/src/ScriptEngine.h b/libraries/script-engine/src/ScriptEngine.h
index 7a9af2278c..3001666b5d 100644
--- a/libraries/script-engine/src/ScriptEngine.h
+++ b/libraries/script-engine/src/ScriptEngine.h
@@ -526,6 +526,9 @@ public:
void setUserLoaded(bool isUserLoaded) { _isUserLoaded = isUserLoaded; }
bool isUserLoaded() const { return _isUserLoaded; }
+ void setQuitWhenFinished(const bool quitWhenFinished) { _quitWhenFinished = quitWhenFinished; }
+ bool isQuitWhenFinished() const { return _quitWhenFinished; }
+
// NOTE - this is used by the TypedArray implementation. we need to review this for thread safety
ArrayBufferClass* getArrayBufferClass() { return _arrayBufferClass; }
@@ -768,6 +771,8 @@ protected:
std::atomic<bool> _isUserLoaded { false };
bool _isReloading { false };
+ std::atomic<bool> _quitWhenFinished;
+
ArrayBufferClass* _arrayBufferClass;
AssetScriptingInterface* _assetScriptingInterface;
diff --git a/libraries/script-engine/src/ScriptEngines.cpp b/libraries/script-engine/src/ScriptEngines.cpp
index f2ed296b63..ad6e1debe9 100644
--- a/libraries/script-engine/src/ScriptEngines.cpp
+++ b/libraries/script-engine/src/ScriptEngines.cpp
@@ -347,7 +347,8 @@ void ScriptEngines::saveScripts() {
{
QReadLocker lock(&_scriptEnginesHashLock);
for (auto it = _scriptEnginesHash.begin(); it != _scriptEnginesHash.end(); ++it) {
- if (it.value() && it.value()->isUserLoaded()) {
+ // Save user-loaded scripts, only if they are set to quit when finished
+ if (it.value() && it.value()->isUserLoaded() && !it.value()->isQuitWhenFinished()) {
auto normalizedUrl = normalizeScriptURL(it.key());
list.append(normalizedUrl.toString());
}
@@ -456,7 +457,7 @@ void ScriptEngines::reloadAllScripts() {
}
ScriptEnginePointer ScriptEngines::loadScript(const QUrl& scriptFilename, bool isUserLoaded, bool loadScriptFromEditor,
- bool activateMainWindow, bool reload) {
+ bool activateMainWindow, bool reload, bool quitWhenFinished) {
if (thread() != QThread::currentThread()) {
ScriptEnginePointer result { nullptr };
BLOCKING_INVOKE_METHOD(this, "loadScript", Q_RETURN_ARG(ScriptEnginePointer, result),
@@ -488,6 +489,7 @@ ScriptEnginePointer ScriptEngines::loadScript(const QUrl& scriptFilename, bool i
scriptEngine = ScriptEnginePointer(new ScriptEngine(_context, NO_SCRIPT, "about:" + scriptFilename.fileName()));
addScriptEngine(scriptEngine);
scriptEngine->setUserLoaded(isUserLoaded);
+ scriptEngine->setQuitWhenFinished(quitWhenFinished);
if (scriptFilename.isEmpty() || !scriptUrl.isValid()) {
launchScriptEngine(scriptEngine);
@@ -496,6 +498,11 @@ ScriptEnginePointer ScriptEngines::loadScript(const QUrl& scriptFilename, bool i
connect(scriptEngine.data(), &ScriptEngine::scriptLoaded, this, &ScriptEngines::onScriptEngineLoaded);
connect(scriptEngine.data(), &ScriptEngine::errorLoadingScript, this, &ScriptEngines::onScriptEngineError);
+ // Shutdown Interface when script finishes, if requested
+ if (quitWhenFinished) {
+ connect(scriptEngine.data(), &ScriptEngine::finished, this, &ScriptEngines::quitWhenFinished);
+ }
+
// get the script engine object to load the script at the designated script URL
scriptEngine->loadURL(scriptUrl, reload);
}
@@ -536,6 +543,10 @@ void ScriptEngines::onScriptEngineLoaded(const QString& rawScriptURL) {
emit scriptCountChanged();
}
+void ScriptEngines::quitWhenFinished() {
+ qApp->quit();
+}
+
int ScriptEngines::runScriptInitializers(ScriptEnginePointer scriptEngine) {
int ii=0;
for (auto initializer : _scriptInitializers) {
diff --git a/libraries/script-engine/src/ScriptEngines.h b/libraries/script-engine/src/ScriptEngines.h
index 376bae4827..4d5964e462 100644
--- a/libraries/script-engine/src/ScriptEngines.h
+++ b/libraries/script-engine/src/ScriptEngines.h
@@ -88,10 +88,11 @@ public:
* @param {boolean} [loadScriptFromEditor=false]
* @param {boolean} [activateMainWindow=false]
* @param {boolean} [reload=false]
+ * @param {boolean} [quitWhenFinished=false]
* @returns {boolean}
*/
Q_INVOKABLE ScriptEnginePointer loadScript(const QUrl& scriptFilename = QString(),
- bool isUserLoaded = true, bool loadScriptFromEditor = false, bool activateMainWindow = false, bool reload = false);
+ bool isUserLoaded = true, bool loadScriptFromEditor = false, bool activateMainWindow = false, bool reload = false, bool quitWhenFinished = false);
/**jsdoc
* @function ScriptDiscoveryService.stopScript
@@ -266,6 +267,7 @@ protected:
ScriptEnginePointer reloadScript(const QString& scriptName, bool isUserLoaded = true) { return loadScript(scriptName, isUserLoaded, false, false, true); }
void removeScriptEngine(ScriptEnginePointer);
void onScriptEngineLoaded(const QString& scriptFilename);
+ void quitWhenFinished();
void onScriptEngineError(const QString& scriptFilename);
void launchScriptEngine(ScriptEnginePointer);
diff --git a/tools/auto-tester/src/Downloader.cpp b/tools/auto-tester/src/Downloader.cpp
index 030aa95a19..530a3b61bd 100644
--- a/tools/auto-tester/src/Downloader.cpp
+++ b/tools/auto-tester/src/Downloader.cpp
@@ -9,6 +9,8 @@
//
#include "Downloader.h"
+#include <QtWidgets/QMessageBox>
+
Downloader::Downloader(QUrl imageUrl, QObject *parent) : QObject(parent) {
connect(
&_networkAccessManager, SIGNAL (finished(QNetworkReply*)),
@@ -20,6 +22,12 @@ Downloader::Downloader(QUrl imageUrl, QObject *parent) : QObject(parent) {
}
void Downloader::fileDownloaded(QNetworkReply* reply) {
+ QNetworkReply::NetworkError error = reply->error();
+ if (error != QNetworkReply::NetworkError::NoError) {
+ QMessageBox::information(0, "Test Aborted", "Failed to download image: " + reply->errorString());
+ return;
+ }
+
_downloadedData = reply->readAll();
//emit a signal
diff --git a/tools/auto-tester/src/Test.cpp b/tools/auto-tester/src/Test.cpp
index 99f9025fdd..0eec03a782 100644
--- a/tools/auto-tester/src/Test.cpp
+++ b/tools/auto-tester/src/Test.cpp
@@ -27,7 +27,7 @@ Test::Test() {
mismatchWindow.setModal(true);
}
-bool Test::createTestResultsFolderPath(QString directory) {
+bool Test::createTestResultsFolderPath(const QString& directory) {
QDateTime now = QDateTime::currentDateTime();
testResultsFolderPath = directory + "/" + TEST_RESULTS_FOLDER + "--" + now.toString(DATETIME_FORMAT);
QDir testResultsFolder(testResultsFolderPath);
@@ -72,7 +72,7 @@ bool Test::compareImageLists(bool isInteractiveMode, QProgressBar* progressBar)
QImage expectedImage(expectedImagesFullFilenames[i]);
if (resultImage.width() != expectedImage.width() || resultImage.height() != expectedImage.height()) {
- messageBox.critical(0, "Internal error: " + QString(__FILE__) + ":" + QString::number(__LINE__), "Images are not the same size");
+ QMessageBox::critical(0, "Internal error: " + QString(__FILE__) + ":" + QString::number(__LINE__), "Images are not the same size");
exit(-1);
}
@@ -80,7 +80,7 @@ bool Test::compareImageLists(bool isInteractiveMode, QProgressBar* progressBar)
try {
similarityIndex = imageComparer.compareImages(resultImage, expectedImage);
} catch (...) {
- messageBox.critical(0, "Internal error: " + QString(__FILE__) + ":" + QString::number(__LINE__), "Image not in expected format");
+ QMessageBox::critical(0, "Internal error: " + QString(__FILE__) + ":" + QString::number(__LINE__), "Image not in expected format");
exit(-1);
}
@@ -125,22 +125,22 @@ bool Test::compareImageLists(bool isInteractiveMode, QProgressBar* progressBar)
return success;
}
-void Test::appendTestResultsToFile(QString testResultsFolderPath, TestFailure testFailure, QPixmap comparisonImage) {
+void Test::appendTestResultsToFile(const QString& testResultsFolderPath, TestFailure testFailure, QPixmap comparisonImage) {
if (!QDir().exists(testResultsFolderPath)) {
- messageBox.critical(0, "Internal error: " + QString(__FILE__) + ":" + QString::number(__LINE__), "Folder " + testResultsFolderPath + " not found");
+ QMessageBox::critical(0, "Internal error: " + QString(__FILE__) + ":" + QString::number(__LINE__), "Folder " + testResultsFolderPath + " not found");
exit(-1);
}
QString failureFolderPath { testResultsFolderPath + "/" + "Failure_" + QString::number(index) };
if (!QDir().mkdir(failureFolderPath)) {
- messageBox.critical(0, "Internal error: " + QString(__FILE__) + ":" + QString::number(__LINE__), "Failed to create folder " + failureFolderPath);
+ QMessageBox::critical(0, "Internal error: " + QString(__FILE__) + ":" + QString::number(__LINE__), "Failed to create folder " + failureFolderPath);
exit(-1);
}
++index;
QFile descriptionFile(failureFolderPath + "/" + TEST_RESULTS_FILENAME);
if (!descriptionFile.open(QIODevice::ReadWrite)) {
- messageBox.critical(0, "Internal error: " + QString(__FILE__) + ":" + QString::number(__LINE__), "Failed to create file " + TEST_RESULTS_FILENAME);
+ QMessageBox::critical(0, "Internal error: " + QString(__FILE__) + ":" + QString::number(__LINE__), "Failed to create file " + TEST_RESULTS_FILENAME);
exit(-1);
}
@@ -160,24 +160,30 @@ void Test::appendTestResultsToFile(QString testResultsFolderPath, TestFailure te
sourceFile = testFailure._pathname + testFailure._expectedImageFilename;
destinationFile = failureFolderPath + "/" + "Expected Image.jpg";
if (!QFile::copy(sourceFile, destinationFile)) {
- messageBox.critical(0, "Internal error: " + QString(__FILE__) + ":" + QString::number(__LINE__), "Failed to copy " + sourceFile + " to " + destinationFile);
+ QMessageBox::critical(0, "Internal error: " + QString(__FILE__) + ":" + QString::number(__LINE__), "Failed to copy " + sourceFile + " to " + destinationFile);
exit(-1);
}
sourceFile = testFailure._pathname + testFailure._actualImageFilename;
destinationFile = failureFolderPath + "/" + "Actual Image.jpg";
if (!QFile::copy(sourceFile, destinationFile)) {
- messageBox.critical(0, "Internal error: " + QString(__FILE__) + ":" + QString::number(__LINE__), "Failed to copy " + sourceFile + " to " + destinationFile);
+ QMessageBox::critical(0, "Internal error: " + QString(__FILE__) + ":" + QString::number(__LINE__), "Failed to copy " + sourceFile + " to " + destinationFile);
exit(-1);
}
comparisonImage.save(failureFolderPath + "/" + "Difference Image.jpg");
}
-void Test::startTestsEvaluation() {
- // Get list of JPEG images in folder, sorted by name
- pathToTestResultsDirectory = QFileDialog::getExistingDirectory(nullptr, "Please select folder containing the test images", ".", QFileDialog::ShowDirsOnly);
- if (pathToTestResultsDirectory == "") {
+void Test::startTestsEvaluation(const QString& testFolder) {
+ QString pathToTestResultsDirectory;
+ if (testFolder.isNull()) {
+ // Get list of JPEG images in folder, sorted by name
+ pathToTestResultsDirectory = QFileDialog::getExistingDirectory(nullptr, "Please select folder containing the test images", ".", QFileDialog::ShowDirsOnly);
+ } else {
+ pathToTestResultsDirectory = testFolder;
+ }
+
+ if (pathToTestResultsDirectory == QString()) {
return;
}
@@ -221,8 +227,8 @@ void Test::startTestsEvaluation() {
QString expectedImageFilenameTail = currentFilename.left(currentFilename.length() - 4).right(NUM_DIGITS);
QString expectedImageStoredFilename = EXPECTED_IMAGE_PREFIX + expectedImageFilenameTail + ".png";
- QString imageURLString("https://github.com/" + githubUser + "/hifi_tests/blob/" + gitHubBranch + "/" +
- expectedImagePartialSourceDirectory + "/" + expectedImageStoredFilename + "?raw=true");
+ QString imageURLString("https://raw.githubusercontent.com/" + githubUser + "/hifi_tests/" + gitHubBranch + "/" +
+ expectedImagePartialSourceDirectory + "/" + expectedImageStoredFilename);
expectedImagesURLs << imageURLString;
@@ -237,19 +243,21 @@ void Test::startTestsEvaluation() {
autoTester->downloadImages(expectedImagesURLs, pathToTestResultsDirectory, expectedImagesFilenames);
}
-void Test::finishTestsEvaluation(bool interactiveMode, QProgressBar* progressBar) {
- bool success = compareImageLists(interactiveMode, progressBar);
+void Test::finishTestsEvaluation(bool isRunningFromCommandline, bool interactiveMode, QProgressBar* progressBar) {
+ bool success = compareImageLists((!isRunningFromCommandline && interactiveMode), progressBar);
- if (success) {
- messageBox.information(0, "Success", "All images are as expected");
- } else {
- messageBox.information(0, "Failure", "One or more images are not as expected");
+ if (!isRunningFromCommandline) {
+ if (success) {
+ QMessageBox::information(0, "Success", "All images are as expected");
+ } else {
+ QMessageBox::information(0, "Failure", "One or more images are not as expected");
+ }
}
zipAndDeleteTestResultsFolder();
}
-bool Test::isAValidDirectory(QString pathname) {
+bool Test::isAValidDirectory(const QString& pathname) {
// Only process directories
QDir dir(pathname);
if (!dir.exists()) {
@@ -264,7 +272,7 @@ bool Test::isAValidDirectory(QString pathname) {
return true;
}
-QString Test::extractPathFromTestsDown(QString fullPath) {
+QString Test::extractPathFromTestsDown(const QString& fullPath) {
// `fullPath` includes the full path to the test. We need the portion below (and including) `tests`
QStringList pathParts = fullPath.split('/');
int i{ 0 };
@@ -273,7 +281,7 @@ QString Test::extractPathFromTestsDown(QString fullPath) {
}
if (i == pathParts.length()) {
- messageBox.critical(0, "Internal error: " + QString(__FILE__) + ":" + QString::number(__LINE__), "Bad testPathname");
+ QMessageBox::critical(0, "Internal error: " + QString(__FILE__) + ":" + QString::number(__LINE__), "Bad testPathname");
exit(-1);
}
@@ -342,14 +350,14 @@ void Test::createAllRecursiveScripts() {
}
}
- messageBox.information(0, "Success", "Scripts have been created");
+ QMessageBox::information(0, "Success", "Scripts have been created");
}
-void Test::createRecursiveScript(QString topLevelDirectory, bool interactiveMode) {
+void Test::createRecursiveScript(const QString& topLevelDirectory, bool interactiveMode) {
const QString recursiveTestsFilename("testRecursive.js");
QFile allTestsFilename(topLevelDirectory + "/" + recursiveTestsFilename);
if (!allTestsFilename.open(QIODevice::WriteOnly | QIODevice::Text)) {
- messageBox.critical(0,
+ QMessageBox::critical(0,
"Internal error: " + QString(__FILE__) + ":" + QString::number(__LINE__),
"Failed to create \"" + recursiveTestsFilename + "\" in directory \"" + topLevelDirectory + "\""
);
@@ -358,12 +366,15 @@ void Test::createRecursiveScript(QString topLevelDirectory, bool interactiveMode
}
QTextStream textStream(&allTestsFilename);
- textStream << "// This is an automatically generated file, created by auto-tester" << endl << endl;
+
+ const QString DATE_TIME_FORMAT("MMM d yyyy, h:mm");
+ textStream << "// This is an automatically generated file, created by auto-tester on " << QDateTime::currentDateTime().toString(DATE_TIME_FORMAT) << endl << endl;
textStream << "var autoTester = Script.require(\"https://github.com/" + githubUser + "/hifi_tests/blob/"
- + gitHubBranch + "/tests/utils/autoTester.js?raw=true\");" << endl;
+ + gitHubBranch + "/tests/utils/autoTester.js?raw=true\");" << endl << endl;
- textStream << "autoTester.enableRecursive();" << endl << endl;
+ textStream << "autoTester.enableRecursive();" << endl;
+ textStream << "autoTester.enableAuto();" << endl << endl;
QVector<QString> testPathnames;
@@ -398,7 +409,7 @@ void Test::createRecursiveScript(QString topLevelDirectory, bool interactiveMode
}
if (interactiveMode && testPathnames.length() <= 0) {
- messageBox.information(0, "Failure", "No \"" + TEST_FILENAME + "\" files found");
+ QMessageBox::information(0, "Failure", "No \"" + TEST_FILENAME + "\" files found");
allTestsFilename.close();
return;
}
@@ -409,7 +420,7 @@ void Test::createRecursiveScript(QString topLevelDirectory, bool interactiveMode
allTestsFilename.close();
if (interactiveMode) {
- messageBox.information(0, "Success", "Script has been created");
+ QMessageBox::information(0, "Success", "Script has been created");
}
}
@@ -434,7 +445,7 @@ void Test::createTest() {
QString fullCurrentFilename = imageSourceDirectory + "/" + currentFilename;
if (isInSnapshotFilenameFormat("jpg", currentFilename)) {
if (i >= maxImages) {
- messageBox.critical(0, "Error", "More than " + QString::number(maxImages) + " images not supported");
+ QMessageBox::critical(0, "Error", "More than " + QString::number(maxImages) + " images not supported");
exit(-1);
}
QString newFilename = "ExpectedImage_" + QString::number(i - 1).rightJustified(5, '0') + ".png";
@@ -443,14 +454,14 @@ void Test::createTest() {
try {
copyJPGtoPNG(fullCurrentFilename, fullNewFileName);
} catch (...) {
- messageBox.critical(0, "Error", "Could not delete existing file: " + currentFilename + "\nTest creation aborted");
+ QMessageBox::critical(0, "Error", "Could not delete existing file: " + currentFilename + "\nTest creation aborted");
exit(-1);
}
++i;
}
}
- messageBox.information(0, "Success", "Test images have been created");
+ QMessageBox::information(0, "Success", "Test images have been created");
}
ExtractedText Test::getTestScriptLines(QString testFileName) {
@@ -459,7 +470,7 @@ ExtractedText Test::getTestScriptLines(QString testFileName) {
QFile inputFile(testFileName);
inputFile.open(QIODevice::ReadOnly);
if (!inputFile.isOpen()) {
- messageBox.critical(0,
+ QMessageBox::critical(0,
"Internal error: " + QString(__FILE__) + ":" + QString::number(__LINE__),
"Failed to open \"" + testFileName
);
@@ -498,6 +509,12 @@ ExtractedText Test::getTestScriptLines(QString testFileName) {
const QString regexAssertGPU(ws + functionAssertGPU + ws + "\\(" + ws + quotedString + ".*");
const QRegularExpression lineAssertGPU = QRegularExpression(regexAssertGPU);
+ // Assert the correct amount of memory
+ const QString functionAssertPhysicalMemoryGB(ws + "autoTester" + ws + "\\." + ws + "assertPhysicalMemoryGB");
+ const QString regexAssertPhysicalMemoryGB(ws + functionAssertPhysicalMemoryGB + ws + "\\(" + ws + quotedString + ".*");
+ const QRegularExpression lineAssertPhysicalMemoryGB = QRegularExpression(regexAssertPhysicalMemoryGB);
+
+
// Each step is either of the following forms:
// autoTester.addStepSnapshot("Take snapshot"...
// autoTester.addStep("Clean up after test"...
@@ -514,18 +531,7 @@ ExtractedText Test::getTestScriptLines(QString testFileName) {
if (lineContainingTitle.match(line).hasMatch()) {
QStringList tokens = line.split('"');
relevantTextFromTest.title = tokens[1];
- } else if (lineAssertPlatform.match(line).hasMatch()) {
- QStringList platforms = line.split('"');
- relevantTextFromTest.platform = platforms[1];
- } else if (lineAssertDisplay.match(line).hasMatch()) {
- QStringList displays = line.split('"');
- relevantTextFromTest.display = displays[1];
- } else if (lineAssertCPU.match(line).hasMatch()) {
- QStringList cpus = line.split('"');
- relevantTextFromTest.cpu = cpus[1];
- } else if (lineAssertGPU.match(line).hasMatch()) {
- QStringList gpus = line.split('"');
- relevantTextFromTest.gpu = gpus[1];
+
} else if (lineStepSnapshot.match(line).hasMatch()) {
QStringList tokens = line.split('"');
QString nameOfStep = tokens[1];
@@ -534,6 +540,7 @@ ExtractedText Test::getTestScriptLines(QString testFileName) {
step->text = nameOfStep;
step->takeSnapshot = true;
relevantTextFromTest.stepList.emplace_back(step);
+
} else if (lineStep.match(line).hasMatch()) {
QStringList tokens = line.split('"');
QString nameOfStep = tokens[1];
@@ -593,15 +600,15 @@ void Test::createAllMDFiles() {
}
}
- messageBox.information(0, "Success", "MD files have been created");
+ QMessageBox::information(0, "Success", "MD files have been created");
}
-void Test::createMDFile(QString testDirectory) {
+void Test::createMDFile(const QString& testDirectory) {
// Verify folder contains test.js file
QString testFileName(testDirectory + "/" + TEST_FILENAME);
QFileInfo testFileInfo(testFileName);
if (!testFileInfo.exists()) {
- messageBox.critical(0, "Error", "Could not find file: " + TEST_FILENAME);
+ QMessageBox::critical(0, "Error", "Could not find file: " + TEST_FILENAME);
return;
}
@@ -610,7 +617,7 @@ void Test::createMDFile(QString testDirectory) {
QString mdFilename(testDirectory + "/" + "test.md");
QFile mdFile(mdFilename);
if (!mdFile.open(QIODevice::WriteOnly)) {
- messageBox.critical(0, "Internal error: " + QString(__FILE__) + ":" + QString::number(__LINE__), "Failed to create file " + mdFilename);
+ QMessageBox::critical(0, "Internal error: " + QString(__FILE__) + ":" + QString::number(__LINE__), "Failed to create file " + mdFilename);
exit(-1);
}
@@ -628,64 +635,24 @@ void Test::createMDFile(QString testDirectory) {
stream << "## Preconditions" << "\n";
stream << "- In an empty region of a domain with editing rights." << "\n\n";
- // Platform
- QStringList platforms = testScriptLines.platform.split(" ");;
- stream << "## Platforms\n";
- stream << "Run the test on each of the following platforms\n";
- for (int i = 0; i < platforms.size(); ++i) {
- // Note that the platforms parameter may include extra spaces, these appear as empty strings in the list
- if (platforms[i] != QString()) {
- stream << " - " << platforms[i] << "\n";
- }
- }
-
- // Display
- QStringList displays = testScriptLines.display.split(" ");
- stream << "## Displays\n";
- stream << "Run the test on each of the following displays\n";
- for (int i = 0; i < displays.size(); ++i) {
- // Note that the displays parameter may include extra spaces, these appear as empty strings in the list
- if (displays[i] != QString()) {
- stream << " - " << displays[i] << "\n";
- }
- }
-
- // CPU
- QStringList cpus = testScriptLines.cpu.split(" ");
- stream << "## Processors\n";
- stream << "Run the test on each of the following processors\n";
- for (int i = 0; i < cpus.size(); ++i) {
- // Note that the cpus parameter may include extra spaces, these appear as empty strings in the list
- if (cpus[i] != QString()) {
- stream << " - " << cpus[i] << "\n";
- }
- }
-
- // GPU
- QStringList gpus = testScriptLines.gpu.split(" ");
- stream << "## Graphics Cards\n";
- stream << "Run the test on graphics cards from each of the following vendors\n";
- for (int i = 0; i < gpus.size(); ++i) {
- // Note that the gpus parameter may include extra spaces, these appear as empty strings in the list
- if (gpus[i] != QString()) {
- stream << " - " << gpus[i] << "\n";
- }
- }
-
stream << "## Steps\n";
stream << "Press space bar to advance step by step\n\n";
+ // Note that snapshots of step n are taken in step n+1
+ // (this implies that if the LAST step requests a snapshot then this will not work - caveat emptor)
int snapShotIndex { 0 };
for (size_t i = 0; i < testScriptLines.stepList.size(); ++i) {
stream << "### Step " << QString::number(i + 1) << "\n";
stream << "- " << testScriptLines.stepList[i]->text << "\n";
- if (testScriptLines.stepList[i]->takeSnapshot) {
+ if ((i + 1 < testScriptLines.stepList.size()) && testScriptLines.stepList[i + 1]->takeSnapshot) {
stream << "- ![](./ExpectedImage_" << QString::number(snapShotIndex).rightJustified(5, '0') << ".png)\n";
++snapShotIndex;
}
}
mdFile.close();
+
+ QMessageBox::information(0, "Success", "Test MD file " + mdFilename + " has been created");
}
void Test::createTestsOutline() {
@@ -698,7 +665,7 @@ void Test::createTestsOutline() {
QString mdFilename(testsRootDirectory + "/" + testsOutlineFilename);
QFile mdFile(mdFilename);
if (!mdFile.open(QIODevice::WriteOnly)) {
- messageBox.critical(0, "Internal error: " + QString(__FILE__) + ":" + QString::number(__LINE__), "Failed to create file " + mdFilename);
+ QMessageBox::critical(0, "Internal error: " + QString(__FILE__) + ":" + QString::number(__LINE__), "Failed to create file " + mdFilename);
exit(-1);
}
@@ -756,10 +723,10 @@ void Test::createTestsOutline() {
mdFile.close();
- messageBox.information(0, "Success", "Test outline file " + testsOutlineFilename + " has been created");
+ QMessageBox::information(0, "Success", "Test outline file " + testsOutlineFilename + " has been created");
}
-void Test::copyJPGtoPNG(QString sourceJPGFullFilename, QString destinationPNGFullFilename) {
+void Test::copyJPGtoPNG(const QString& sourceJPGFullFilename, const QString& destinationPNGFullFilename) {
QFile::remove(destinationPNGFullFilename);
QImageReader reader;
@@ -772,7 +739,7 @@ void Test::copyJPGtoPNG(QString sourceJPGFullFilename, QString destinationPNGFul
writer.write(image);
}
-QStringList Test::createListOfAll_imagesInDirectory(QString imageFormat, QString pathToImageDirectory) {
+QStringList Test::createListOfAll_imagesInDirectory(const QString& imageFormat, const QString& pathToImageDirectory) {
imageDirectory = QDir(pathToImageDirectory);
QStringList nameFilters;
nameFilters << "*." + imageFormat;
@@ -785,7 +752,7 @@ QStringList Test::createListOfAll_imagesInDirectory(QString imageFormat, QString
// Filename (i.e. without extension) contains _tests_ (this is based on all test scripts being within the tests folder
// Last 5 characters in filename are digits
// Extension is jpg
-bool Test::isInSnapshotFilenameFormat(QString imageFormat, QString filename) {
+bool Test::isInSnapshotFilenameFormat(const QString& imageFormat, const QString& filename) {
QStringList filenameParts = filename.split(".");
bool filnameHasNoPeriods = (filenameParts.size() == 2);
@@ -802,7 +769,7 @@ bool Test::isInSnapshotFilenameFormat(QString imageFormat, QString filename) {
// For a file named "D_GitHub_hifi-tests_tests_content_entity_zone_create_0.jpg", the test directory is
// D:/GitHub/hifi-tests/tests/content/entity/zone/create
// This method assumes the filename is in the correct format
-QString Test::getExpectedImageDestinationDirectory(QString filename) {
+QString Test::getExpectedImageDestinationDirectory(const QString& filename) {
QString filenameWithoutExtension = filename.split(".")[0];
QStringList filenameParts = filenameWithoutExtension.split("_");
@@ -819,7 +786,7 @@ QString Test::getExpectedImageDestinationDirectory(QString filename) {
// is ...tests/content/entity/zone/create
// This is used to create the full URL
// This method assumes the filename is in the correct format
-QString Test::getExpectedImagePartialSourceDirectory(QString filename) {
+QString Test::getExpectedImagePartialSourceDirectory(const QString& filename) {
QString filenameWithoutExtension = filename.split(".")[0];
QStringList filenameParts = filenameWithoutExtension.split("_");
@@ -831,7 +798,7 @@ QString Test::getExpectedImagePartialSourceDirectory(QString filename) {
}
if (i < 0) {
- messageBox.critical(0, "Internal error: " + QString(__FILE__) + ":" + QString::number(__LINE__), "Bad filename");
+ QMessageBox::critical(0, "Internal error: " + QString(__FILE__) + ":" + QString::number(__LINE__), "Bad filename");
exit(-1);
}
diff --git a/tools/auto-tester/src/Test.h b/tools/auto-tester/src/Test.h
index e69459fef2..bc28d6ad0a 100644
--- a/tools/auto-tester/src/Test.h
+++ b/tools/auto-tester/src/Test.h
@@ -30,10 +30,6 @@ using StepList = std::vector<Step*>;
class ExtractedText {
public:
QString title;
- QString platform;
- QString display;
- QString cpu;
- QString gpu;
StepList stepList;
};
@@ -41,61 +37,58 @@ class Test {
public:
Test();
- void startTestsEvaluation();
- void finishTestsEvaluation(bool interactiveMode, QProgressBar* progressBar);
+ void startTestsEvaluation(const QString& testFolder = QString());
+ void finishTestsEvaluation(bool isRunningFromCommandline, bool interactiveMode, QProgressBar* progressBar);
void createRecursiveScript();
void createAllRecursiveScripts();
- void createRecursiveScript(QString topLevelDirectory, bool interactiveMode);
+ void createRecursiveScript(const QString& topLevelDirectory, bool interactiveMode);
void createTest();
void createMDFile();
void createAllMDFiles();
- void createMDFile(QString topLevelDirectory);
+ void createMDFile(const QString& topLevelDirectory);
void createTestsOutline();
bool compareImageLists(bool isInteractiveMode, QProgressBar* progressBar);
- QStringList createListOfAll_imagesInDirectory(QString imageFormat, QString pathToImageDirectory);
+ QStringList createListOfAll_imagesInDirectory(const QString& imageFormat, const QString& pathToImageDirectory);
- bool isInSnapshotFilenameFormat(QString imageFormat, QString filename);
+ bool isInSnapshotFilenameFormat(const QString& imageFormat, const QString& filename);
void importTest(QTextStream& textStream, const QString& testPathname);
- void appendTestResultsToFile(QString testResultsFolderPath, TestFailure testFailure, QPixmap comparisonImage);
+ void appendTestResultsToFile(const QString& testResultsFolderPath, TestFailure testFailure, QPixmap comparisonImage);
- bool createTestResultsFolderPath(QString directory);
+ bool createTestResultsFolderPath(const QString& directory);
void zipAndDeleteTestResultsFolder();
- bool isAValidDirectory(QString pathname);
- QString extractPathFromTestsDown(QString fullPath);
- QString getExpectedImageDestinationDirectory(QString filename);
- QString getExpectedImagePartialSourceDirectory(QString filename);
+ bool isAValidDirectory(const QString& pathname);
+ QString extractPathFromTestsDown(const QString& fullPath);
+ QString getExpectedImageDestinationDirectory(const QString& filename);
+ QString getExpectedImagePartialSourceDirectory(const QString& filename);
- void copyJPGtoPNG(QString sourceJPGFullFilename, QString destinationPNGFullFilename);
+ void copyJPGtoPNG(const QString& sourceJPGFullFilename, const QString& destinationPNGFullFilename);
private:
const QString TEST_FILENAME { "test.js" };
const QString TEST_RESULTS_FOLDER { "TestResults" };
const QString TEST_RESULTS_FILENAME { "TestResults.txt" };
- QMessageBox messageBox;
-
QDir imageDirectory;
MismatchWindow mismatchWindow;
ImageComparer imageComparer;
- QString testResultsFolderPath { "" };
+ QString testResultsFolderPath;
int index { 1 };
// Expected images are in the format ExpectedImage_dddd.jpg (d == decimal digit)
const int NUM_DIGITS { 5 };
const QString EXPECTED_IMAGE_PREFIX { "ExpectedImage_" };
- QString pathToTestResultsDirectory;
QStringList expectedImagesFilenames;
QStringList expectedImagesFullFilenames;
QStringList resultImagesFullFilenames;
diff --git a/tools/auto-tester/src/main.cpp b/tools/auto-tester/src/main.cpp
index cd0ce22b13..ffa7a0b237 100644
--- a/tools/auto-tester/src/main.cpp
+++ b/tools/auto-tester/src/main.cpp
@@ -13,10 +13,23 @@
AutoTester* autoTester;
int main(int argc, char *argv[]) {
+ // Only parameter is "--testFolder"
+ QString testFolder;
+ if (argc == 3) {
+ if (QString(argv[1]) == "--testFolder") {
+ testFolder = QString(argv[2]);
+ }
+ }
+
QApplication application(argc, argv);
autoTester = new AutoTester();
- autoTester->show();
+
+ if (!testFolder.isNull()) {
+ autoTester->runFromCommandLine(testFolder);
+ } else {
+ autoTester->show();
+ }
return application.exec();
}
diff --git a/tools/auto-tester/src/ui/AutoTester.cpp b/tools/auto-tester/src/ui/AutoTester.cpp
index 21acfe9569..db9974a250 100644
--- a/tools/auto-tester/src/ui/AutoTester.cpp
+++ b/tools/auto-tester/src/ui/AutoTester.cpp
@@ -15,9 +15,17 @@ AutoTester::AutoTester(QWidget *parent) : QMainWindow(parent) {
ui.checkBoxInteractiveMode->setChecked(true);
ui.progressBar->setVisible(false);
- test = new Test();
-
signalMapper = new QSignalMapper();
+
+ connect(ui.actionClose, &QAction::triggered, this, &AutoTester::on_closeButton_clicked);
+ connect(ui.actionAbout, &QAction::triggered, this, &AutoTester::about);
+
+ test = new Test();
+}
+
+void AutoTester::runFromCommandLine(const QString& testFolder) {
+ isRunningFromCommandline = true;
+ test->startTestsEvaluation(testFolder);
}
void AutoTester::on_evaluateTestsButton_clicked() {
@@ -90,13 +98,21 @@ void AutoTester::saveImage(int index) {
image = image.convertToFormat(QImage::Format_ARGB32);
QString fullPathname = _directoryName + "/" + _filenames[index];
- image.save(fullPathname, 0, 100);
+ if (!image.save(fullPathname, 0, 100)) {
+ QMessageBox::information(0, "Test Aborted", "Failed to save image: " + _filenames[index]);
+ ui.progressBar->setVisible(false);
+ return;
+ }
++_numberOfImagesDownloaded;
if (_numberOfImagesDownloaded == _numberOfImagesToDownload) {
- test->finishTestsEvaluation(ui.checkBoxInteractiveMode->isChecked(), ui.progressBar);
+ test->finishTestsEvaluation(isRunningFromCommandline, ui.checkBoxInteractiveMode->isChecked(), ui.progressBar);
} else {
ui.progressBar->setValue(_numberOfImagesDownloaded);
}
}
+
+void AutoTester::about() {
+ QMessageBox::information(0, "About", QString("Built ") + __DATE__ + " : " + __TIME__);
+}
diff --git a/tools/auto-tester/src/ui/AutoTester.h b/tools/auto-tester/src/ui/AutoTester.h
index 1788e97177..fe37f2298d 100644
--- a/tools/auto-tester/src/ui/AutoTester.h
+++ b/tools/auto-tester/src/ui/AutoTester.h
@@ -22,6 +22,9 @@ class AutoTester : public QMainWindow {
public:
AutoTester(QWidget *parent = Q_NULLPTR);
+
+ void runFromCommandLine(const QString& testFolder);
+
void downloadImage(const QUrl& url);
void downloadImages(const QStringList& URLs, const QString& directoryName, const QStringList& filenames);
@@ -37,6 +40,8 @@ private slots:
void saveImage(int index);
+ void about();
+
private:
Ui::AutoTesterClass ui;
Test* test;
@@ -50,9 +55,11 @@ private:
// Used to enable passing a parameter to slots
QSignalMapper* signalMapper;
- int _numberOfImagesToDownload;
- int _numberOfImagesDownloaded;
- int _index;
+ int _numberOfImagesToDownload { 0 };
+ int _numberOfImagesDownloaded { 0 };
+ int _index { 0 };
+
+ bool isRunningFromCommandline { false };
};
#endif // hifi_AutoTester_h
\ No newline at end of file
diff --git a/tools/auto-tester/src/ui/AutoTester.ui b/tools/auto-tester/src/ui/AutoTester.ui
index 2eb1314481..8c534eb7c7 100644
--- a/tools/auto-tester/src/ui/AutoTester.ui
+++ b/tools/auto-tester/src/ui/AutoTester.ui
@@ -95,7 +95,7 @@
<number>24</number>
</property>
</widget>
- <widget class="QPushButton" name="createRecursiveScriptsRecursivelyButton">
+ <widget class="QPushButton" name="createAllRecursiveScriptsButton">
<property name="geometry">
<rect>
<x>360</x>
@@ -157,6 +157,20 @@
<height>21</height>
</rect>
</property>
+ <widget class="QMenu" name="menuFile">
+ <property name="title">
+ <string>File</string>
+ </property>
+ <addaction name="actionClose"/>
+ </widget>
+ <widget class="QMenu" name="menuHelp">
+ <property name="title">
+ <string>Help</string>
+ </property>
+ <addaction name="actionAbout"/>
+ </widget>
+ <addaction name="menuFile"/>
+ <addaction name="menuHelp"/>
</widget>
<widget class="QToolBar" name="mainToolBar">
<attribute name="toolBarArea">
@@ -167,6 +181,16 @@
</attribute>
</widget>
<widget class="QStatusBar" name="statusBar"/>
+ <action name="actionClose">
+ <property name="text">
+ <string>Close</string>
+ </property>
+ </action>
+ <action name="actionAbout">
+ <property name="text">
+ <string>About</string>
+ </property>
+ </action>
</widget>
<layoutdefault spacing="6" margin="11"/>
<resources/>