Mirrors/overte
3
0
Fork 0
mirror of https://github.com/overte-org/overte.git synced 2025-04-08 11:54:20 +02:00
Code Issues Projects Releases Packages Wiki Activity Actions 2

Merge pull request #13046 from NissimHadar/commandLineParametersForAutotester

Browse source 
Command line parameters for autotester
This commit is contained in:
Sam Gateau 2018-05-09 10:51:31 -07:00 committed by GitHub
commit 17350a79dd
No known key found for this signature in database
GPG key ID: 4AEE18F83AFDEB23
14 changed files with 256 additions and 151 deletions
Show stats Download patch file Download diff file Expand all files Collapse all files

61
interface/src/Application.cpp
View file

@ -743,6 +743,11 @@ extern DisplayPluginList getDisplayPlugins();
extern InputPluginList getInputPlugins();
extern void saveInputPluginSettings(const InputPluginList& plugins);
// Parameters used for running tests from teh command line
const QString TEST_SCRIPT_COMMAND { "--testScript" };
const QString TEST_QUIT_WHEN_FINISHED_OPTION { "quitWhenFinished" };
const QString TEST_SNAPSHOT_LOCATION_COMMAND { "--testSnapshotLocation" };
bool setupEssentials(int& argc, char** argv, bool runningMarkerExisted) {
const char** constArgv = const_cast<const char**>(argv);
@ -777,7 +782,22 @@ bool setupEssentials(int& argc, char** argv, bool runningMarkerExisted) {
static const auto SUPPRESS_SETTINGS_RESET = "--suppress-settings-reset";
bool suppressPrompt = cmdOptionExists(argc, const_cast<const char**>(argv), SUPPRESS_SETTINGS_RESET);
bool previousSessionCrashed = CrashHandler::checkForResetSettings(runningMarkerExisted, suppressPrompt);
// Ignore any previous crashes if running from command line with a test script.
bool inTestMode { false };
for (int i = 0; i < argc; ++i) {
QString parameter(argv[i]);
if (parameter == TEST_SCRIPT_COMMAND) {
inTestMode = true;
break;
}
}
bool previousSessionCrashed { false };
if (!inTestMode) {
previousSessionCrashed = CrashHandler::checkForResetSettings(runningMarkerExisted, suppressPrompt);
}
// get dir to use for cache
static const auto CACHE_SWITCH = "--cache";
QString cacheDir = getCmdOption(argc, const_cast<const char**>(argv), CACHE_SWITCH);
@ -996,13 +1016,30 @@ Application::Application(int& argc, char** argv, QElapsedTimer& startupTimer, bo
setProperty(hifi::properties::STEAM, (steamClient && steamClient->isRunning()));
setProperty(hifi::properties::CRASHED, _previousSessionCrashed);
{
const QString TEST_SCRIPT = "--testScript";
const QStringList args = arguments();
for (int i = 0; i < args.size() - 1; ++i) {
if (args.at(i) == TEST_SCRIPT) {
if (args.at(i) == TEST_SCRIPT_COMMAND && (i + 1) < args.size()) {
QString testScriptPath = args.at(i + 1);
if (QFileInfo(testScriptPath).exists()) {
// If the URL scheme is http(s) or ftp, then use as is, else - treat it as a local file
// This is done so as not break previous command line scripts
if (testScriptPath.left(URL_SCHEME_HTTP.length()) == URL_SCHEME_HTTP || testScriptPath.left(URL_SCHEME_FTP.length()) == URL_SCHEME_FTP) {
setProperty(hifi::properties::TEST, QUrl::fromUserInput(testScriptPath));
} else if (QFileInfo(testScriptPath).exists()) {
setProperty(hifi::properties::TEST, QUrl::fromLocalFile(testScriptPath));
}
// quite when finished parameter must directly follow the test script
if ((i + 2) < args.size() && args.at(i + 2) == TEST_QUIT_WHEN_FINISHED_OPTION) {
quitWhenFinished = true;
}
} else if (args.at(i) == TEST_SNAPSHOT_LOCATION_COMMAND) {
// Set test snapshot location only if it is a writeable directory
QString pathname(args.at(i + 1));
QFileInfo fileInfo(pathname);
if (fileInfo.isDir() && fileInfo.isWritable()) {
testSnapshotLocation = pathname;
}
}
}
@ -2135,7 +2172,17 @@ Application::Application(int& argc, char** argv, QElapsedTimer& startupTimer, bo
if (testProperty.isValid()) {
auto scriptEngines = DependencyManager::get<ScriptEngines>();
const auto testScript = property(hifi::properties::TEST).toUrl();
scriptEngines->loadScript(testScript, false);
// Set last parameter to exit interface when the test script finishes, if so requested
scriptEngines->loadScript(testScript, false, false, false, false, quitWhenFinished);
// This is done so we don't get a "connection time-out" message when we haven't passed in a URL.
if (arguments().contains("--url")) {
auto reply = SandboxUtils::getStatus();
connect(reply, &QNetworkReply::finished, this, [=] {
handleSandboxStatus(reply);
});
}
} else {
PROFILE_RANGE(render, "GetSandboxStatus");
auto reply = SandboxUtils::getStatus();
@ -7465,7 +7512,7 @@ void Application::loadAvatarBrowser() const {
void Application::takeSnapshot(bool notify, bool includeAnimated, float aspectRatio, const QString& filename) {
postLambdaEvent([notify, includeAnimated, aspectRatio, filename, this] {
// Get a screenshot and save it
QString path = Snapshot::saveSnapshot(getActiveDisplayPlugin()->getScreenshot(aspectRatio), filename);
QString path = Snapshot::saveSnapshot(getActiveDisplayPlugin()->getScreenshot(aspectRatio), filename, testSnapshotLocation);
// If we're not doing an animated snapshot as well...
if (!includeAnimated) {
// Tell the dependency manager that the capture of the still snapshot has taken place.
@ -7479,7 +7526,7 @@ void Application::takeSnapshot(bool notify, bool includeAnimated, float aspectRa
void Application::takeSecondaryCameraSnapshot(const QString& filename) {
postLambdaEvent([filename, this] {
QString snapshotPath = Snapshot::saveSnapshot(getActiveDisplayPlugin()->getSecondaryCameraScreenshot(), filename);
QString snapshotPath = Snapshot::saveSnapshot(getActiveDisplayPlugin()->getSecondaryCameraScreenshot(), filename, testSnapshotLocation);
emit DependencyManager::get<WindowScriptingInterface>()->stillSnapshotTaken(snapshotPath, true);
});
}

3
interface/src/Application.h
View file

@ -750,5 +750,8 @@ private:
std::atomic<bool> _pendingIdleEvent { true };
std::atomic<bool> _pendingRenderEvent { true };
QString testSnapshotLocation;
bool quitWhenFinished { false };
};
#endif // hifi_Application_h

14
interface/src/ui/Snapshot.cpp
View file

@ -74,9 +74,9 @@ SnapshotMetaData* Snapshot::parseSnapshotData(QString snapshotPath) {
return data;
}
QString Snapshot::saveSnapshot(QImage image, const QString& filename) {
QString Snapshot::saveSnapshot(QImage image, const QString& filename, const QString& pathname) {
QFile* snapshotFile = savedFileForSnapshot(image, false, filename);
QFile* snapshotFile = savedFileForSnapshot(image, false, filename, pathname);
// we don't need the snapshot file, so close it, grab its filename and delete it
snapshotFile->close();
@ -93,7 +93,7 @@ QTemporaryFile* Snapshot::saveTempSnapshot(QImage image) {
return static_cast<QTemporaryFile*>(savedFileForSnapshot(image, true));
}
QFile* Snapshot::savedFileForSnapshot(QImage & shot, bool isTemporary, const QString& userSelectedFilename) {
QFile* Snapshot::savedFileForSnapshot(QImage & shot, bool isTemporary, const QString& userSelectedFilename, const QString& userSelectedPathname) {
// adding URL to snapshot
QUrl currentURL = DependencyManager::get<AddressManager>()->currentPublicAddress();
@ -118,7 +118,13 @@ QFile* Snapshot::savedFileForSnapshot(QImage & shot, bool isTemporary, const QSt
const int IMAGE_QUALITY = 100;
if (!isTemporary) {
QString snapshotFullPath = snapshotsLocation.get();
// If user has requested specific path then use it, else use the application value
QString snapshotFullPath;
if (!userSelectedPathname.isNull()) {
snapshotFullPath = userSelectedPathname;
} else {
snapshotFullPath = snapshotsLocation.get();
}
if (snapshotFullPath.isEmpty()) {
snapshotFullPath = OffscreenUi::getExistingDirectory(nullptr, "Choose Snapshots Directory", QStandardPaths::writableLocation(QStandardPaths::DesktopLocation));

7
interface/src/ui/Snapshot.h
View file

@ -38,7 +38,7 @@ class Snapshot : public QObject, public Dependency {
Q_OBJECT
SINGLETON_DEPENDENCY
public:
static QString saveSnapshot(QImage image, const QString& filename);
static QString saveSnapshot(QImage image, const QString& filename, const QString& pathname = QString());
static QTemporaryFile* saveTempSnapshot(QImage image);
static SnapshotMetaData* parseSnapshotData(QString snapshotPath);
@ -52,7 +52,10 @@ public slots:
Q_INVOKABLE QString getSnapshotsLocation();
Q_INVOKABLE void setSnapshotsLocation(const QString& location);
private:
static QFile* savedFileForSnapshot(QImage & image, bool isTemporary, const QString& userSelectedFilename = QString());
static QFile* savedFileForSnapshot(QImage& image,
bool isTemporary,
const QString& userSelectedFilename = QString(),
const QString& userSelectedPathname = QString());
};
#endif // hifi_Snapshot_h

5
libraries/script-engine/src/ScriptEngine.h
View file

@ -526,6 +526,9 @@ public:
void setUserLoaded(bool isUserLoaded) { _isUserLoaded = isUserLoaded; }
bool isUserLoaded() const { return _isUserLoaded; }
void setQuitWhenFinished(const bool quitWhenFinished) { _quitWhenFinished = quitWhenFinished; }
bool isQuitWhenFinished() const { return _quitWhenFinished; }
// NOTE - this is used by the TypedArray implementation. we need to review this for thread safety
ArrayBufferClass* getArrayBufferClass() { return _arrayBufferClass; }
@ -768,6 +771,8 @@ protected:
std::atomic<bool> _isUserLoaded { false };
bool _isReloading { false };
std::atomic<bool> _quitWhenFinished;
ArrayBufferClass* _arrayBufferClass;
AssetScriptingInterface* _assetScriptingInterface;

15
libraries/script-engine/src/ScriptEngines.cpp
View file

@ -347,7 +347,8 @@ void ScriptEngines::saveScripts() {
{
QReadLocker lock(&_scriptEnginesHashLock);
for (auto it = _scriptEnginesHash.begin(); it != _scriptEnginesHash.end(); ++it) {
if (it.value() && it.value()->isUserLoaded()) {
// Save user-loaded scripts, only if they are set to quit when finished
if (it.value() && it.value()->isUserLoaded() && !it.value()->isQuitWhenFinished()) {
auto normalizedUrl = normalizeScriptURL(it.key());
list.append(normalizedUrl.toString());
}
@ -456,7 +457,7 @@ void ScriptEngines::reloadAllScripts() {
}
ScriptEnginePointer ScriptEngines::loadScript(const QUrl& scriptFilename, bool isUserLoaded, bool loadScriptFromEditor,
bool activateMainWindow, bool reload) {
bool activateMainWindow, bool reload, bool quitWhenFinished) {
if (thread() != QThread::currentThread()) {
ScriptEnginePointer result { nullptr };
BLOCKING_INVOKE_METHOD(this, "loadScript", Q_RETURN_ARG(ScriptEnginePointer, result),
@ -488,6 +489,7 @@ ScriptEnginePointer ScriptEngines::loadScript(const QUrl& scriptFilename, bool i
scriptEngine = ScriptEnginePointer(new ScriptEngine(_context, NO_SCRIPT, "about:" + scriptFilename.fileName()));
addScriptEngine(scriptEngine);
scriptEngine->setUserLoaded(isUserLoaded);
scriptEngine->setQuitWhenFinished(quitWhenFinished);
if (scriptFilename.isEmpty() || !scriptUrl.isValid()) {
launchScriptEngine(scriptEngine);
@ -496,6 +498,11 @@ ScriptEnginePointer ScriptEngines::loadScript(const QUrl& scriptFilename, bool i
connect(scriptEngine.data(), &ScriptEngine::scriptLoaded, this, &ScriptEngines::onScriptEngineLoaded);
connect(scriptEngine.data(), &ScriptEngine::errorLoadingScript, this, &ScriptEngines::onScriptEngineError);
// Shutdown Interface when script finishes, if requested
if (quitWhenFinished) {
connect(scriptEngine.data(), &ScriptEngine::finished, this, &ScriptEngines::quitWhenFinished);
}
// get the script engine object to load the script at the designated script URL
scriptEngine->loadURL(scriptUrl, reload);
}
@ -536,6 +543,10 @@ void ScriptEngines::onScriptEngineLoaded(const QString& rawScriptURL) {
emit scriptCountChanged();
}
void ScriptEngines::quitWhenFinished() {
qApp->quit();
}
int ScriptEngines::runScriptInitializers(ScriptEnginePointer scriptEngine) {
int ii=0;
for (auto initializer : _scriptInitializers) {

4
libraries/script-engine/src/ScriptEngines.h
View file

@ -88,10 +88,11 @@ public:
* @param {boolean} [loadScriptFromEditor=false]
* @param {boolean} [activateMainWindow=false]
* @param {boolean} [reload=false]
* @param {boolean} [quitWhenFinished=false]
* @returns {boolean}
*/
Q_INVOKABLE ScriptEnginePointer loadScript(const QUrl& scriptFilename = QString(),
bool isUserLoaded = true, bool loadScriptFromEditor = false, bool activateMainWindow = false, bool reload = false);
bool isUserLoaded = true, bool loadScriptFromEditor = false, bool activateMainWindow = false, bool reload = false, bool quitWhenFinished = false);
/**jsdoc
* @function ScriptDiscoveryService.stopScript
@ -266,6 +267,7 @@ protected:
ScriptEnginePointer reloadScript(const QString& scriptName, bool isUserLoaded = true) { return loadScript(scriptName, isUserLoaded, false, false, true); }
void removeScriptEngine(ScriptEnginePointer);
void onScriptEngineLoaded(const QString& scriptFilename);
void quitWhenFinished();
void onScriptEngineError(const QString& scriptFilename);
void launchScriptEngine(ScriptEnginePointer);

8
tools/auto-tester/src/Downloader.cpp
View file

@ -9,6 +9,8 @@
//
#include "Downloader.h"
#include <QtWidgets/QMessageBox>
Downloader::Downloader(QUrl imageUrl, QObject *parent) : QObject(parent) {
connect(
&_networkAccessManager, SIGNAL (finished(QNetworkReply*)),
@ -20,6 +22,12 @@ Downloader::Downloader(QUrl imageUrl, QObject *parent) : QObject(parent) {
}
void Downloader::fileDownloaded(QNetworkReply* reply) {
QNetworkReply::NetworkError error = reply->error();
if (error != QNetworkReply::NetworkError::NoError) {
QMessageBox::information(0, "Test Aborted", "Failed to download image: " + reply->errorString());
return;
}
_downloadedData = reply->readAll();
//emit a signal

177
tools/auto-tester/src/Test.cpp
View file

@ -27,7 +27,7 @@ Test::Test() {
mismatchWindow.setModal(true);
}
bool Test::createTestResultsFolderPath(QString directory) {
bool Test::createTestResultsFolderPath(const QString& directory) {
QDateTime now = QDateTime::currentDateTime();
testResultsFolderPath = directory + "/" + TEST_RESULTS_FOLDER + "--" + now.toString(DATETIME_FORMAT);
QDir testResultsFolder(testResultsFolderPath);
@ -72,7 +72,7 @@ bool Test::compareImageLists(bool isInteractiveMode, QProgressBar* progressBar)
QImage expectedImage(expectedImagesFullFilenames[i]);
if (resultImage.width() != expectedImage.width() || resultImage.height() != expectedImage.height()) {
messageBox.critical(0, "Internal error: " + QString(__FILE__) + ":" + QString::number(__LINE__), "Images are not the same size");
QMessageBox::critical(0, "Internal error: " + QString(__FILE__) + ":" + QString::number(__LINE__), "Images are not the same size");
exit(-1);
}
@ -80,7 +80,7 @@ bool Test::compareImageLists(bool isInteractiveMode, QProgressBar* progressBar)
try {
similarityIndex = imageComparer.compareImages(resultImage, expectedImage);
} catch (...) {
messageBox.critical(0, "Internal error: " + QString(__FILE__) + ":" + QString::number(__LINE__), "Image not in expected format");
QMessageBox::critical(0, "Internal error: " + QString(__FILE__) + ":" + QString::number(__LINE__), "Image not in expected format");
exit(-1);
}
@ -125,22 +125,22 @@ bool Test::compareImageLists(bool isInteractiveMode, QProgressBar* progressBar)
return success;
}
void Test::appendTestResultsToFile(QString testResultsFolderPath, TestFailure testFailure, QPixmap comparisonImage) {
void Test::appendTestResultsToFile(const QString& testResultsFolderPath, TestFailure testFailure, QPixmap comparisonImage) {
if (!QDir().exists(testResultsFolderPath)) {
messageBox.critical(0, "Internal error: " + QString(__FILE__) + ":" + QString::number(__LINE__), "Folder " + testResultsFolderPath + " not found");
QMessageBox::critical(0, "Internal error: " + QString(__FILE__) + ":" + QString::number(__LINE__), "Folder " + testResultsFolderPath + " not found");
exit(-1);
}
QString failureFolderPath { testResultsFolderPath + "/" + "Failure_" + QString::number(index) };
if (!QDir().mkdir(failureFolderPath)) {
messageBox.critical(0, "Internal error: " + QString(__FILE__) + ":" + QString::number(__LINE__), "Failed to create folder " + failureFolderPath);
QMessageBox::critical(0, "Internal error: " + QString(__FILE__) + ":" + QString::number(__LINE__), "Failed to create folder " + failureFolderPath);
exit(-1);
}
++index;
QFile descriptionFile(failureFolderPath + "/" + TEST_RESULTS_FILENAME);
if (!descriptionFile.open(QIODevice::ReadWrite)) {
messageBox.critical(0, "Internal error: " + QString(__FILE__) + ":" + QString::number(__LINE__), "Failed to create file " + TEST_RESULTS_FILENAME);
QMessageBox::critical(0, "Internal error: " + QString(__FILE__) + ":" + QString::number(__LINE__), "Failed to create file " + TEST_RESULTS_FILENAME);
exit(-1);
}
@ -160,24 +160,30 @@ void Test::appendTestResultsToFile(QString testResultsFolderPath, TestFailure te
sourceFile = testFailure._pathname + testFailure._expectedImageFilename;
destinationFile = failureFolderPath + "/" + "Expected Image.jpg";
if (!QFile::copy(sourceFile, destinationFile)) {
messageBox.critical(0, "Internal error: " + QString(__FILE__) + ":" + QString::number(__LINE__), "Failed to copy " + sourceFile + " to " + destinationFile);
QMessageBox::critical(0, "Internal error: " + QString(__FILE__) + ":" + QString::number(__LINE__), "Failed to copy " + sourceFile + " to " + destinationFile);
exit(-1);
}
sourceFile = testFailure._pathname + testFailure._actualImageFilename;
destinationFile = failureFolderPath + "/" + "Actual Image.jpg";
if (!QFile::copy(sourceFile, destinationFile)) {
messageBox.critical(0, "Internal error: " + QString(__FILE__) + ":" + QString::number(__LINE__), "Failed to copy " + sourceFile + " to " + destinationFile);
QMessageBox::critical(0, "Internal error: " + QString(__FILE__) + ":" + QString::number(__LINE__), "Failed to copy " + sourceFile + " to " + destinationFile);
exit(-1);
}
comparisonImage.save(failureFolderPath + "/" + "Difference Image.jpg");
}
void Test::startTestsEvaluation() {
// Get list of JPEG images in folder, sorted by name
pathToTestResultsDirectory = QFileDialog::getExistingDirectory(nullptr, "Please select folder containing the test images", ".", QFileDialog::ShowDirsOnly);
if (pathToTestResultsDirectory == "") {
void Test::startTestsEvaluation(const QString& testFolder) {
QString pathToTestResultsDirectory;
if (testFolder.isNull()) {
// Get list of JPEG images in folder, sorted by name
pathToTestResultsDirectory = QFileDialog::getExistingDirectory(nullptr, "Please select folder containing the test images", ".", QFileDialog::ShowDirsOnly);
} else {
pathToTestResultsDirectory = testFolder;
}
if (pathToTestResultsDirectory == QString()) {
return;
}
@ -221,8 +227,8 @@ void Test::startTestsEvaluation() {
QString expectedImageFilenameTail = currentFilename.left(currentFilename.length() - 4).right(NUM_DIGITS);
QString expectedImageStoredFilename = EXPECTED_IMAGE_PREFIX + expectedImageFilenameTail + ".png";
QString imageURLString("https://github.com/" + githubUser + "/hifi_tests/blob/" + gitHubBranch + "/" +
expectedImagePartialSourceDirectory + "/" + expectedImageStoredFilename + "?raw=true");
QString imageURLString("https://raw.githubusercontent.com/" + githubUser + "/hifi_tests/" + gitHubBranch + "/" +
expectedImagePartialSourceDirectory + "/" + expectedImageStoredFilename);
expectedImagesURLs << imageURLString;
@ -237,19 +243,21 @@ void Test::startTestsEvaluation() {
autoTester->downloadImages(expectedImagesURLs, pathToTestResultsDirectory, expectedImagesFilenames);
}
void Test::finishTestsEvaluation(bool interactiveMode, QProgressBar* progressBar) {
bool success = compareImageLists(interactiveMode, progressBar);
void Test::finishTestsEvaluation(bool isRunningFromCommandline, bool interactiveMode, QProgressBar* progressBar) {
bool success = compareImageLists((!isRunningFromCommandline && interactiveMode), progressBar);
if (success) {
messageBox.information(0, "Success", "All images are as expected");
} else {
messageBox.information(0, "Failure", "One or more images are not as expected");
if (!isRunningFromCommandline) {
if (success) {
QMessageBox::information(0, "Success", "All images are as expected");
} else {
QMessageBox::information(0, "Failure", "One or more images are not as expected");
}
}
zipAndDeleteTestResultsFolder();
}
bool Test::isAValidDirectory(QString pathname) {
bool Test::isAValidDirectory(const QString& pathname) {
// Only process directories
QDir dir(pathname);
if (!dir.exists()) {
@ -264,7 +272,7 @@ bool Test::isAValidDirectory(QString pathname) {
return true;
}
QString Test::extractPathFromTestsDown(QString fullPath) {
QString Test::extractPathFromTestsDown(const QString& fullPath) {
// `fullPath` includes the full path to the test. We need the portion below (and including) `tests`
QStringList pathParts = fullPath.split('/');
int i{ 0 };
@ -273,7 +281,7 @@ QString Test::extractPathFromTestsDown(QString fullPath) {
}
if (i == pathParts.length()) {
messageBox.critical(0, "Internal error: " + QString(__FILE__) + ":" + QString::number(__LINE__), "Bad testPathname");
QMessageBox::critical(0, "Internal error: " + QString(__FILE__) + ":" + QString::number(__LINE__), "Bad testPathname");
exit(-1);
}
@ -342,14 +350,14 @@ void Test::createAllRecursiveScripts() {
}
}
messageBox.information(0, "Success", "Scripts have been created");
QMessageBox::information(0, "Success", "Scripts have been created");
}
void Test::createRecursiveScript(QString topLevelDirectory, bool interactiveMode) {
void Test::createRecursiveScript(const QString& topLevelDirectory, bool interactiveMode) {
const QString recursiveTestsFilename("testRecursive.js");
QFile allTestsFilename(topLevelDirectory + "/" + recursiveTestsFilename);
if (!allTestsFilename.open(QIODevice::WriteOnly | QIODevice::Text)) {
messageBox.critical(0,
QMessageBox::critical(0,
"Internal error: " + QString(__FILE__) + ":" + QString::number(__LINE__),
"Failed to create \"" + recursiveTestsFilename + "\" in directory \"" + topLevelDirectory + "\""
);
@ -358,12 +366,15 @@ void Test::createRecursiveScript(QString topLevelDirectory, bool interactiveMode
}
QTextStream textStream(&allTestsFilename);
textStream << "// This is an automatically generated file, created by auto-tester" << endl << endl;
const QString DATE_TIME_FORMAT("MMM d yyyy, h:mm");
textStream << "// This is an automatically generated file, created by auto-tester on " << QDateTime::currentDateTime().toString(DATE_TIME_FORMAT) << endl << endl;
textStream << "var autoTester = Script.require(\"https://github.com/" + githubUser + "/hifi_tests/blob/"
+ gitHubBranch + "/tests/utils/autoTester.js?raw=true\");" << endl;
+ gitHubBranch + "/tests/utils/autoTester.js?raw=true\");" << endl << endl;
textStream << "autoTester.enableRecursive();" << endl << endl;
textStream << "autoTester.enableRecursive();" << endl;
textStream << "autoTester.enableAuto();" << endl << endl;
QVector<QString> testPathnames;
@ -398,7 +409,7 @@ void Test::createRecursiveScript(QString topLevelDirectory, bool interactiveMode
}
if (interactiveMode && testPathnames.length() <= 0) {
messageBox.information(0, "Failure", "No \"" + TEST_FILENAME + "\" files found");
QMessageBox::information(0, "Failure", "No \"" + TEST_FILENAME + "\" files found");
allTestsFilename.close();
return;
}
@ -409,7 +420,7 @@ void Test::createRecursiveScript(QString topLevelDirectory, bool interactiveMode
allTestsFilename.close();
if (interactiveMode) {
messageBox.information(0, "Success", "Script has been created");
QMessageBox::information(0, "Success", "Script has been created");
}
}
@ -434,7 +445,7 @@ void Test::createTest() {
QString fullCurrentFilename = imageSourceDirectory + "/" + currentFilename;
if (isInSnapshotFilenameFormat("jpg", currentFilename)) {
if (i >= maxImages) {
messageBox.critical(0, "Error", "More than " + QString::number(maxImages) + " images not supported");
QMessageBox::critical(0, "Error", "More than " + QString::number(maxImages) + " images not supported");
exit(-1);
}
QString newFilename = "ExpectedImage_" + QString::number(i - 1).rightJustified(5, '0') + ".png";
@ -443,14 +454,14 @@ void Test::createTest() {
try {
copyJPGtoPNG(fullCurrentFilename, fullNewFileName);
} catch (...) {
messageBox.critical(0, "Error", "Could not delete existing file: " + currentFilename + "\nTest creation aborted");
QMessageBox::critical(0, "Error", "Could not delete existing file: " + currentFilename + "\nTest creation aborted");
exit(-1);
}
++i;
}
}
messageBox.information(0, "Success", "Test images have been created");
QMessageBox::information(0, "Success", "Test images have been created");
}
ExtractedText Test::getTestScriptLines(QString testFileName) {
@ -459,7 +470,7 @@ ExtractedText Test::getTestScriptLines(QString testFileName) {
QFile inputFile(testFileName);
inputFile.open(QIODevice::ReadOnly);
if (!inputFile.isOpen()) {
messageBox.critical(0,
QMessageBox::critical(0,
"Internal error: " + QString(__FILE__) + ":" + QString::number(__LINE__),
"Failed to open \"" + testFileName
);
@ -498,6 +509,12 @@ ExtractedText Test::getTestScriptLines(QString testFileName) {
const QString regexAssertGPU(ws + functionAssertGPU + ws + "\\(" + ws + quotedString + ".*");
const QRegularExpression lineAssertGPU = QRegularExpression(regexAssertGPU);
// Assert the correct amount of memory
const QString functionAssertPhysicalMemoryGB(ws + "autoTester" + ws + "\\." + ws + "assertPhysicalMemoryGB");
const QString regexAssertPhysicalMemoryGB(ws + functionAssertPhysicalMemoryGB + ws + "\\(" + ws + quotedString + ".*");
const QRegularExpression lineAssertPhysicalMemoryGB = QRegularExpression(regexAssertPhysicalMemoryGB);
// Each step is either of the following forms:
// autoTester.addStepSnapshot("Take snapshot"...
// autoTester.addStep("Clean up after test"...
@ -514,18 +531,7 @@ ExtractedText Test::getTestScriptLines(QString testFileName) {
if (lineContainingTitle.match(line).hasMatch()) {
QStringList tokens = line.split('"');
relevantTextFromTest.title = tokens[1];
} else if (lineAssertPlatform.match(line).hasMatch()) {
QStringList platforms = line.split('"');
relevantTextFromTest.platform = platforms[1];
} else if (lineAssertDisplay.match(line).hasMatch()) {
QStringList displays = line.split('"');
relevantTextFromTest.display = displays[1];
} else if (lineAssertCPU.match(line).hasMatch()) {
QStringList cpus = line.split('"');
relevantTextFromTest.cpu = cpus[1];
} else if (lineAssertGPU.match(line).hasMatch()) {
QStringList gpus = line.split('"');
relevantTextFromTest.gpu = gpus[1];
} else if (lineStepSnapshot.match(line).hasMatch()) {
QStringList tokens = line.split('"');
QString nameOfStep = tokens[1];
@ -534,6 +540,7 @@ ExtractedText Test::getTestScriptLines(QString testFileName) {
step->text = nameOfStep;
step->takeSnapshot = true;
relevantTextFromTest.stepList.emplace_back(step);
} else if (lineStep.match(line).hasMatch()) {
QStringList tokens = line.split('"');
QString nameOfStep = tokens[1];
@ -593,15 +600,15 @@ void Test::createAllMDFiles() {
}
}
messageBox.information(0, "Success", "MD files have been created");
QMessageBox::information(0, "Success", "MD files have been created");
}
void Test::createMDFile(QString testDirectory) {
void Test::createMDFile(const QString& testDirectory) {
// Verify folder contains test.js file
QString testFileName(testDirectory + "/" + TEST_FILENAME);
QFileInfo testFileInfo(testFileName);
if (!testFileInfo.exists()) {
messageBox.critical(0, "Error", "Could not find file: " + TEST_FILENAME);
QMessageBox::critical(0, "Error", "Could not find file: " + TEST_FILENAME);
return;
}
@ -610,7 +617,7 @@ void Test::createMDFile(QString testDirectory) {
QString mdFilename(testDirectory + "/" + "test.md");
QFile mdFile(mdFilename);
if (!mdFile.open(QIODevice::WriteOnly)) {
messageBox.critical(0, "Internal error: " + QString(__FILE__) + ":" + QString::number(__LINE__), "Failed to create file " + mdFilename);
QMessageBox::critical(0, "Internal error: " + QString(__FILE__) + ":" + QString::number(__LINE__), "Failed to create file " + mdFilename);
exit(-1);
}
@ -628,64 +635,24 @@ void Test::createMDFile(QString testDirectory) {
stream << "## Preconditions" << "\n";
stream << "- In an empty region of a domain with editing rights." << "\n\n";
// Platform
QStringList platforms = testScriptLines.platform.split(" ");;
stream << "## Platforms\n";
stream << "Run the test on each of the following platforms\n";
for (int i = 0; i < platforms.size(); ++i) {
// Note that the platforms parameter may include extra spaces, these appear as empty strings in the list
if (platforms[i] != QString()) {
stream << " - " << platforms[i] << "\n";
}
}
// Display
QStringList displays = testScriptLines.display.split(" ");
stream << "## Displays\n";
stream << "Run the test on each of the following displays\n";
for (int i = 0; i < displays.size(); ++i) {
// Note that the displays parameter may include extra spaces, these appear as empty strings in the list
if (displays[i] != QString()) {
stream << " - " << displays[i] << "\n";
}
}
// CPU
QStringList cpus = testScriptLines.cpu.split(" ");
stream << "## Processors\n";
stream << "Run the test on each of the following processors\n";
for (int i = 0; i < cpus.size(); ++i) {
// Note that the cpus parameter may include extra spaces, these appear as empty strings in the list
if (cpus[i] != QString()) {
stream << " - " << cpus[i] << "\n";
}
}
// GPU
QStringList gpus = testScriptLines.gpu.split(" ");
stream << "## Graphics Cards\n";
stream << "Run the test on graphics cards from each of the following vendors\n";
for (int i = 0; i < gpus.size(); ++i) {
// Note that the gpus parameter may include extra spaces, these appear as empty strings in the list
if (gpus[i] != QString()) {
stream << " - " << gpus[i] << "\n";
}
}
stream << "## Steps\n";
stream << "Press space bar to advance step by step\n\n";
// Note that snapshots of step n are taken in step n+1
// (this implies that if the LAST step requests a snapshot then this will not work - caveat emptor)
int snapShotIndex { 0 };
for (size_t i = 0; i < testScriptLines.stepList.size(); ++i) {
stream << "### Step " << QString::number(i + 1) << "\n";
stream << "- " << testScriptLines.stepList[i]->text << "\n";
if (testScriptLines.stepList[i]->takeSnapshot) {
if ((i + 1 < testScriptLines.stepList.size()) && testScriptLines.stepList[i + 1]->takeSnapshot) {
stream << "- ![](./ExpectedImage_" << QString::number(snapShotIndex).rightJustified(5, '0') << ".png)\n";
++snapShotIndex;
}
}
mdFile.close();
QMessageBox::information(0, "Success", "Test MD file " + mdFilename + " has been created");
}
void Test::createTestsOutline() {
@ -698,7 +665,7 @@ void Test::createTestsOutline() {
QString mdFilename(testsRootDirectory + "/" + testsOutlineFilename);
QFile mdFile(mdFilename);
if (!mdFile.open(QIODevice::WriteOnly)) {
messageBox.critical(0, "Internal error: " + QString(__FILE__) + ":" + QString::number(__LINE__), "Failed to create file " + mdFilename);
QMessageBox::critical(0, "Internal error: " + QString(__FILE__) + ":" + QString::number(__LINE__), "Failed to create file " + mdFilename);
exit(-1);
}
@ -756,10 +723,10 @@ void Test::createTestsOutline() {
mdFile.close();
messageBox.information(0, "Success", "Test outline file " + testsOutlineFilename + " has been created");
QMessageBox::information(0, "Success", "Test outline file " + testsOutlineFilename + " has been created");
}
void Test::copyJPGtoPNG(QString sourceJPGFullFilename, QString destinationPNGFullFilename) {
void Test::copyJPGtoPNG(const QString& sourceJPGFullFilename, const QString& destinationPNGFullFilename) {
QFile::remove(destinationPNGFullFilename);
QImageReader reader;
@ -772,7 +739,7 @@ void Test::copyJPGtoPNG(QString sourceJPGFullFilename, QString destinationPNGFul
writer.write(image);
}
QStringList Test::createListOfAll_imagesInDirectory(QString imageFormat, QString pathToImageDirectory) {
QStringList Test::createListOfAll_imagesInDirectory(const QString& imageFormat, const QString& pathToImageDirectory) {
imageDirectory = QDir(pathToImageDirectory);
QStringList nameFilters;
nameFilters << "*." + imageFormat;
@ -785,7 +752,7 @@ QStringList Test::createListOfAll_imagesInDirectory(QString imageFormat, QString
// Filename (i.e. without extension) contains _tests_ (this is based on all test scripts being within the tests folder
// Last 5 characters in filename are digits
// Extension is jpg
bool Test::isInSnapshotFilenameFormat(QString imageFormat, QString filename) {
bool Test::isInSnapshotFilenameFormat(const QString& imageFormat, const QString& filename) {
QStringList filenameParts = filename.split(".");
bool filnameHasNoPeriods = (filenameParts.size() == 2);
@ -802,7 +769,7 @@ bool Test::isInSnapshotFilenameFormat(QString imageFormat, QString filename) {
// For a file named "D_GitHub_hifi-tests_tests_content_entity_zone_create_0.jpg", the test directory is
// D:/GitHub/hifi-tests/tests/content/entity/zone/create
// This method assumes the filename is in the correct format
QString Test::getExpectedImageDestinationDirectory(QString filename) {
QString Test::getExpectedImageDestinationDirectory(const QString& filename) {
QString filenameWithoutExtension = filename.split(".")[0];
QStringList filenameParts = filenameWithoutExtension.split("_");
@ -819,7 +786,7 @@ QString Test::getExpectedImageDestinationDirectory(QString filename) {
// is ...tests/content/entity/zone/create
// This is used to create the full URL
// This method assumes the filename is in the correct format
QString Test::getExpectedImagePartialSourceDirectory(QString filename) {
QString Test::getExpectedImagePartialSourceDirectory(const QString& filename) {
QString filenameWithoutExtension = filename.split(".")[0];
QStringList filenameParts = filenameWithoutExtension.split("_");
@ -831,7 +798,7 @@ QString Test::getExpectedImagePartialSourceDirectory(QString filename) {
}
if (i < 0) {
messageBox.critical(0, "Internal error: " + QString(__FILE__) + ":" + QString::number(__LINE__), "Bad filename");
QMessageBox::critical(0, "Internal error: " + QString(__FILE__) + ":" + QString::number(__LINE__), "Bad filename");
exit(-1);
}

35
tools/auto-tester/src/Test.h
View file

@ -30,10 +30,6 @@ using StepList = std::vector<Step*>;
class ExtractedText {
public:
QString title;
QString platform;
QString display;
QString cpu;
QString gpu;
StepList stepList;
};
@ -41,61 +37,58 @@ class Test {
public:
Test();
void startTestsEvaluation();
void finishTestsEvaluation(bool interactiveMode, QProgressBar* progressBar);
void startTestsEvaluation(const QString& testFolder = QString());
void finishTestsEvaluation(bool isRunningFromCommandline, bool interactiveMode, QProgressBar* progressBar);
void createRecursiveScript();
void createAllRecursiveScripts();
void createRecursiveScript(QString topLevelDirectory, bool interactiveMode);
void createRecursiveScript(const QString& topLevelDirectory, bool interactiveMode);
void createTest();
void createMDFile();
void createAllMDFiles();
void createMDFile(QString topLevelDirectory);
void createMDFile(const QString& topLevelDirectory);
void createTestsOutline();
bool compareImageLists(bool isInteractiveMode, QProgressBar* progressBar);
QStringList createListOfAll_imagesInDirectory(QString imageFormat, QString pathToImageDirectory);
QStringList createListOfAll_imagesInDirectory(const QString& imageFormat, const QString& pathToImageDirectory);
bool isInSnapshotFilenameFormat(QString imageFormat, QString filename);
bool isInSnapshotFilenameFormat(const QString& imageFormat, const QString& filename);
void importTest(QTextStream& textStream, const QString& testPathname);
void appendTestResultsToFile(QString testResultsFolderPath, TestFailure testFailure, QPixmap comparisonImage);
void appendTestResultsToFile(const QString& testResultsFolderPath, TestFailure testFailure, QPixmap comparisonImage);
bool createTestResultsFolderPath(QString directory);
bool createTestResultsFolderPath(const QString& directory);
void zipAndDeleteTestResultsFolder();
bool isAValidDirectory(QString pathname);
QString extractPathFromTestsDown(QString fullPath);
QString getExpectedImageDestinationDirectory(QString filename);
QString getExpectedImagePartialSourceDirectory(QString filename);
bool isAValidDirectory(const QString& pathname);
QString extractPathFromTestsDown(const QString& fullPath);
QString getExpectedImageDestinationDirectory(const QString& filename);
QString getExpectedImagePartialSourceDirectory(const QString& filename);
void copyJPGtoPNG(QString sourceJPGFullFilename, QString destinationPNGFullFilename);
void copyJPGtoPNG(const QString& sourceJPGFullFilename, const QString& destinationPNGFullFilename);
private:
const QString TEST_FILENAME { "test.js" };
const QString TEST_RESULTS_FOLDER { "TestResults" };
const QString TEST_RESULTS_FILENAME { "TestResults.txt" };
QMessageBox messageBox;
QDir imageDirectory;
MismatchWindow mismatchWindow;
ImageComparer imageComparer;
QString testResultsFolderPath { "" };
QString testResultsFolderPath;
int index { 1 };
// Expected images are in the format ExpectedImage_dddd.jpg (d == decimal digit)
const int NUM_DIGITS { 5 };
const QString EXPECTED_IMAGE_PREFIX { "ExpectedImage_" };
QString pathToTestResultsDirectory;
QStringList expectedImagesFilenames;
QStringList expectedImagesFullFilenames;
QStringList resultImagesFullFilenames;

15
tools/auto-tester/src/main.cpp
View file

@ -13,10 +13,23 @@
AutoTester* autoTester;
int main(int argc, char *argv[]) {
// Only parameter is "--testFolder"
QString testFolder;
if (argc == 3) {
if (QString(argv[1]) == "--testFolder") {
testFolder = QString(argv[2]);
}
}
QApplication application(argc, argv);
autoTester = new AutoTester();
autoTester->show();
if (!testFolder.isNull()) {
autoTester->runFromCommandLine(testFolder);
} else {
autoTester->show();
}
return application.exec();
}

24
tools/auto-tester/src/ui/AutoTester.cpp
View file

@ -15,9 +15,17 @@ AutoTester::AutoTester(QWidget *parent) : QMainWindow(parent) {
ui.checkBoxInteractiveMode->setChecked(true);
ui.progressBar->setVisible(false);
test = new Test();
signalMapper = new QSignalMapper();
connect(ui.actionClose, &QAction::triggered, this, &AutoTester::on_closeButton_clicked);
connect(ui.actionAbout, &QAction::triggered, this, &AutoTester::about);
test = new Test();
}
void AutoTester::runFromCommandLine(const QString& testFolder) {
isRunningFromCommandline = true;
test->startTestsEvaluation(testFolder);
}
void AutoTester::on_evaluateTestsButton_clicked() {
@ -90,13 +98,21 @@ void AutoTester::saveImage(int index) {
image = image.convertToFormat(QImage::Format_ARGB32);
QString fullPathname = _directoryName + "/" + _filenames[index];
image.save(fullPathname, 0, 100);
if (!image.save(fullPathname, 0, 100)) {
QMessageBox::information(0, "Test Aborted", "Failed to save image: " + _filenames[index]);
ui.progressBar->setVisible(false);
return;
}
++_numberOfImagesDownloaded;
if (_numberOfImagesDownloaded == _numberOfImagesToDownload) {
test->finishTestsEvaluation(ui.checkBoxInteractiveMode->isChecked(), ui.progressBar);
test->finishTestsEvaluation(isRunningFromCommandline, ui.checkBoxInteractiveMode->isChecked(), ui.progressBar);
} else {
ui.progressBar->setValue(_numberOfImagesDownloaded);
}
}
void AutoTester::about() {
QMessageBox::information(0, "About", QString("Built ") + __DATE__ + " : " + __TIME__);
}

13
tools/auto-tester/src/ui/AutoTester.h
View file

@ -22,6 +22,9 @@ class AutoTester : public QMainWindow {
public:
AutoTester(QWidget *parent = Q_NULLPTR);
void runFromCommandLine(const QString& testFolder);
void downloadImage(const QUrl& url);
void downloadImages(const QStringList& URLs, const QString& directoryName, const QStringList& filenames);
@ -37,6 +40,8 @@ private slots:
void saveImage(int index);
void about();
private:
Ui::AutoTesterClass ui;
Test* test;
@ -50,9 +55,11 @@ private:
// Used to enable passing a parameter to slots
QSignalMapper* signalMapper;
int _numberOfImagesToDownload;
int _numberOfImagesDownloaded;
int _index;
int _numberOfImagesToDownload { 0 };
int _numberOfImagesDownloaded { 0 };
int _index { 0 };
bool isRunningFromCommandline { false };
};
#endif // hifi_AutoTester_h

26
tools/auto-tester/src/ui/AutoTester.ui
View file

@ -95,7 +95,7 @@
<number>24</number>
</property>
</widget>
<widget class="QPushButton" name="createRecursiveScriptsRecursivelyButton">
<widget class="QPushButton" name="createAllRecursiveScriptsButton">
<property name="geometry">
<rect>
<x>360</x>
@ -157,6 +157,20 @@
<height>21</height>
</rect>
</property>
<widget class="QMenu" name="menuFile">
<property name="title">
<string>File</string>
</property>
<addaction name="actionClose"/>
</widget>
<widget class="QMenu" name="menuHelp">
<property name="title">
<string>Help</string>
</property>
<addaction name="actionAbout"/>
</widget>
<addaction name="menuFile"/>
<addaction name="menuHelp"/>
</widget>
<widget class="QToolBar" name="mainToolBar">
<attribute name="toolBarArea">
@ -167,6 +181,16 @@
</attribute>
</widget>
<widget class="QStatusBar" name="statusBar"/>
<action name="actionClose">
<property name="text">
<string>Close</string>
</property>
</action>
<action name="actionAbout">
<property name="text">
<string>About</string>
</property>
</action>
</widget>
<layoutdefault spacing="6" margin="11"/>
<resources/>