This commit is contained in:
NissimHadar 2018-07-25 15:49:41 -07:00
parent 85d74afac4
commit 816b3a4ef6
8 changed files with 188 additions and 173 deletions

View file

@ -18,14 +18,14 @@
#include <quazip5/quazip.h>
#include <quazip5/JlCompress.h>
#include "TestSuiteCreator.h"
#include "TestRailInterface.h"
#include "ui/AutoTester.h"
extern AutoTester* autoTester;
#include <math.h>
Test::Test() {
mismatchWindow.setModal(true);
_mismatchWindow.setModal(true);
if (autoTester) {
autoTester->setUserText("highfidelity");
@ -35,35 +35,35 @@ Test::Test() {
bool Test::createTestResultsFolderPath(const QString& directory) {
QDateTime now = QDateTime::currentDateTime();
testResultsFolderPath = directory + "/" + TEST_RESULTS_FOLDER + "--" + now.toString(DATETIME_FORMAT);
QDir testResultsFolder(testResultsFolderPath);
_testResultsFolderPath = directory + "/" + TEST_RESULTS_FOLDER + "--" + now.toString(DATETIME_FORMAT);
QDir testResultsFolder(_testResultsFolderPath);
// Create a new test results folder
return QDir().mkdir(testResultsFolderPath);
return QDir().mkdir(_testResultsFolderPath);
}
void Test::zipAndDeleteTestResultsFolder() {
QString zippedResultsFileName { testResultsFolderPath + ".zip" };
QString zippedResultsFileName { _testResultsFolderPath + ".zip" };
QFileInfo fileInfo(zippedResultsFileName);
if (!fileInfo.exists()) {
QFile::remove(zippedResultsFileName);
}
QDir testResultsFolder(testResultsFolderPath);
QDir testResultsFolder(_testResultsFolderPath);
if (!testResultsFolder.isEmpty()) {
JlCompress::compressDir(testResultsFolderPath + ".zip", testResultsFolderPath);
JlCompress::compressDir(_testResultsFolderPath + ".zip", _testResultsFolderPath);
}
testResultsFolder.removeRecursively();
//In all cases, for the next evaluation
testResultsFolderPath = "";
index = 1;
_testResultsFolderPath = "";
_index = 1;
}
bool Test::compareImageLists(bool isInteractiveMode, QProgressBar* progressBar) {
progressBar->setMinimum(0);
progressBar->setMaximum(expectedImagesFullFilenames.length() - 1);
progressBar->setMaximum(_expectedImagesFullFilenames.length() - 1);
progressBar->setValue(0);
progressBar->setVisible(true);
@ -71,10 +71,10 @@ bool Test::compareImageLists(bool isInteractiveMode, QProgressBar* progressBar)
// Quit loop if user has aborted due to a failed test.
bool success{ true };
bool keepOn{ true };
for (int i = 0; keepOn && i < expectedImagesFullFilenames.length(); ++i) {
for (int i = 0; keepOn && i < _expectedImagesFullFilenames.length(); ++i) {
// First check that images are the same size
QImage resultImage(resultImagesFullFilenames[i]);
QImage expectedImage(expectedImagesFullFilenames[i]);
QImage resultImage(_resultImagesFullFilenames[i]);
QImage expectedImage(_expectedImagesFullFilenames[i]);
double similarityIndex; // in [-1.0 .. 1.0], where 1.0 means images are identical
@ -83,30 +83,30 @@ bool Test::compareImageLists(bool isInteractiveMode, QProgressBar* progressBar)
QMessageBox::critical(0, "Internal error: " + QString(__FILE__) + ":" + QString::number(__LINE__), "Images are not the same size");
similarityIndex = -100.0;
} else {
similarityIndex = imageComparer.compareImages(resultImage, expectedImage);
similarityIndex = _imageComparer.compareImages(resultImage, expectedImage);
}
if (similarityIndex < THRESHOLD) {
TestFailure testFailure = TestFailure{
(float)similarityIndex,
expectedImagesFullFilenames[i].left(expectedImagesFullFilenames[i].lastIndexOf("/") + 1), // path to the test (including trailing /)
QFileInfo(expectedImagesFullFilenames[i].toStdString().c_str()).fileName(), // filename of expected image
QFileInfo(resultImagesFullFilenames[i].toStdString().c_str()).fileName() // filename of result image
_expectedImagesFullFilenames[i].left(_expectedImagesFullFilenames[i].lastIndexOf("/") + 1), // path to the test (including trailing /)
QFileInfo(_expectedImagesFullFilenames[i].toStdString().c_str()).fileName(), // filename of expected image
QFileInfo(_resultImagesFullFilenames[i].toStdString().c_str()).fileName() // filename of result image
};
mismatchWindow.setTestFailure(testFailure);
_mismatchWindow.setTestFailure(testFailure);
if (!isInteractiveMode) {
appendTestResultsToFile(testResultsFolderPath, testFailure, mismatchWindow.getComparisonImage());
appendTestResultsToFile(_testResultsFolderPath, testFailure, _mismatchWindow.getComparisonImage());
success = false;
} else {
mismatchWindow.exec();
_mismatchWindow.exec();
switch (mismatchWindow.getUserResponse()) {
switch (_mismatchWindow.getUserResponse()) {
case USER_RESPONSE_PASS:
break;
case USE_RESPONSE_FAIL:
appendTestResultsToFile(testResultsFolderPath, testFailure, mismatchWindow.getComparisonImage());
appendTestResultsToFile(_testResultsFolderPath, testFailure, _mismatchWindow.getComparisonImage());
success = false;
break;
case USER_RESPONSE_ABORT:
@ -127,20 +127,20 @@ bool Test::compareImageLists(bool isInteractiveMode, QProgressBar* progressBar)
return success;
}
void Test::appendTestResultsToFile(const QString& testResultsFolderPath, TestFailure testFailure, QPixmap comparisonImage) {
if (!QDir().exists(testResultsFolderPath)) {
QMessageBox::critical(0, "Internal error: " + QString(__FILE__) + ":" + QString::number(__LINE__), "Folder " + testResultsFolderPath + " not found");
void Test::appendTestResultsToFile(const QString& _testResultsFolderPath, TestFailure testFailure, QPixmap comparisonImage) {
if (!QDir().exists(_testResultsFolderPath)) {
QMessageBox::critical(0, "Internal error: " + QString(__FILE__) + ":" + QString::number(__LINE__), "Folder " + _testResultsFolderPath + " not found");
exit(-1);
}
QString err = QString::number(testFailure._error).left(6);
QString failureFolderPath { testResultsFolderPath + "/" + err + "-Failure_" + QString::number(index) + "--" + testFailure._actualImageFilename.left(testFailure._actualImageFilename.length() - 4) };
QString failureFolderPath { _testResultsFolderPath + "/" + err + "-Failure_" + QString::number(_index) + "--" + testFailure._actualImageFilename.left(testFailure._actualImageFilename.length() - 4) };
if (!QDir().mkdir(failureFolderPath)) {
QMessageBox::critical(0, "Internal error: " + QString(__FILE__) + ":" + QString::number(__LINE__), "Failed to create folder " + failureFolderPath);
exit(-1);
}
++index;
++_index;
QFile descriptionFile(failureFolderPath + "/" + TEST_RESULTS_FILENAME);
if (!descriptionFile.open(QIODevice::ReadWrite)) {
@ -153,7 +153,7 @@ void Test::appendTestResultsToFile(const QString& testResultsFolderPath, TestFai
stream << "Test failed in folder " << testFailure._pathname.left(testFailure._pathname.length() - 1) << endl; // remove trailing '/'
stream << "Expected image was " << testFailure._expectedImageFilename << endl;
stream << "Actual image was " << testFailure._actualImageFilename << endl;
stream << "Similarity index was " << testFailure._error << endl;
stream << "Similarity _index was " << testFailure._error << endl;
descriptionFile.close();
@ -181,26 +181,26 @@ void Test::appendTestResultsToFile(const QString& testResultsFolderPath, TestFai
void Test::startTestsEvaluation(const QString& testFolder, const QString& branchFromCommandLine, const QString& userFromCommandLine) {
if (testFolder.isNull()) {
// Get list of JPEG images in folder, sorted by name
QString previousSelection = snapshotDirectory;
QString previousSelection = _snapshotDirectory;
QString parent = previousSelection.left(previousSelection.lastIndexOf('/'));
if (!parent.isNull() && parent.right(1) != "/") {
parent += "/";
}
snapshotDirectory = QFileDialog::getExistingDirectory(nullptr, "Please select folder containing the test images", parent,
_snapshotDirectory = QFileDialog::getExistingDirectory(nullptr, "Please select folder containing the test images", parent,
QFileDialog::ShowDirsOnly);
// If user cancelled then restore previous selection and return
if (snapshotDirectory == "") {
snapshotDirectory = previousSelection;
if (_snapshotDirectory == "") {
_snapshotDirectory = previousSelection;
return;
}
} else {
snapshotDirectory = testFolder;
exitWhenComplete = true;
_snapshotDirectory = testFolder;
_exitWhenComplete = true;
}
// Quit if test results folder could not be created
if (!createTestResultsFolderPath(snapshotDirectory)) {
if (!createTestResultsFolderPath(_snapshotDirectory)) {
return;
}
@ -208,20 +208,20 @@ void Test::startTestsEvaluation(const QString& testFolder, const QString& branch
// The expected images are represented as a URL to enable download from GitHub
// Images that are in the wrong format are ignored.
QStringList sortedTestResultsFilenames = createListOfAll_imagesInDirectory("png", snapshotDirectory);
QStringList sortedTestResultsFilenames = createListOfAll_imagesInDirectory("png", _snapshotDirectory);
QStringList expectedImagesURLs;
resultImagesFullFilenames.clear();
expectedImagesFilenames.clear();
expectedImagesFullFilenames.clear();
_resultImagesFullFilenames.clear();
_expectedImagesFilenames.clear();
_expectedImagesFullFilenames.clear();
QString branch = (branchFromCommandLine.isNull()) ? autoTester->getSelectedBranch() : branchFromCommandLine;
QString user = (userFromCommandLine.isNull()) ? autoTester->getSelectedUser() : userFromCommandLine;
foreach(QString currentFilename, sortedTestResultsFilenames) {
QString fullCurrentFilename = snapshotDirectory + "/" + currentFilename;
QString fullCurrentFilename = _snapshotDirectory + "/" + currentFilename;
if (isInSnapshotFilenameFormat("png", currentFilename)) {
resultImagesFullFilenames << fullCurrentFilename;
_resultImagesFullFilenames << fullCurrentFilename;
QString expectedImagePartialSourceDirectory = getExpectedImagePartialSourceDirectory(currentFilename);
@ -238,12 +238,12 @@ void Test::startTestsEvaluation(const QString& testFolder, const QString& branch
// The image retrieved from GitHub needs a unique name
QString expectedImageFilename = currentFilename.replace("/", "_").replace(".png", "_EI.png");
expectedImagesFilenames << expectedImageFilename;
expectedImagesFullFilenames << snapshotDirectory + "/" + expectedImageFilename;
_expectedImagesFilenames << expectedImageFilename;
_expectedImagesFullFilenames << _snapshotDirectory + "/" + expectedImageFilename;
}
}
autoTester->downloadImages(expectedImagesURLs, snapshotDirectory, expectedImagesFilenames);
autoTester->downloadImages(expectedImagesURLs, _snapshotDirectory, _expectedImagesFilenames);
}
void Test::finishTestsEvaluation(bool isRunningFromCommandline, bool interactiveMode, QProgressBar* progressBar) {
@ -259,7 +259,7 @@ void Test::finishTestsEvaluation(bool isRunningFromCommandline, bool interactive
zipAndDeleteTestResultsFolder();
if (exitWhenComplete) {
if (_exitWhenComplete) {
exit(0);
}
}
@ -311,46 +311,46 @@ void Test::includeTest(QTextStream& textStream, const QString& testPathname) {
// This script will run all text.js scripts in every applicable sub-folder
void Test::createRecursiveScript() {
// Select folder to start recursing from
QString previousSelection = testDirectory;
QString previousSelection = _testDirectory;
QString parent = previousSelection.left(previousSelection.lastIndexOf('/'));
if (!parent.isNull() && parent.right(1) != "/") {
parent += "/";
}
testDirectory =
_testDirectory =
QFileDialog::getExistingDirectory(nullptr, "Please select folder that will contain the top level test script", parent,
QFileDialog::ShowDirsOnly);
// If user cancelled then restore previous selection and return
if (testDirectory == "") {
testDirectory = previousSelection;
if (_testDirectory == "") {
_testDirectory = previousSelection;
return;
}
createRecursiveScript(testDirectory, true);
createRecursiveScript(_testDirectory, true);
}
// This method creates a `testRecursive.js` script in every sub-folder.
void Test::createAllRecursiveScripts() {
// Select folder to start recursing from
QString previousSelection = testsRootDirectory;
QString previousSelection = _testsRootDirectory;
QString parent = previousSelection.left(previousSelection.lastIndexOf('/'));
if (!parent.isNull() && parent.right(1) != "/") {
parent += "/";
}
testsRootDirectory = QFileDialog::getExistingDirectory(nullptr, "Please select the root folder for the recursive scripts",
_testsRootDirectory = QFileDialog::getExistingDirectory(nullptr, "Please select the root folder for the recursive scripts",
parent, QFileDialog::ShowDirsOnly);
// If user cancelled then restore previous selection and return
if (testsRootDirectory == "") {
testsRootDirectory = previousSelection;
if (_testsRootDirectory == "") {
_testsRootDirectory = previousSelection;
return;
}
createRecursiveScript(testsRootDirectory, false);
createRecursiveScript(_testsRootDirectory, false);
QDirIterator it(testsRootDirectory.toStdString().c_str(), QDirIterator::Subdirectories);
QDirIterator it(_testsRootDirectory.toStdString().c_str(), QDirIterator::Subdirectories);
while (it.hasNext()) {
QString directory = it.next();
@ -478,42 +478,42 @@ void Test::createRecursiveScript(const QString& topLevelDirectory, bool interact
void Test::createTests() {
// Rename files sequentially, as ExpectedResult_00000.jpeg, ExpectedResult_00001.jpg and so on
// Any existing expected result images will be deleted
QString previousSelection = snapshotDirectory;
QString previousSelection = _snapshotDirectory;
QString parent = previousSelection.left(previousSelection.lastIndexOf('/'));
if (!parent.isNull() && parent.right(1) != "/") {
parent += "/";
}
snapshotDirectory = QFileDialog::getExistingDirectory(nullptr, "Please select folder containing the test images", parent,
_snapshotDirectory = QFileDialog::getExistingDirectory(nullptr, "Please select folder containing the test images", parent,
QFileDialog::ShowDirsOnly);
// If user cancelled then restore previous selection and return
if (snapshotDirectory == "") {
snapshotDirectory = previousSelection;
if (_snapshotDirectory == "") {
_snapshotDirectory = previousSelection;
return;
}
previousSelection = testsRootDirectory;
previousSelection = _testsRootDirectory;
parent = previousSelection.left(previousSelection.lastIndexOf('/'));
if (!parent.isNull() && parent.right(1) != "/") {
parent += "/";
}
testsRootDirectory = QFileDialog::getExistingDirectory(nullptr, "Please select test root folder", parent,
_testsRootDirectory = QFileDialog::getExistingDirectory(nullptr, "Please select test root folder", parent,
QFileDialog::ShowDirsOnly);
// If user cancelled then restore previous selection and return
if (testsRootDirectory == "") {
testsRootDirectory = previousSelection;
if (_testsRootDirectory == "") {
_testsRootDirectory = previousSelection;
return;
}
QStringList sortedImageFilenames = createListOfAll_imagesInDirectory("png", snapshotDirectory);
QStringList sortedImageFilenames = createListOfAll_imagesInDirectory("png", _snapshotDirectory);
int i = 1;
const int maxImages = pow(10, NUM_DIGITS);
foreach (QString currentFilename, sortedImageFilenames) {
QString fullCurrentFilename = snapshotDirectory + "/" + currentFilename;
QString fullCurrentFilename = _snapshotDirectory + "/" + currentFilename;
if (isInSnapshotFilenameFormat("png", currentFilename)) {
if (i >= maxImages) {
QMessageBox::critical(0, "Error", "More than " + QString::number(maxImages) + " images not supported");
@ -523,17 +523,17 @@ void Test::createTests() {
// Path to test is extracted from the file name
// Example:
// filename is tests.engine.interaction.pointer.laser.distanceScaleEnd.00000.jpg
// path is <testDirectory>/engine/interaction/pointer/laser/distanceScaleEnd
// path is <_testDirectory>/engine/interaction/pointer/laser/distanceScaleEnd
//
// Note: we don't use the first part and the last 2 parts of the filename at this stage
//
QStringList pathParts = currentFilename.split(".");
QString fullNewFileName = testsRootDirectory;
QString fullNewFileName = _testsRootDirectory;
for (int j = 1; j < pathParts.size() - 2; ++j) {
fullNewFileName += "/" + pathParts[j];
}
// The image index is the penultimate component of the path parts (the last being the file extension)
// The image _index is the penultimate component of the path parts (the last being the file extension)
QString newFilename = "ExpectedImage_" + pathParts[pathParts.size() - 2].rightJustified(5, '0') + ".png";
fullNewFileName += "/" + newFilename;
@ -622,51 +622,51 @@ ExtractedText Test::getTestScriptLines(QString testFileName) {
// The folder selected must contain a script named "test.js", the file produced is named "test.md"
void Test::createMDFile() {
// Folder selection
QString previousSelection = testDirectory;
QString previousSelection = _testDirectory;
QString parent = previousSelection.left(previousSelection.lastIndexOf('/'));
if (!parent.isNull() && parent.right(1) != "/") {
parent += "/";
}
testDirectory = QFileDialog::getExistingDirectory(nullptr, "Please select folder containing the test", parent,
_testDirectory = QFileDialog::getExistingDirectory(nullptr, "Please select folder containing the test", parent,
QFileDialog::ShowDirsOnly);
// If user cancelled then restore previous selection and return
if (testDirectory == "") {
testDirectory = previousSelection;
if (_testDirectory == "") {
_testDirectory = previousSelection;
return;
}
createMDFile(testDirectory);
createMDFile(_testDirectory);
QMessageBox::information(0, "Success", "MD file has been created");
}
void Test::createAllMDFiles() {
// Select folder to start recursing from
QString previousSelection = testsRootDirectory;
QString previousSelection = _testsRootDirectory;
QString parent = previousSelection.left(previousSelection.lastIndexOf('/'));
if (!parent.isNull() && parent.right(1) != "/") {
parent += "/";
}
testsRootDirectory = QFileDialog::getExistingDirectory(nullptr, "Please select the root folder for the MD files", parent,
_testsRootDirectory = QFileDialog::getExistingDirectory(nullptr, "Please select the root folder for the MD files", parent,
QFileDialog::ShowDirsOnly);
// If user cancelled then restore previous selection and return
if (testsRootDirectory == "") {
testsRootDirectory = previousSelection;
if (_testsRootDirectory == "") {
_testsRootDirectory = previousSelection;
return;
}
// First test if top-level folder has a test.js file
const QString testPathname{ testsRootDirectory + "/" + TEST_FILENAME };
const QString testPathname{ _testsRootDirectory + "/" + TEST_FILENAME };
QFileInfo fileInfo(testPathname);
if (fileInfo.exists()) {
createMDFile(testsRootDirectory);
createMDFile(_testsRootDirectory);
}
QDirIterator it(testsRootDirectory.toStdString().c_str(), QDirIterator::Subdirectories);
QDirIterator it(_testsRootDirectory.toStdString().c_str(), QDirIterator::Subdirectories);
while (it.hasNext()) {
QString directory = it.next();
@ -686,9 +686,9 @@ void Test::createAllMDFiles() {
QMessageBox::information(0, "Success", "MD files have been created");
}
void Test::createMDFile(const QString& testDirectory) {
void Test::createMDFile(const QString& _testDirectory) {
// Verify folder contains test.js file
QString testFileName(testDirectory + "/" + TEST_FILENAME);
QString testFileName(_testDirectory + "/" + TEST_FILENAME);
QFileInfo testFileInfo(testFileName);
if (!testFileInfo.exists()) {
QMessageBox::critical(0, "Error", "Could not find file: " + TEST_FILENAME);
@ -697,7 +697,7 @@ void Test::createMDFile(const QString& testDirectory) {
ExtractedText testScriptLines = getTestScriptLines(testFileName);
QString mdFilename(testDirectory + "/" + "test.md");
QString mdFilename(_testDirectory + "/" + "test.md");
QFile mdFile(mdFilename);
if (!mdFile.open(QIODevice::WriteOnly)) {
QMessageBox::critical(0, "Internal error: " + QString(__FILE__) + ":" + QString::number(__LINE__), "Failed to create file " + mdFilename);
@ -711,7 +711,7 @@ void Test::createMDFile(const QString& testDirectory) {
stream << "# " << testName << "\n";
// Find the relevant part of the path to the test (i.e. from "tests" down
QString partialPath = extractPathFromTestsDown(testDirectory);
QString partialPath = extractPathFromTestsDown(_testDirectory);
stream << "## Run this script URL: [Manual](./test.js?raw=true) [Auto](./testAuto.js?raw=true)(from menu/Edit/Open and Run scripts from URL...)." << "\n\n";
@ -735,23 +735,23 @@ void Test::createMDFile(const QString& testDirectory) {
}
void Test::createTestsOutline() {
QString previousSelection = testDirectory;
QString previousSelection = _testDirectory;
QString parent = previousSelection.left(previousSelection.lastIndexOf('/'));
if (!parent.isNull() && parent.right(1) != "/") {
parent += "/";
}
testDirectory =
_testDirectory =
QFileDialog::getExistingDirectory(nullptr, "Please select the tests root folder", parent, QFileDialog::ShowDirsOnly);
// If user cancelled then restore previous selection and return
if (testDirectory == "") {
testDirectory = previousSelection;
if (_testDirectory == "") {
_testDirectory = previousSelection;
return;
}
const QString testsOutlineFilename { "testsOutline.md" };
QString mdFilename(testDirectory + "/" + testsOutlineFilename);
QString mdFilename(_testDirectory + "/" + testsOutlineFilename);
QFile mdFile(mdFilename);
if (!mdFile.open(QIODevice::WriteOnly)) {
QMessageBox::critical(0, "Internal error: " + QString(__FILE__) + ":" + QString::number(__LINE__), "Failed to create file " + mdFilename);
@ -765,10 +765,10 @@ void Test::createTestsOutline() {
stream << "Directories with an appended (*) have an automatic test\n\n";
// We need to know our current depth, as this isn't given by QDirIterator
int rootDepth { testDirectory.count('/') };
int rootDepth { _testDirectory.count('/') };
// Each test is shown as the folder name linking to the matching GitHub URL, and the path to the associated test.md file
QDirIterator it(testDirectory.toStdString().c_str(), QDirIterator::Subdirectories);
QDirIterator it(_testDirectory.toStdString().c_str(), QDirIterator::Subdirectories);
while (it.hasNext()) {
QString directory = it.next();
@ -823,31 +823,40 @@ void Test::createTestsOutline() {
}
void Test::createTestRailTestSuite() {
QString previousSelection = testDirectory;
QString previousSelection = _testDirectory;
QString parent = previousSelection.left(previousSelection.lastIndexOf('/'));
if (!parent.isNull() && parent.right(1) != "/") {
parent += "/";
}
testDirectory =
_testDirectory =
QFileDialog::getExistingDirectory(nullptr, "Please select the tests root folder", parent, QFileDialog::ShowDirsOnly);
// If user cancelled then restore previous selection and return
if (testDirectory == "") {
testDirectory = previousSelection;
if (_testDirectory == "") {
_testDirectory = previousSelection;
return;
}
TestSuiteCreator testSuiteCreator;
testSuiteCreator.createTestSuite(testDirectory, autoTester->getSelectedUser(), autoTester->getSelectedBranch());
QString outputDirectory = QFileDialog::getExistingDirectory(nullptr, "Please select a folder to store generated files in",
parent, QFileDialog::ShowDirsOnly);
// If user cancelled then restore previous selection and return
if (outputDirectory == "") {
return;
}
TestRailInterface testRailInterface;
testRailInterface.createTestSuite(_testDirectory, outputDirectory, autoTester->getSelectedUser(),
autoTester->getSelectedBranch());
}
QStringList Test::createListOfAll_imagesInDirectory(const QString& imageFormat, const QString& pathToImageDirectory) {
imageDirectory = QDir(pathToImageDirectory);
_imageDirectory = QDir(pathToImageDirectory);
QStringList nameFilters;
nameFilters << "*." + imageFormat;
return imageDirectory.entryList(nameFilters, QDir::Files, QDir::Name);
return _imageDirectory.entryList(nameFilters, QDir::Files, QDir::Name);
}
// Snapshots are files in the following format:

View file

@ -70,6 +70,8 @@ public:
QString getExpectedImageDestinationDirectory(const QString& filename);
QString getExpectedImagePartialSourceDirectory(const QString& filename);
ExtractedText getTestScriptLines(QString testFileName);
private:
const QString TEST_FILENAME { "test.js" };
const QString TEST_RESULTS_FOLDER { "TestResults" };
@ -77,14 +79,14 @@ private:
const double THRESHOLD{ 0.96 };
QDir imageDirectory;
QDir _imageDirectory;
MismatchWindow mismatchWindow;
MismatchWindow _mismatchWindow;
ImageComparer imageComparer;
ImageComparer _imageComparer;
QString testResultsFolderPath;
int index { 1 };
QString _testResultsFolderPath;
int _index { 1 };
// Expected images are in the format ExpectedImage_dddd.jpg (d == decimal digit)
const int NUM_DIGITS { 5 };
@ -94,28 +96,26 @@ private:
// The first is the directory containing the test we are working with
// The second is the root directory of all tests
// The third contains the snapshots taken for test runs that need to be evaluated
QString testDirectory;
QString testsRootDirectory;
QString snapshotDirectory;
QString _testDirectory;
QString _testsRootDirectory;
QString _snapshotDirectory;
QStringList expectedImagesFilenames;
QStringList expectedImagesFullFilenames;
QStringList resultImagesFullFilenames;
QStringList _expectedImagesFilenames;
QStringList _expectedImagesFullFilenames;
QStringList _resultImagesFullFilenames;
// Used for accessing GitHub
const QString GIT_HUB_REPOSITORY{ "hifi_tests" };
const QString DATETIME_FORMAT{ "yyyy-MM-dd_hh-mm-ss" };
ExtractedText getTestScriptLines(QString testFileName);
// NOTE: these need to match the appropriate var's in autoTester.js
// var advanceKey = "n";
// var pathSeparator = ".";
const QString ADVANCE_KEY{ "n" };
const QString PATH_SEPARATOR{ "." };
bool exitWhenComplete{ false };
bool _exitWhenComplete{ false };
};
#endif // hifi_test_h

View file

@ -1,5 +1,5 @@
//
// TestSuiteCreator.cpp
// TestRailInterface.cpp
//
// Created by Nissim Hadar on 6 Jul 2018.
// Copyright 2013 High Fidelity, Inc.
@ -8,7 +8,7 @@
// See the accompanying file LICENSE or http://www.apache.org/licenses/LICENSE-2.0.html
//
#include "TestSuiteCreator.h"
#include "TestRailInterface.h"
#include "Test.h"
#include <QDateTime>
@ -16,7 +16,10 @@
#include <QMessageBox>
#include <QTextStream>
void TestSuiteCreator::createTestSuite(const QString& testDirectory, const QString& user, const QString& branch) {
void TestRailInterface::createTestSuite(const QString& testDirectory,
const QString& outputDirectory,
const QString& user,
const QString& branch) {
QDomProcessingInstruction instruction = document.createProcessingInstruction("xml", "version='1.0' encoding='UTF-8'");
document.appendChild(instruction);
@ -37,7 +40,7 @@ void TestSuiteCreator::createTestSuite(const QString& testDirectory, const QStri
root.appendChild(topLevelSection);
// Write to file
const QString testRailsFilename{ "D:/t/TestRailSuite.xml" };
const QString testRailsFilename{ outputDirectory + "/TestRailSuite.xml" };
QFile file(testRailsFilename);
if (!file.open(QIODevice::WriteOnly | QIODevice::Text)) {
QMessageBox::critical(0, "Internal error: " + QString(__FILE__) + ":" + QString::number(__LINE__), "Could not create XML file");
@ -52,7 +55,7 @@ void TestSuiteCreator::createTestSuite(const QString& testDirectory, const QStri
QMessageBox::information(0, "Success", "TestRail XML file has been created");
}
QDomElement TestSuiteCreator::processDirectory(const QString& directory, const QString& user, const QString& branch, const QDomElement& element) {
QDomElement TestRailInterface::processDirectory(const QString& directory, const QString& user, const QString& branch, const QDomElement& element) {
QDomElement result = element;
// Loop over all entries in directory
@ -95,7 +98,7 @@ QDomElement TestSuiteCreator::processDirectory(const QString& directory, const Q
return result;
}
QDomElement TestSuiteCreator::processTest(const QString& fullDirectory, const QString& test, const QString& user, const QString& branch, const QDomElement& element) {
QDomElement TestRailInterface::processTest(const QString& fullDirectory, const QString& test, const QString& user, const QString& branch, const QDomElement& element) {
QDomElement result = element;
QDomElement caseElement = document.createElement("case");

View file

@ -1,5 +1,5 @@
//
// TestSuiteCreator.h
// TestRailInterface.h
//
// Created by Nissim Hadar on 6 Jul 2018.
// Copyright 2013 High Fidelity, Inc.
@ -8,16 +8,19 @@
// See the accompanying file LICENSE or http://www.apache.org/licenses/LICENSE-2.0.html
//
#ifndef hifi_test_suite_creator_h
#define hifi_test_suite_creator_h
#ifndef hifi_test_testrail_interface_h
#define hifi_test_testrail_interface_h
#include <QDirIterator>
#include <QtXml/QDomDocument>
#include <QString>
class TestSuiteCreator {
class TestRailInterface {
public:
void createTestSuite(const QString& testDirectory, const QString& user, const QString& branch);
void createTestSuite(const QString& testDirectory,
const QString& outputDirectory,
const QString& user,
const QString& branch);
QDomElement processDirectory(const QString& directory, const QString& user, const QString& branch, const QDomElement& element);
QDomElement processTest(const QString& fullDirectory, const QString& test, const QString& user, const QString& branch, const QDomElement& element);
@ -26,4 +29,4 @@ private:
QDomDocument document;
};
#endif // hifi_test_suite_creator_h
#endif

View file

@ -16,60 +16,60 @@
#endif
AutoTester::AutoTester(QWidget *parent) : QMainWindow(parent) {
ui.setupUi(this);
ui.checkBoxInteractiveMode->setChecked(true);
ui.progressBar->setVisible(false);
_ui.setupUi(this);
_ui.checkBoxInteractiveMode->setChecked(true);
_ui.progressBar->setVisible(false);
signalMapper = new QSignalMapper();
_signalMapper = new QSignalMapper();
connect(ui.actionClose, &QAction::triggered, this, &AutoTester::on_closeButton_clicked);
connect(ui.actionAbout, &QAction::triggered, this, &AutoTester::about);
connect(_ui.actionClose, &QAction::triggered, this, &AutoTester::on_closeButton_clicked);
connect(_ui.actionAbout, &QAction::triggered, this, &AutoTester::about);
#ifndef Q_OS_WIN
ui.hideTaskbarButton->setVisible(false);
ui.showTaskbarButton->setVisible(false);
_ui.hideTaskbarButton->setVisible(false);
_ui.showTaskbarButton->setVisible(false);
#endif
}
void AutoTester::setup() {
test = new Test();
_test = new Test();
}
void AutoTester::runFromCommandLine(const QString& testFolder, const QString& branch, const QString& user) {
isRunningFromCommandline = true;
test->startTestsEvaluation(testFolder, branch, user);
_isRunningFromCommandline = true;
_test->startTestsEvaluation(testFolder, branch, user);
}
void AutoTester::on_evaluateTestsButton_clicked() {
test->startTestsEvaluation();
_test->startTestsEvaluation();
}
void AutoTester::on_createRecursiveScriptButton_clicked() {
test->createRecursiveScript();
_test->createRecursiveScript();
}
void AutoTester::on_createAllRecursiveScriptsButton_clicked() {
test->createAllRecursiveScripts();
_test->createAllRecursiveScripts();
}
void AutoTester::on_createTestsButton_clicked() {
test->createTests();
_test->createTests();
}
void AutoTester::on_createMDFileButton_clicked() {
test->createMDFile();
_test->createMDFile();
}
void AutoTester::on_createAllMDFilesButton_clicked() {
test->createAllMDFiles();
_test->createAllMDFiles();
}
void AutoTester::on_createTestsOutlineButton_clicked() {
test->createTestsOutline();
_test->createTestsOutline();
}
void AutoTester::on_createTestRailTestSuiteButton_clicked() {
test->createTestRailTestSuite();
_test->createTestRailTestSuite();
}
// To toggle between show and hide
@ -101,10 +101,10 @@ void AutoTester::on_closeButton_clicked() {
}
void AutoTester::downloadImage(const QUrl& url) {
downloaders.emplace_back(new Downloader(url, this));
connect(downloaders[_index], SIGNAL (downloaded()), signalMapper, SLOT (map()));
_downloaders.emplace_back(new Downloader(url, this));
connect(_downloaders[_index], SIGNAL (downloaded()), _signalMapper, SLOT (map()));
signalMapper->setMapping(downloaders[_index], _index);
_signalMapper->setMapping(_downloaders[_index], _index);
++_index;
}
@ -117,39 +117,39 @@ void AutoTester::downloadImages(const QStringList& URLs, const QString& director
_numberOfImagesDownloaded = 0;
_index = 0;
ui.progressBar->setMinimum(0);
ui.progressBar->setMaximum(_numberOfImagesToDownload - 1);
ui.progressBar->setValue(0);
ui.progressBar->setVisible(true);
_ui.progressBar->setMinimum(0);
_ui.progressBar->setMaximum(_numberOfImagesToDownload - 1);
_ui.progressBar->setValue(0);
_ui.progressBar->setVisible(true);
downloaders.clear();
_downloaders.clear();
for (int i = 0; i < _numberOfImagesToDownload; ++i) {
QUrl imageURL(URLs[i]);
downloadImage(imageURL);
}
connect(signalMapper, SIGNAL (mapped(int)), this, SLOT (saveImage(int)));
connect(_signalMapper, SIGNAL (mapped(int)), this, SLOT (saveImage(int)));
}
void AutoTester::saveImage(int index) {
try {
QFile file(_directoryName + "/" + _filenames[index]);
file.open(QIODevice::WriteOnly);
file.write(downloaders[index]->downloadedData());
file.write(_downloaders[index]->downloadedData());
file.close();
} catch (...) {
QMessageBox::information(0, "Test Aborted", "Failed to save image: " + _filenames[index]);
ui.progressBar->setVisible(false);
_ui.progressBar->setVisible(false);
return;
}
++_numberOfImagesDownloaded;
if (_numberOfImagesDownloaded == _numberOfImagesToDownload) {
disconnect(signalMapper, SIGNAL (mapped(int)), this, SLOT (saveImage(int)));
test->finishTestsEvaluation(isRunningFromCommandline, ui.checkBoxInteractiveMode->isChecked(), ui.progressBar);
disconnect(_signalMapper, SIGNAL (mapped(int)), this, SLOT (saveImage(int)));
_test->finishTestsEvaluation(_isRunningFromCommandline, _ui.checkBoxInteractiveMode->isChecked(), _ui.progressBar);
} else {
ui.progressBar->setValue(_numberOfImagesDownloaded);
_ui.progressBar->setValue(_numberOfImagesDownloaded);
}
}
@ -158,18 +158,18 @@ void AutoTester::about() {
}
void AutoTester::setUserText(const QString& user) {
ui.userTextEdit->setText(user);
_ui.userTextEdit->setText(user);
}
QString AutoTester::getSelectedUser()
{
return ui.userTextEdit->toPlainText();
return _ui.userTextEdit->toPlainText();
}
void AutoTester::setBranchText(const QString& branch) {
ui.branchTextEdit->setText(branch);
_ui.branchTextEdit->setText(branch);
}
QString AutoTester::getSelectedBranch() {
return ui.branchTextEdit->toPlainText();
return _ui.branchTextEdit->toPlainText();
}

View file

@ -56,23 +56,23 @@ private slots:
void about();
private:
Ui::AutoTesterClass ui;
Test* test;
Ui::AutoTesterClass _ui;
Test* _test;
std::vector<Downloader*> downloaders;
std::vector<Downloader*> _downloaders;
// local storage for parameters - folder to store downloaded files in, and a list of their names
QString _directoryName;
QStringList _filenames;
// Used to enable passing a parameter to slots
QSignalMapper* signalMapper;
QSignalMapper* _signalMapper;
int _numberOfImagesToDownload { 0 };
int _numberOfImagesDownloaded { 0 };
int _index { 0 };
bool isRunningFromCommandline { false };
bool _isRunningFromCommandline { false };
};
#endif // hifi_AutoTester_h

View file

@ -66,14 +66,14 @@ void MismatchWindow::setTestFailure(TestFailure testFailure) {
QPixmap expectedPixmap = QPixmap(testFailure._pathname + testFailure._expectedImageFilename);
QPixmap actualPixmap = QPixmap(testFailure._pathname + testFailure._actualImageFilename);
diffPixmap = computeDiffPixmap(
_diffPixmap = computeDiffPixmap(
QImage(testFailure._pathname + testFailure._expectedImageFilename),
QImage(testFailure._pathname + testFailure._actualImageFilename)
);
expectedImage->setPixmap(expectedPixmap);
resultImage->setPixmap(actualPixmap);
diffImage->setPixmap(diffPixmap);
diffImage->setPixmap(_diffPixmap);
}
void MismatchWindow::on_passTestButton_clicked() {
@ -92,5 +92,5 @@ void MismatchWindow::on_abortTestsButton_clicked() {
}
QPixmap MismatchWindow::getComparisonImage() {
return diffPixmap;
return _diffPixmap;
}

View file

@ -36,7 +36,7 @@ private slots:
private:
UserResponse _userResponse{ USER_RESPONSE_INVALID };
QPixmap diffPixmap;
QPixmap _diffPixmap;
};