Mirrors/overte-AleziaKurdis
3
0
Fork 0
mirror of https://github.com/AleziaKurdis/overte.git synced 2025-04-19 08:18:05 +02:00
Code Issues Projects Releases Packages Wiki Activity Actions

Merge pull request #12771 from NissimHadar/addMDtoAutoTester

Browse source 
Add .md creation to autoTester
This commit is contained in:
Sam Gateau 2018-04-04 12:30:11 -07:00 committed by GitHub
commit 4870ce1d76
No known key found for this signature in database
GPG key ID: 4AEE18F83AFDEB23
5 changed files with 280 additions and 48 deletions
Show stats Download patch file Download diff file Expand all files Collapse all files

264
tools/auto-tester/src/Test.cpp
View file

@ -24,15 +24,11 @@ extern AutoTester* autoTester;
#include <math.h>
Test::Test() {
QString regex(EXPECTED_IMAGE_PREFIX + QString("\\\\d").repeated(NUM_DIGITS) + ".png");
expectedImageFilenameFormat = QRegularExpression(regex);
mismatchWindow.setModal(true);
}
bool Test::createTestResultsFolderPath(QString directory) {
QDateTime now = QDateTime::currentDateTime();
QDateTime now = QDateTime::currentDateTime();
testResultsFolderPath = directory + "/" + TEST_RESULTS_FOLDER + "--" + now.toString(DATETIME_FORMAT);
QDir testResultsFolder(testResultsFolderPath);
@ -76,7 +72,7 @@ bool Test::compareImageLists(bool isInteractiveMode, QProgressBar* progressBar)
QImage expectedImage(expectedImagesFullFilenames[i]);
if (resultImage.width() != expectedImage.width() || resultImage.height() != expectedImage.height()) {
messageBox.critical(0, "Internal error #1", "Images are not the same size");
messageBox.critical(0, "Internal error: " + QString(__FILE__) + ":" + QString::number(__LINE__), "Images are not the same size");
exit(-1);
}
@ -84,7 +80,7 @@ bool Test::compareImageLists(bool isInteractiveMode, QProgressBar* progressBar)
try {
similarityIndex = imageComparer.compareImages(resultImage, expectedImage);
} catch (...) {
messageBox.critical(0, "Internal error #2", "Image not in expected format");
messageBox.critical(0, "Internal error: " + QString(__FILE__) + ":" + QString::number(__LINE__), "Image not in expected format");
exit(-1);
}
@ -131,20 +127,20 @@ bool Test::compareImageLists(bool isInteractiveMode, QProgressBar* progressBar)
void Test::appendTestResultsToFile(QString testResultsFolderPath, TestFailure testFailure, QPixmap comparisonImage) {
if (!QDir().exists(testResultsFolderPath)) {
messageBox.critical(0, "Internal error #3", "Folder " + testResultsFolderPath + " not found");
messageBox.critical(0, "Internal error: " + QString(__FILE__) + ":" + QString::number(__LINE__), "Folder " + testResultsFolderPath + " not found");
exit(-1);
}
QString failureFolderPath { testResultsFolderPath + "/" + "Failure_" + QString::number(index) };
if (!QDir().mkdir(failureFolderPath)) {
messageBox.critical(0, "Internal error #4", "Failed to create folder " + failureFolderPath);
messageBox.critical(0, "Internal error: " + QString(__FILE__) + ":" + QString::number(__LINE__), "Failed to create folder " + failureFolderPath);
exit(-1);
}
++index;
QFile descriptionFile(failureFolderPath + "/" + TEST_RESULTS_FILENAME);
if (!descriptionFile.open(QIODevice::ReadWrite)) {
messageBox.critical(0, "Internal error #5", "Failed to create file " + TEST_RESULTS_FILENAME);
messageBox.critical(0, "Internal error: " + QString(__FILE__) + ":" + QString::number(__LINE__), "Failed to create file " + TEST_RESULTS_FILENAME);
exit(-1);
}
@ -164,14 +160,14 @@ void Test::appendTestResultsToFile(QString testResultsFolderPath, TestFailure te
sourceFile = testFailure._pathname + testFailure._expectedImageFilename;
destinationFile = failureFolderPath + "/" + "Expected Image.jpg";
if (!QFile::copy(sourceFile, destinationFile)) {
messageBox.critical(0, "Internal error #6", "Failed to copy " + sourceFile + " to " + destinationFile);
messageBox.critical(0, "Internal error: " + QString(__FILE__) + ":" + QString::number(__LINE__), "Failed to copy " + sourceFile + " to " + destinationFile);
exit(-1);
}
sourceFile = testFailure._pathname + testFailure._actualImageFilename;
destinationFile = failureFolderPath + "/" + "Actual Image.jpg";
if (!QFile::copy(sourceFile, destinationFile)) {
messageBox.critical(0, "Internal error #7", "Failed to copy " + sourceFile + " to " + destinationFile);
messageBox.critical(0, "Internal error: " + QString(__FILE__) + ":" + QString::number(__LINE__), "Failed to copy " + sourceFile + " to " + destinationFile);
exit(-1);
}
@ -209,11 +205,6 @@ void Test::startTestsEvaluation() {
QStringList sortedTestResultsFilenames = createListOfAll_imagesInDirectory("png", pathToTestResultsDirectory);
QStringList expectedImagesURLs;
const QString URLPrefix("https://raw.githubusercontent.com");
const QString githubUser("NissimHadar");
const QString testsRepo("hifi_tests");
const QString branch("addRecursionToAutotester");
resultImagesFullFilenames.clear();
expectedImagesFilenames.clear();
expectedImagesFullFilenames.clear();
@ -226,16 +217,16 @@ void Test::startTestsEvaluation() {
QString expectedImagePartialSourceDirectory = getExpectedImagePartialSourceDirectory(currentFilename);
// Images are stored on GitHub as ExpectedImage_ddddd.png
// Extract the digits at the end of the filename (exluding the file extension)
// Extract the digits at the end of the filename (excluding the file extension)
QString expectedImageFilenameTail = currentFilename.left(currentFilename.length() - 4).right(NUM_DIGITS);
QString expectedImageStoredFilename = EXPECTED_IMAGE_PREFIX + expectedImageFilenameTail + ".png";
QString imageURLString(URLPrefix + "/" + githubUser + "/" + testsRepo + "/" + branch + "/" +
expectedImagePartialSourceDirectory + "/" + expectedImageStoredFilename);
QString imageURLString("https://github.com/" + githubUser + "/hifi_tests/blob/" + gitHubBranch + "/" +
expectedImagePartialSourceDirectory + "/" + expectedImageStoredFilename + "?raw=true");
expectedImagesURLs << imageURLString;
// The image retrieved from Github needs a unique name
// The image retrieved from GitHub needs a unique name
QString expectedImageFilename = currentFilename.replace("/", "_").replace(".", "_EI.");
expectedImagesFilenames << expectedImageFilename;
@ -273,25 +264,31 @@ bool Test::isAValidDirectory(QString pathname) {
return true;
}
void Test::importTest(QTextStream& textStream, const QString& testPathname) {
// `testPathname` includes the full path to the test. We need the portion below (and including) `tests`
QStringList filenameParts = testPathname.split('/');
QString Test::extractPathFromTestsDown(QString fullPath) {
// `fullPath` includes the full path to the test. We need the portion below (and including) `tests`
QStringList pathParts = fullPath.split('/');
int i{ 0 };
while (i < filenameParts.length() && filenameParts[i] != "tests") {
while (i < pathParts.length() && pathParts[i] != "tests") {
++i;
}
if (i == filenameParts.length()) {
messageBox.critical(0, "Internal error #10", "Bad testPathname");
if (i == pathParts.length()) {
messageBox.critical(0, "Internal error: " + QString(__FILE__) + ":" + QString::number(__LINE__), "Bad testPathname");
exit(-1);
}
QString filename;
for (int j = i; j < filenameParts.length(); ++j) {
filename += "/" + filenameParts[j];
QString partialPath;
for (int j = i; j < pathParts.length(); ++j) {
partialPath += "/" + pathParts[j];
}
textStream << "Script.include(\"" << "https://raw.githubusercontent.com/" << user << "/hifi_tests/" << branch << filename + "\");" << endl;
return partialPath;
}
void Test::importTest(QTextStream& textStream, const QString& testPathname) {
QString partialPath = extractPathFromTestsDown(testPathname);
textStream << "Script.include(\"" << "https://github.com/" << githubUser
<< "/hifi_tests/blob/" << gitHubBranch << partialPath + "?raw=true\");" << endl;
}
// Creates a single script in a user-selected folder.
@ -353,7 +350,7 @@ void Test::createRecursiveScript(QString topLevelDirectory, bool interactiveMode
QFile allTestsFilename(topLevelDirectory + "/" + recursiveTestsFilename);
if (!allTestsFilename.open(QIODevice::WriteOnly | QIODevice::Text)) {
messageBox.critical(0,
"Internal Error #8",
"Internal error: " + QString(__FILE__) + ":" + QString::number(__LINE__),
"Failed to create \"" + recursiveTestsFilename + "\" in directory \"" + topLevelDirectory + "\""
);
@ -363,7 +360,9 @@ void Test::createRecursiveScript(QString topLevelDirectory, bool interactiveMode
QTextStream textStream(&allTestsFilename);
textStream << "// This is an automatically generated file, created by auto-tester" << endl << endl;
textStream << "var autoTester = Script.require(\"https://raw.githubusercontent.com/" + user + "/hifi_tests/" + branch + "/tests/utils/autoTester.js\");" << endl;
textStream << "var autoTester = Script.require(\"https://github.com/" + githubUser + "/hifi_tests/blob/"
+ gitHubBranch + "/tests/utils/autoTester.js?raw=true\");" << endl;
textStream << "autoTester.enableRecursive();" << endl << endl;
QVector<QString> testPathnames;
@ -454,6 +453,203 @@ void Test::createTest() {
messageBox.information(0, "Success", "Test images have been created");
}
ExtractedText Test::getTestScriptLines(QString testFileName) {
ExtractedText relevantTextFromTest;
QFile inputFile(testFileName);
inputFile.open(QIODevice::ReadOnly);
if (!inputFile.isOpen()) {
messageBox.critical(0,
"Internal error: " + QString(__FILE__) + ":" + QString::number(__LINE__),
"Failed to open \"" + testFileName
);
}
QTextStream stream(&inputFile);
QString line = stream.readLine();
// Name of test is the string in the following line:
// autoTester.perform("Apply Material Entities to Avatars", Script.resolvePath("."), function(testType) {...
const QString ws("\\h*"); //white-space character
const QString functionPerformName(ws + "autoTester" + ws + "\\." + ws + "perform");
const QString quotedString("\\\".+\\\"");
const QString ownPath("Script" + ws + "\\." + ws + "resolvePath" + ws + "\\(" + ws + "\\\"\\.\\\"" + ws + "\\)");
const QString functionParameter("function" + ws + "\\(testType" + ws + "\\)");
QString regexTestTitle(ws + functionPerformName + "\\(" + quotedString + "\\," + ws + ownPath + "\\," + ws + functionParameter + ws + "{" + ".*");
QRegularExpression lineContainingTitle = QRegularExpression(regexTestTitle);
// Assert platform checks that test is running on the correct OS
const QString functionAssertPlatform(ws + "autoTester" + ws + "\\." + ws + "assertPlatform");
const QString regexAssertPlatform(ws + functionAssertPlatform + ws + "\\(" + ws + quotedString + ".*");
const QRegularExpression lineAssertPlatform = QRegularExpression(regexAssertPlatform);
// Assert display checks that test is running on the correct display
const QString functionAssertDisplay(ws + "autoTester" + ws + "\\." + ws + "assertDisplay");
const QString regexAssertDisplay(ws + functionAssertDisplay + ws + "\\(" + ws + quotedString + ".*");
const QRegularExpression lineAssertDisplay = QRegularExpression(regexAssertDisplay);
// Assert CPU checks that test is running on the correct type of CPU
const QString functionAssertCPU(ws + "autoTester" + ws + "\\." + ws + "assertCPU");
const QString regexAssertCPU(ws + functionAssertCPU + ws + "\\(" + ws + quotedString + ".*");
const QRegularExpression lineAssertCPU = QRegularExpression(regexAssertCPU);
// Assert GPU checks that test is running on the correct type of GPU
const QString functionAssertGPU(ws + "autoTester" + ws + "\\." + ws + "assertGPU");
const QString regexAssertGPU(ws + functionAssertGPU + ws + "\\(" + ws + quotedString + ".*");
const QRegularExpression lineAssertGPU = QRegularExpression(regexAssertGPU);
// Each step is either of the following forms:
// autoTester.addStepSnapshot("Take snapshot"...
// autoTester.addStep("Clean up after test"...
const QString functionAddStepSnapshotName(ws + "autoTester" + ws + "\\." + ws + "addStepSnapshot");
const QString regexStepSnapshot(ws + functionAddStepSnapshotName + ws + "\\(" + ws + quotedString + ".*");
const QRegularExpression lineStepSnapshot = QRegularExpression(regexStepSnapshot);
const QString functionAddStepName(ws + "autoTester" + ws + "\\." + ws + "addStep");
const QString regexStep(ws + functionAddStepName + ws + "\\(" + ws + quotedString + ws + "\\)" + ".*");
const QRegularExpression lineStep = QRegularExpression(regexStep);
while (!line.isNull()) {
line = stream.readLine();
if (lineContainingTitle.match(line).hasMatch()) {
QStringList tokens = line.split('"');
relevantTextFromTest.title = tokens[1];
} else if (lineAssertPlatform.match(line).hasMatch()) {
QStringList platforms = line.split('"');
relevantTextFromTest.platform = platforms[1];
} else if (lineAssertDisplay.match(line).hasMatch()) {
QStringList displays = line.split('"');
relevantTextFromTest.display = displays[1];
} else if (lineAssertCPU.match(line).hasMatch()) {
QStringList cpus = line.split('"');
relevantTextFromTest.cpu = cpus[1];
} else if (lineAssertGPU.match(line).hasMatch()) {
QStringList gpus = line.split('"');
relevantTextFromTest.gpu = gpus[1];
} else if (lineStepSnapshot.match(line).hasMatch()) {
QStringList tokens = line.split('"');
QString nameOfStep = tokens[1];
Step *step = new Step();
step->text = nameOfStep;
step->takeSnapshot = true;
relevantTextFromTest.stepList.emplace_back(step);
} else if (lineStep.match(line).hasMatch()) {
QStringList tokens = line.split('"');
QString nameOfStep = tokens[1];
Step *step = new Step();
step->text = nameOfStep;
step->takeSnapshot = false;
relevantTextFromTest.stepList.emplace_back(step);
}
}
inputFile.close();
return relevantTextFromTest;
}
// Create an MD file for a user-selected test.
// The folder selected must contain a script named "test.js", the file produced is named "test.md"
void Test::createMDFile() {
// Folder selection
QString testDirectory = QFileDialog::getExistingDirectory(nullptr, "Please select folder containing the test images", ".", QFileDialog::ShowDirsOnly);
if (testDirectory == "") {
return;
}
// Verify folder contains test.js file
QString testFileName(testDirectory + "/" + TEST_FILENAME);
QFileInfo testFileInfo(testFileName);
if (!testFileInfo.exists()) {
messageBox.critical(0, "Error", "Could not find file: " + TEST_FILENAME);
return;
}
ExtractedText testScriptLines = getTestScriptLines(testFileName);
QString mdFilename(testDirectory + "/" + "test.md");
QFile mdFile(mdFilename);
if (!mdFile.open(QIODevice::ReadWrite)) {
messageBox.critical(0, "Internal error: " + QString(__FILE__) + ":" + QString::number(__LINE__), "Failed to create file " + mdFilename);
exit(-1);
}
QTextStream stream(&mdFile);
//Test title
QString testName = testScriptLines.title;
stream << "# " << testName << "\n";
// Find the relevant part of the path to the test (i.e. from "tests" down
QString partialPath = extractPathFromTestsDown(testDirectory);
stream << "## Run this script URL: [Manual](./test.js?raw=true) [Auto](./testAuto.js?raw=true)(from menu/Edit/Open and Run scripts from URL...)." << "\n\n";
stream << "## Preconditions" << "\n";
stream << "- In an empty region of a domain with editing rights." << "\n\n";
// Platform
QStringList platforms = testScriptLines.platform.split(" ");;
stream << "## Platforms\n";
stream << "Run the test on each of the following platforms\n";
for (int i = 0; i < platforms.size(); ++i) {
// Note that the platforms parameter may include extra spaces, these appear as empty strings in the list
if (platforms[i] != QString()) {
stream << " - " << platforms[i] << "\n";
}
}
// Display
QStringList displays = testScriptLines.display.split(" ");
stream << "## Displays\n";
stream << "Run the test on each of the following displays\n";
for (int i = 0; i < displays.size(); ++i) {
// Note that the displays parameter may include extra spaces, these appear as empty strings in the list
if (displays[i] != QString()) {
stream << " - " << displays[i] << "\n";
}
}
// CPU
QStringList cpus = testScriptLines.cpu.split(" ");
stream << "## Processors\n";
stream << "Run the test on each of the following processors\n";
for (int i = 0; i < cpus.size(); ++i) {
// Note that the cpus parameter may include extra spaces, these appear as empty strings in the list
if (cpus[i] != QString()) {
stream << " - " << cpus[i] << "\n";
}
}
// GPU
QStringList gpus = testScriptLines.gpu.split(" ");
stream << "## Graphics Cards\n";
stream << "Run the test on graphics cards from each of the following vendors\n";
for (int i = 0; i < gpus.size(); ++i) {
// Note that the gpus parameter may include extra spaces, these appear as empty strings in the list
if (gpus[i] != QString()) {
stream << " - " << gpus[i] << "\n";
}
}
stream << "## Steps\n";
stream << "Press space bar to advance step by step\n\n";
int snapShotIndex { 0 };
for (size_t i = 0; i < testScriptLines.stepList.size(); ++i) {
stream << "### Step " << QString::number(i) << "\n";
stream << "- " << testScriptLines.stepList[i]->text << "\n";
if (testScriptLines.stepList[i]->takeSnapshot) {
stream << "- ![](./ExpectedImage_" << QString::number(snapShotIndex).rightJustified(5, '0') << ".png)\n";
++snapShotIndex;
}
}
mdFile.close();
}
void Test::copyJPGtoPNG(QString sourceJPGFullFilename, QString destinationPNGFullFilename) {
QFile::remove(destinationPNGFullFilename);
@ -526,7 +722,7 @@ QString Test::getExpectedImagePartialSourceDirectory(QString filename) {
}
if (i < 0) {
messageBox.critical(0, "Internal error #9", "Bad filename");
messageBox.critical(0, "Internal error: " + QString(__FILE__) + ":" + QString::number(__LINE__), "Bad filename");
exit(-1);
}

30
tools/auto-tester/src/Test.h
View file

@ -19,6 +19,24 @@
#include "ImageComparer.h"
#include "ui/MismatchWindow.h"
class Step {
public:
QString text;
bool takeSnapshot;
};
using StepList = std::vector<Step*>;
class ExtractedText {
public:
QString title;
QString platform;
QString display;
QString cpu;
QString gpu;
StepList stepList;
};
class Test {
public:
Test();
@ -31,7 +49,7 @@ public:
void createRecursiveScript(QString topLevelDirectory, bool interactiveMode);
void createTest();
void deleteOldSnapshots();
void createMDFile();
bool compareImageLists(bool isInteractiveMode, QProgressBar* progressBar);
@ -47,7 +65,7 @@ public:
void zipAndDeleteTestResultsFolder();
bool isAValidDirectory(QString pathname);
QString extractPathFromTestsDown(QString fullPath);
QString getExpectedImageDestinationDirectory(QString filename);
QString getExpectedImagePartialSourceDirectory(QString filename);
@ -62,8 +80,6 @@ private:
QDir imageDirectory;
QRegularExpression expectedImageFilenameFormat;
MismatchWindow mismatchWindow;
ImageComparer imageComparer;
@ -81,9 +97,11 @@ private:
QStringList resultImagesFullFilenames;
// Used for accessing GitHub
const QString user { "NissimHadar" };
const QString branch { "addRecursionToAutotester" };
const QString githubUser{ "highfidelity" };
const QString gitHubBranch { "master" };
const QString DATETIME_FORMAT { "yyyy-MM-dd_hh-mm-ss" };
ExtractedText getTestScriptLines(QString testFileName);
};
#endif // hifi_test_h

6
tools/auto-tester/src/ui/AutoTester.cpp
View file

@ -33,7 +33,11 @@ void AutoTester::on_createRecursiveScriptsRecursivelyButton_clicked() {
}
void AutoTester::on_createTestButton_clicked() {
test->createTest();
test->createTest();
}
void AutoTester::on_createMDFileButton_clicked() {
test->createMDFile();
}
void AutoTester::on_closeButton_clicked() {

5
tools/auto-tester/src/ui/AutoTester.h
View file

@ -29,8 +29,9 @@ private slots:
void on_evaluateTestsButton_clicked();
void on_createRecursiveScriptButton_clicked();
void on_createRecursiveScriptsRecursivelyButton_clicked();
void on_createTestButton_clicked();
void on_closeButton_clicked();
void on_createTestButton_clicked();
void on_createMDFileButton_clicked();
void on_closeButton_clicked();
void saveImage(int index);

23
tools/auto-tester/src/ui/AutoTester.ui
View file

@ -7,7 +7,7 @@
<x>0</x>
<y>0</y>
<width>607</width>
<height>395</height>
<height>514</height>
</rect>
</property>
<property name="windowTitle">
@ -18,7 +18,7 @@
<property name="geometry">
<rect>
<x>20</x>
<y>300</y>
<y>420</y>
<width>220</width>
<height>40</height>
</rect>
@ -44,7 +44,7 @@
<property name="geometry">
<rect>
<x>20</x>
<y>135</y>
<y>255</y>
<width>220</width>
<height>40</height>
</rect>
@ -70,7 +70,7 @@
<property name="geometry">
<rect>
<x>23</x>
<y>100</y>
<y>220</y>
<width>131</width>
<height>20</height>
</rect>
@ -86,7 +86,7 @@
<property name="geometry">
<rect>
<x>20</x>
<y>190</y>
<y>310</y>
<width>255</width>
<height>23</height>
</rect>
@ -108,6 +108,19 @@
<string>Create Recursive Scripts Recursively</string>
</property>
</widget>
<widget class="QPushButton" name="createMDFileButton">
<property name="geometry">
<rect>
<x>20</x>
<y>90</y>
<width>220</width>
<height>40</height>
</rect>
</property>
<property name="text">
<string>Create MD file</string>
</property>
</widget>
</widget>
<widget class="QMenuBar" name="menuBar">
<property name="geometry">