Mirrors/overte-lubosz
3
0
Fork 0
mirror of https://github.com/lubosz/overte.git synced 2025-08-07 16:41:02 +02:00
Code Issues Projects Releases Packages Wiki Activity Actions

Merge branch 'master' of github.com:highfidelity/hifi into perfTest_1

Browse source
This commit is contained in:
NissimHadar 2018-05-14 10:44:57 -07:00
commit b5096758fb
2 changed files with 97 additions and 75 deletions
Show stats Download patch file Download diff file Expand all files Collapse all files

166
tools/auto-tester/src/Test.cpp
View file

@ -175,40 +175,39 @@ void Test::appendTestResultsToFile(const QString& testResultsFolderPath, TestFai
} }
void Test::startTestsEvaluation(const QString& testFolder) { void Test::startTestsEvaluation(const QString& testFolder) {
QString pathToTestResultsDirectory; // Get list of JPEG images in folder, sorted by name
if (testFolder.isNull()) { QString previousSelection = snapshotDirectory;
// Get list of JPEG images in folder, sorted by name
pathToTestResultsDirectory = QFileDialog::getExistingDirectory(nullptr, "Please select folder containing the test images", ".", QFileDialog::ShowDirsOnly);
} else {
pathToTestResultsDirectory = testFolder;
}
if (pathToTestResultsDirectory == QString()) { snapshotDirectory = QFileDialog::getExistingDirectory(nullptr, "Please select folder containing the test images",
previousSelection, QFileDialog::ShowDirsOnly);
// If user cancelled then restore previous selection and return
if (snapshotDirectory == "") {
snapshotDirectory = previousSelection;
return; return;
} }
// Quit if test results folder could not be created // Quit if test results folder could not be created
if (!createTestResultsFolderPath(pathToTestResultsDirectory)) { if (!createTestResultsFolderPath(snapshotDirectory)) {
return; return;
} }
// Before any processing - all images are converted to PNGs, as this is the format stored on GitHub // Before any processing - all images are converted to PNGs, as this is the format stored on GitHub
QStringList sortedSnapshotFilenames = createListOfAll_imagesInDirectory("jpg", pathToTestResultsDirectory); QStringList sortedSnapshotFilenames = createListOfAll_imagesInDirectory("jpg", snapshotDirectory);
foreach(QString filename, sortedSnapshotFilenames) { foreach(QString filename, sortedSnapshotFilenames) {
QStringList stringParts = filename.split("."); QStringList stringParts = filename.split(".");
copyJPGtoPNG( copyJPGtoPNG(snapshotDirectory + "/" + stringParts[0] + ".jpg",
pathToTestResultsDirectory + "/" + stringParts[0] + ".jpg", snapshotDirectory + "/" + stringParts[0] + ".png"
pathToTestResultsDirectory + "/" + stringParts[0] + ".png"
); );
QFile::remove(pathToTestResultsDirectory + "/" + stringParts[0] + ".jpg"); QFile::remove(snapshotDirectory + "/" + stringParts[0] + ".jpg");
} }
// Create two lists. The first is the test results, the second is the expected images // Create two lists. The first is the test results, the second is the expected images
// The expected images are represented as a URL to enable download from GitHub // The expected images are represented as a URL to enable download from GitHub
// Images that are in the wrong format are ignored. // Images that are in the wrong format are ignored.
QStringList sortedTestResultsFilenames = createListOfAll_imagesInDirectory("png", pathToTestResultsDirectory); QStringList sortedTestResultsFilenames = createListOfAll_imagesInDirectory("png", snapshotDirectory);
QStringList expectedImagesURLs; QStringList expectedImagesURLs;
resultImagesFullFilenames.clear(); resultImagesFullFilenames.clear();
@ -216,7 +215,7 @@ void Test::startTestsEvaluation(const QString& testFolder) {
expectedImagesFullFilenames.clear(); expectedImagesFullFilenames.clear();
foreach(QString currentFilename, sortedTestResultsFilenames) { foreach(QString currentFilename, sortedTestResultsFilenames) {
QString fullCurrentFilename = pathToTestResultsDirectory + "/" + currentFilename; QString fullCurrentFilename = snapshotDirectory + "/" + currentFilename;
if (isInSnapshotFilenameFormat("png", currentFilename)) { if (isInSnapshotFilenameFormat("png", currentFilename)) {
resultImagesFullFilenames << fullCurrentFilename; resultImagesFullFilenames << fullCurrentFilename;
@ -236,11 +235,11 @@ void Test::startTestsEvaluation(const QString& testFolder) {
QString expectedImageFilename = currentFilename.replace("/", "_").replace(".", "_EI."); QString expectedImageFilename = currentFilename.replace("/", "_").replace(".", "_EI.");
expectedImagesFilenames << expectedImageFilename; expectedImagesFilenames << expectedImageFilename;
expectedImagesFullFilenames << pathToTestResultsDirectory + "/" + expectedImageFilename; expectedImagesFullFilenames << snapshotDirectory + "/" + expectedImageFilename;
} }
} }
autoTester->downloadImages(expectedImagesURLs, pathToTestResultsDirectory, expectedImagesFilenames); autoTester->downloadImages(expectedImagesURLs, snapshotDirectory, expectedImagesFilenames);
} }
void Test::finishTestsEvaluation(bool isRunningFromCommandline, bool interactiveMode, QProgressBar* progressBar) { void Test::finishTestsEvaluation(bool isRunningFromCommandline, bool interactiveMode, QProgressBar* progressBar) {
@ -303,25 +302,39 @@ void Test::importTest(QTextStream& textStream, const QString& testPathname) {
// This script will run all text.js scripts in every applicable sub-folder // This script will run all text.js scripts in every applicable sub-folder
void Test::createRecursiveScript() { void Test::createRecursiveScript() {
// Select folder to start recursing from // Select folder to start recursing from
QString topLevelDirectory = QFileDialog::getExistingDirectory(nullptr, "Please select folder that will contain the top level test script", ".", QFileDialog::ShowDirsOnly); QString previousSelection = testDirectory;
if (topLevelDirectory == "") {
testDirectory =
QFileDialog::getExistingDirectory(nullptr, "Please select folder that will contain the top level test script",
previousSelection, QFileDialog::ShowDirsOnly);
// If user cancelled then restore previous selection and return
if (testDirectory == "") {
testDirectory = previousSelection;
return; return;
} }
createRecursiveScript(topLevelDirectory, true); createRecursiveScript(testDirectory, true);
} }
// This method creates a `testRecursive.js` script in every sub-folder. // This method creates a `testRecursive.js` script in every sub-folder.
void Test::createAllRecursiveScripts() { void Test::createAllRecursiveScripts() {
// Select folder to start recursing from // Select folder to start recursing from
QString topLevelDirectory = QFileDialog::getExistingDirectory(nullptr, "Please select the root folder for the recursive scripts", ".", QFileDialog::ShowDirsOnly); QString previousSelection = testDirectory;
if (topLevelDirectory == "") {
testDirectory = QFileDialog::getExistingDirectory(nullptr, "Please select the root folder for the recursive scripts",
previousSelection,
QFileDialog::ShowDirsOnly);
// If user cancelled then restore previous selection and return
if (testDirectory == "") {
testDirectory = previousSelection;
return; return;
} }
createRecursiveScript(topLevelDirectory, false); createRecursiveScript(testDirectory, false);
QDirIterator it(topLevelDirectory.toStdString().c_str(), QDirIterator::Subdirectories); QDirIterator it(testDirectory.toStdString().c_str(), QDirIterator::Subdirectories);
while (it.hasNext()) { while (it.hasNext()) {
QString directory = it.next(); QString directory = it.next();
@ -427,29 +440,42 @@ void Test::createRecursiveScript(const QString& topLevelDirectory, bool interact
void Test::createTest() { void Test::createTest() {
// Rename files sequentially, as ExpectedResult_00000.jpeg, ExpectedResult_00001.jpg and so on // Rename files sequentially, as ExpectedResult_00000.jpeg, ExpectedResult_00001.jpg and so on
// Any existing expected result images will be deleted // Any existing expected result images will be deleted
QString imageSourceDirectory = QFileDialog::getExistingDirectory(nullptr, "Please select folder containing the test images", ".", QFileDialog::ShowDirsOnly); QString previousSelection = snapshotDirectory;
if (imageSourceDirectory == "") {
snapshotDirectory = QFileDialog::getExistingDirectory(nullptr, "Please select folder containing the test images",
previousSelection,
QFileDialog::ShowDirsOnly);
// If user cancelled then restore previous selection and return
if (snapshotDirectory == "") {
snapshotDirectory = previousSelection;
return; return;
} }
QString imageDestinationDirectory = QFileDialog::getExistingDirectory(nullptr, "Please select folder to save the test images", ".", QFileDialog::ShowDirsOnly); previousSelection = testDirectory;
if (imageDestinationDirectory == "") {
QString testDirectory = QFileDialog::getExistingDirectory(nullptr, "Please select folder to save the test images",
previousSelection, QFileDialog::ShowDirsOnly);
// If user cancelled then restore previous selection and return
if (testDirectory == "") {
testDirectory = previousSelection;
return; return;
} }
QStringList sortedImageFilenames = createListOfAll_imagesInDirectory("jpg", imageSourceDirectory); QStringList sortedImageFilenames = createListOfAll_imagesInDirectory("jpg", snapshotDirectory);
int i = 1; int i = 1;
const int maxImages = pow(10, NUM_DIGITS); const int maxImages = pow(10, NUM_DIGITS);
foreach (QString currentFilename, sortedImageFilenames) { foreach (QString currentFilename, sortedImageFilenames) {
QString fullCurrentFilename = imageSourceDirectory + "/" + currentFilename; QString fullCurrentFilename = snapshotDirectory + "/" + currentFilename;
if (isInSnapshotFilenameFormat("jpg", currentFilename)) { if (isInSnapshotFilenameFormat("jpg", currentFilename)) {
if (i >= maxImages) { if (i >= maxImages) {
QMessageBox::critical(0, "Error", "More than " + QString::number(maxImages) + " images not supported"); QMessageBox::critical(0, "Error", "More than " + QString::number(maxImages) + " images not supported");
exit(-1); exit(-1);
} }
QString newFilename = "ExpectedImage_" + QString::number(i - 1).rightJustified(5, '0') + ".png"; QString newFilename = "ExpectedImage_" + QString::number(i - 1).rightJustified(5, '0') + ".png";
QString fullNewFileName = imageDestinationDirectory + "/" + newFilename; QString fullNewFileName = testDirectory + "/" + newFilename;
try { try {
copyJPGtoPNG(fullCurrentFilename, fullNewFileName); copyJPGtoPNG(fullCurrentFilename, fullNewFileName);
@ -489,31 +515,6 @@ ExtractedText Test::getTestScriptLines(QString testFileName) {
QString regexTestTitle(ws + functionPerformName + "\\(" + quotedString + "\\," + ws + ownPath + "\\," + ws + functionParameter + ws + "{" + ".*"); QString regexTestTitle(ws + functionPerformName + "\\(" + quotedString + "\\," + ws + ownPath + "\\," + ws + functionParameter + ws + "{" + ".*");
QRegularExpression lineContainingTitle = QRegularExpression(regexTestTitle); QRegularExpression lineContainingTitle = QRegularExpression(regexTestTitle);
// Assert platform checks that test is running on the correct OS
const QString functionAssertPlatform(ws + "autoTester" + ws + "\\." + ws + "assertPlatform");
const QString regexAssertPlatform(ws + functionAssertPlatform + ws + "\\(" + ws + quotedString + ".*");
const QRegularExpression lineAssertPlatform = QRegularExpression(regexAssertPlatform);
// Assert display checks that test is running on the correct display
const QString functionAssertDisplay(ws + "autoTester" + ws + "\\." + ws + "assertDisplay");
const QString regexAssertDisplay(ws + functionAssertDisplay + ws + "\\(" + ws + quotedString + ".*");
const QRegularExpression lineAssertDisplay = QRegularExpression(regexAssertDisplay);
// Assert CPU checks that test is running on the correct type of CPU
const QString functionAssertCPU(ws + "autoTester" + ws + "\\." + ws + "assertCPU");
const QString regexAssertCPU(ws + functionAssertCPU + ws + "\\(" + ws + quotedString + ".*");
const QRegularExpression lineAssertCPU = QRegularExpression(regexAssertCPU);
// Assert GPU checks that test is running on the correct type of GPU
const QString functionAssertGPU(ws + "autoTester" + ws + "\\." + ws + "assertGPU");
const QString regexAssertGPU(ws + functionAssertGPU + ws + "\\(" + ws + quotedString + ".*");
const QRegularExpression lineAssertGPU = QRegularExpression(regexAssertGPU);
// Assert the correct amount of memory
const QString functionAssertPhysicalMemoryGB(ws + "autoTester" + ws + "\\." + ws + "assertPhysicalMemoryGB");
const QString regexAssertPhysicalMemoryGB(ws + functionAssertPhysicalMemoryGB + ws + "\\(" + ws + quotedString + ".*");
const QRegularExpression lineAssertPhysicalMemoryGB = QRegularExpression(regexAssertPhysicalMemoryGB);
// Each step is either of the following forms: // Each step is either of the following forms:
// autoTester.addStepSnapshot("Take snapshot"... // autoTester.addStepSnapshot("Take snapshot"...
@ -523,7 +524,7 @@ ExtractedText Test::getTestScriptLines(QString testFileName) {
const QRegularExpression lineStepSnapshot = QRegularExpression(regexStepSnapshot); const QRegularExpression lineStepSnapshot = QRegularExpression(regexStepSnapshot);
const QString functionAddStepName(ws + "autoTester" + ws + "\\." + ws + "addStep"); const QString functionAddStepName(ws + "autoTester" + ws + "\\." + ws + "addStep");
const QString regexStep(ws + functionAddStepName + ws + "\\(" + ws + quotedString + ws + "\\)" + ".*"); const QString regexStep(ws + functionAddStepName + ws + "\\(" + ws + quotedString + ".*");
const QRegularExpression lineStep = QRegularExpression(regexStep); const QRegularExpression lineStep = QRegularExpression(regexStep);
while (!line.isNull()) { while (!line.isNull()) {
@ -531,7 +532,6 @@ ExtractedText Test::getTestScriptLines(QString testFileName) {
if (lineContainingTitle.match(line).hasMatch()) { if (lineContainingTitle.match(line).hasMatch()) {
QStringList tokens = line.split('"'); QStringList tokens = line.split('"');
relevantTextFromTest.title = tokens[1]; relevantTextFromTest.title = tokens[1];
} else if (lineStepSnapshot.match(line).hasMatch()) { } else if (lineStepSnapshot.match(line).hasMatch()) {
QStringList tokens = line.split('"'); QStringList tokens = line.split('"');
QString nameOfStep = tokens[1]; QString nameOfStep = tokens[1];
@ -561,29 +561,43 @@ ExtractedText Test::getTestScriptLines(QString testFileName) {
// The folder selected must contain a script named "test.js", the file produced is named "test.md" // The folder selected must contain a script named "test.js", the file produced is named "test.md"
void Test::createMDFile() { void Test::createMDFile() {
// Folder selection // Folder selection
QString testDirectory = QFileDialog::getExistingDirectory(nullptr, "Please select folder containing the test", ".", QFileDialog::ShowDirsOnly); QString previousSelection = testDirectory;
testDirectory = QFileDialog::getExistingDirectory(nullptr, "Please select folder containing the test", previousSelection,
QFileDialog::ShowDirsOnly);
// If user cancelled then restore previous selection and return
if (testDirectory == "") { if (testDirectory == "") {
testDirectory = previousSelection;
return; return;
} }
createMDFile(testDirectory); createMDFile(testDirectory);
QMessageBox::information(0, "Success", "MD file has been created");
} }
void Test::createAllMDFiles() { void Test::createAllMDFiles() {
// Select folder to start recursing from // Select folder to start recursing from
QString topLevelDirectory = QFileDialog::getExistingDirectory(nullptr, "Please select the root folder for the MD files", ".", QFileDialog::ShowDirsOnly); QString previousSelection = testDirectory;
if (topLevelDirectory == "") {
testDirectory = QFileDialog::getExistingDirectory(nullptr, "Please select the root folder for the MD files",
previousSelection, QFileDialog::ShowDirsOnly);
// If user cancelled then restore previous selection and return
if (testDirectory == "") {
testDirectory = previousSelection;
return; return;
} }
// First test if top-level folder has a test.js file // First test if top-level folder has a test.js file
const QString testPathname{ topLevelDirectory + "/" + TEST_FILENAME }; const QString testPathname { testDirectory + "/" + TEST_FILENAME };
QFileInfo fileInfo(testPathname); QFileInfo fileInfo(testPathname);
if (fileInfo.exists()) { if (fileInfo.exists()) {
createMDFile(topLevelDirectory); createMDFile(testDirectory);
} }
QDirIterator it(topLevelDirectory.toStdString().c_str(), QDirIterator::Subdirectories); QDirIterator it(testDirectory.toStdString().c_str(), QDirIterator::Subdirectories);
while (it.hasNext()) { while (it.hasNext()) {
QString directory = it.next(); QString directory = it.next();
@ -638,31 +652,33 @@ void Test::createMDFile(const QString& testDirectory) {
stream << "## Steps\n"; stream << "## Steps\n";
stream << "Press space bar to advance step by step\n\n"; stream << "Press space bar to advance step by step\n\n";
// Note that snapshots of step n are taken in step n+1
// (this implies that if the LAST step requests a snapshot then this will not work - caveat emptor)
int snapShotIndex { 0 }; int snapShotIndex { 0 };
for (size_t i = 0; i < testScriptLines.stepList.size(); ++i) { for (size_t i = 0; i < testScriptLines.stepList.size(); ++i) {
stream << "### Step " << QString::number(i + 1) << "\n"; stream << "### Step " << QString::number(i + 1) << "\n";
stream << "- " << testScriptLines.stepList[i]->text << "\n"; stream << "- " << testScriptLines.stepList[i]->text << "\n";
if ((i + 1 < testScriptLines.stepList.size()) && testScriptLines.stepList[i + 1]->takeSnapshot) { if ((i + 1 < testScriptLines.stepList.size()) && testScriptLines.stepList[i]->takeSnapshot) {
stream << "- ![](./ExpectedImage_" << QString::number(snapShotIndex).rightJustified(5, '0') << ".png)\n"; stream << "- ![](./ExpectedImage_" << QString::number(snapShotIndex).rightJustified(5, '0') << ".png)\n";
++snapShotIndex; ++snapShotIndex;
} }
} }
mdFile.close(); mdFile.close();
QMessageBox::information(0, "Success", "Test MD file " + mdFilename + " has been created");
} }
void Test::createTestsOutline() { void Test::createTestsOutline() {
QString testsRootDirectory = QFileDialog::getExistingDirectory(nullptr, "Please select the tests root folder", ".", QFileDialog::ShowDirsOnly); QString previousSelection = testDirectory;
if (testsRootDirectory == "") {
testDirectory = QFileDialog::getExistingDirectory(nullptr, "Please select the tests root folder", previousSelection,
QFileDialog::ShowDirsOnly);
// If user cancelled then restore previous selection and return
if (testDirectory == "") {
testDirectory = previousSelection;
return; return;
} }
const QString testsOutlineFilename { "testsOutline.md" }; const QString testsOutlineFilename { "testsOutline.md" };
QString mdFilename(testsRootDirectory + "/" + testsOutlineFilename); QString mdFilename(testDirectory + "/" + testsOutlineFilename);
QFile mdFile(mdFilename); QFile mdFile(mdFilename);
if (!mdFile.open(QIODevice::WriteOnly)) { if (!mdFile.open(QIODevice::WriteOnly)) {
QMessageBox::critical(0, "Internal error: " + QString(__FILE__) + ":" + QString::number(__LINE__), "Failed to create file " + mdFilename); QMessageBox::critical(0, "Internal error: " + QString(__FILE__) + ":" + QString::number(__LINE__), "Failed to create file " + mdFilename);
@ -676,10 +692,10 @@ void Test::createTestsOutline() {
stream << "Directories with an appended (*) have an automatic test\n\n"; stream << "Directories with an appended (*) have an automatic test\n\n";
// We need to know our current depth, as this isn't given by QDirIterator // We need to know our current depth, as this isn't given by QDirIterator
int rootDepth { testsRootDirectory.count('/') }; int rootDepth { testDirectory.count('/') };
// Each test is shown as the folder name linking to the matching GitHub URL, and the path to the associated test.md file // Each test is shown as the folder name linking to the matching GitHub URL, and the path to the associated test.md file
QDirIterator it(testsRootDirectory.toStdString().c_str(), QDirIterator::Subdirectories); QDirIterator it(testDirectory.toStdString().c_str(), QDirIterator::Subdirectories);
while (it.hasNext()) { while (it.hasNext()) {
QString directory = it.next(); QString directory = it.next();

6
tools/auto-tester/src/Test.h
View file

@ -89,6 +89,12 @@ private:
const int NUM_DIGITS { 5 }; const int NUM_DIGITS { 5 };
const QString EXPECTED_IMAGE_PREFIX { "ExpectedImage_" }; const QString EXPECTED_IMAGE_PREFIX { "ExpectedImage_" };
// We have two directories to work with.
// The first is the directory containing the test we are working with
// The second contains the snapshots taken for test runs that need to be evaluated
QString testDirectory;
QString snapshotDirectory;
QStringList expectedImagesFilenames; QStringList expectedImagesFilenames;
QStringList expectedImagesFullFilenames; QStringList expectedImagesFullFilenames;
QStringList resultImagesFullFilenames; QStringList resultImagesFullFilenames;