This commit is contained in:
NissimHadar 2019-04-03 07:43:07 -07:00
parent 863fce6321
commit 813d467f49
4 changed files with 46 additions and 33 deletions

View file

@ -133,13 +133,14 @@ void Nitpick::setup() {
); );
} }
void Nitpick::startTestsEvaluation(const bool isRunningFromCommandLine, void Nitpick::startTestsEvaluation(
const bool isRunningFromCommandLine,
const bool isRunningInAutomaticTestRun, const bool isRunningInAutomaticTestRun,
const QString& snapshotDirectory, const QString& snapshotDirectory,
const QString& branch, const QString& branch,
const QString& user const QString& user
) { ) {
_testCreator->startTestsEvaluation(isRunningFromCommandLine, isRunningInAutomaticTestRun, snapshotDirectory, branch, user); _testCreator->startTestsEvaluation(_ui.clientProfileComboBox, isRunningFromCommandLine, isRunningInAutomaticTestRun, snapshotDirectory, branch, user);
} }
void Nitpick::on_tabWidget_currentChanged(int index) { void Nitpick::on_tabWidget_currentChanged(int index) {
@ -257,7 +258,7 @@ void Nitpick::on_showTaskbarPushbutton_clicked() {
} }
void Nitpick::on_evaluateTestsPushbutton_clicked() { void Nitpick::on_evaluateTestsPushbutton_clicked() {
_testCreator->startTestsEvaluation(false, false); _testCreator->startTestsEvaluation(_ui.clientProfileComboBox, false, false);
} }
void Nitpick::on_closePushbutton_clicked() { void Nitpick::on_closePushbutton_clicked() {

View file

@ -28,11 +28,13 @@ public:
void setup(); void setup();
void startTestsEvaluation(const bool isRunningFromCommandLine, void startTestsEvaluation(
const bool isRunningFromCommandLine,
const bool isRunningInAutomaticTestRun, const bool isRunningInAutomaticTestRun,
const QString& snapshotDirectory, const QString& snapshotDirectory,
const QString& branch, const QString& branch,
const QString& user); const QString& user
);
void automaticTestRunEvaluationComplete(QString zippedFolderName, int numberOfFailures); void automaticTestRunEvaluationComplete(QString zippedFolderName, int numberOfFailures);

View file

@ -67,7 +67,7 @@ QString TestCreator::zipAndDeleteTestResultsFolder() {
return zippedResultsFileName; return zippedResultsFileName;
} }
int TestCreator::compareImageLists() { int TestCreator::compareImageLists(const QString& gpuVendor) {
_progressBar->setMinimum(0); _progressBar->setMinimum(0);
_progressBar->setMaximum(_expectedImagesFullFilenames.length() - 1); _progressBar->setMaximum(_expectedImagesFullFilenames.length() - 1);
_progressBar->setValue(0); _progressBar->setValue(0);
@ -109,7 +109,15 @@ int TestCreator::compareImageLists() {
_mismatchWindow.setTestResult(testResult); _mismatchWindow.setTestResult(testResult);
if (similarityIndex < THRESHOLD_GLOBAL || worstTileValue < THRESHOLD_LOCAL) { // Lower threshold for non-Nvidia GPUs
double thresholdGlobal{ 0.9995 };
double thresholdLocal{ 0.6 };
if (gpuVendor != "Nvidia") {
thresholdGlobal = 0.97;
thresholdLocal = 0.2;
}
if (similarityIndex < thresholdGlobal || worstTileValue < thresholdLocal) {
if (!isInteractiveMode) { if (!isInteractiveMode) {
++numberOfFailures; ++numberOfFailures;
appendTestResultsToFile(testResult, _mismatchWindow.getComparisonImage(), _mismatchWindow.getSSIMResultsImage(testResult._ssimResults), true); appendTestResultsToFile(testResult, _mismatchWindow.getComparisonImage(), _mismatchWindow.getSSIMResultsImage(testResult._ssimResults), true);
@ -258,7 +266,9 @@ void::TestCreator::appendTestResultsToFile(QString testResultFilename, bool hasF
} }
} }
void TestCreator::startTestsEvaluation(const bool isRunningFromCommandLine, void TestCreator::startTestsEvaluation(
QComboBox *gpuVendor,
const bool isRunningFromCommandLine,
const bool isRunningInAutomaticTestRun, const bool isRunningInAutomaticTestRun,
const QString& snapshotDirectory, const QString& snapshotDirectory,
const QString& branchFromCommandLine, const QString& branchFromCommandLine,
@ -332,12 +342,12 @@ void TestCreator::startTestsEvaluation(const bool isRunningFromCommandLine,
} }
_downloader->downloadFiles(expectedImagesURLs, _snapshotDirectory, _expectedImagesFilenames, (void *)this); _downloader->downloadFiles(expectedImagesURLs, _snapshotDirectory, _expectedImagesFilenames, (void *)this);
finishTestsEvaluation(); finishTestsEvaluation(gpuVendor->currentText());
} }
void TestCreator::finishTestsEvaluation() { void TestCreator::finishTestsEvaluation(const QString& gpuVendor) {
// First - compare the pairs of images // First - compare the pairs of images
int numberOfFailures = compareImageLists(); int numberOfFailures = compareImageLists(gpuVendor);
// Next - check text results // Next - check text results
numberOfFailures += checkTextResults(); numberOfFailures += checkTextResults();

View file

@ -45,13 +45,16 @@ class TestCreator {
public: public:
TestCreator(QProgressBar* progressBar, QCheckBox* checkBoxInteractiveMode); TestCreator(QProgressBar* progressBar, QCheckBox* checkBoxInteractiveMode);
void startTestsEvaluation(const bool isRunningFromCommandLine, void startTestsEvaluation(
QComboBox *gpuVendor,
const bool isRunningFromCommandLine,
const bool isRunningInAutomaticTestRun, const bool isRunningInAutomaticTestRun,
const QString& snapshotDirectory = QString(), const QString& snapshotDirectory = QString(),
const QString& branchFromCommandLine = QString(), const QString& branchFromCommandLine = QString(),
const QString& userFromCommandLine = QString()); const QString& userFromCommandLine = QString()
);
void finishTestsEvaluation(); void finishTestsEvaluation(const QString& gpuVendor);
void createTests(const QString& clientProfile); void createTests(const QString& clientProfile);
@ -79,7 +82,7 @@ public:
void createRecursiveScript(); void createRecursiveScript();
void createRecursiveScript(const QString& directory, bool interactiveMode); void createRecursiveScript(const QString& directory, bool interactiveMode);
int compareImageLists(); int compareImageLists(const QString& gpuVendor);
int checkTextResults(); int checkTextResults();
QStringList createListOfAll_imagesInDirectory(const QString& imageFormat, const QString& pathToImageDirectory); QStringList createListOfAll_imagesInDirectory(const QString& imageFormat, const QString& pathToImageDirectory);
@ -124,9 +127,6 @@ private:
const QString TEST_RESULTS_FOLDER { "TestResults" }; const QString TEST_RESULTS_FOLDER { "TestResults" };
const QString TEST_RESULTS_FILENAME { "TestResults.txt" }; const QString TEST_RESULTS_FILENAME { "TestResults.txt" };
const double THRESHOLD_GLOBAL{ 0.9995 };
const double THRESHOLD_LOCAL { 0.6 };
QDir _imageDirectory; QDir _imageDirectory;
MismatchWindow _mismatchWindow; MismatchWindow _mismatchWindow;