From 2bcdb129c0598ca7cb4b0ec0e4b6f7eab7e7406d Mon Sep 17 00:00:00 2001 From: NissimHadar Date: Fri, 5 Oct 2018 18:31:49 -0700 Subject: [PATCH] Can create correct Python script to write HTML to AWS. --- tools/auto-tester/README.md | 3 + tools/auto-tester/src/AWSInterface.cpp | 109 ++++++++++++++++++------- tools/auto-tester/src/AWSInterface.h | 18 ++-- 3 files changed, 95 insertions(+), 35 deletions(-) diff --git a/tools/auto-tester/README.md b/tools/auto-tester/README.md index 9463df4b43..afd5d933e7 100644 --- a/tools/auto-tester/README.md +++ b/tools/auto-tester/README.md @@ -25,6 +25,9 @@ Python 3 can be downloaded from: 3. Mac (**macOS 64-bit/32-bit installer** or **macOS 64-bit/32-bit installer**) After installation - create an environment variable called PYTHON_PATH and set it to the folder containing the Python executable. +### AWS interface +Install the latest release of Boto3 via pip: +>pip install boto3 # Create ![](./Create.PNG) diff --git a/tools/auto-tester/src/AWSInterface.cpp b/tools/auto-tester/src/AWSInterface.cpp index f167a5a8ad..56ca00f73a 100644 --- a/tools/auto-tester/src/AWSInterface.cpp +++ b/tools/auto-tester/src/AWSInterface.cpp @@ -19,32 +19,38 @@ AWSInterface::AWSInterface(QObject* parent) : QObject(parent) { } -void AWSInterface::createWebPageFromResults(const QString& testResults, const QString& tempDirectory) { - // Extract test failures from zipped folder - _tempDirectory = tempDirectory; +void AWSInterface::createWebPageFromResults(const QString& testResults, const QString& workingDirectory) { + _testResults = testResults; + _workingDirectory = workingDirectory; - QDir().mkdir(_tempDirectory); - JlCompress::extractDir(testResults, _tempDirectory); + extractTestFailuresFromZippedFolder(); + createHTMLFile(); - createHTMLFile(testResults, tempDirectory); + if (QMessageBox::Yes == QMessageBox(QMessageBox::Information, "HTML has been created", "Do you want to update AWS?", QMessageBox::Yes | QMessageBox::No).exec()) { + updateAWS(); + } } -void AWSInterface::createHTMLFile(const QString& testResults, const QString& tempDirectory) { +void AWSInterface::extractTestFailuresFromZippedFolder() { + QDir().mkdir(_workingDirectory); + JlCompress::extractDir(_testResults, _workingDirectory); +} + +void AWSInterface::createHTMLFile() { // For file named `D:/tt/snapshots/TestResults--2018-10-03_15-35-28(9433)[DESKTOP-PMKNLSQ].zip` // - the HTML will be in a folder named `TestResults--2018-10-03_15-35-28(9433)[DESKTOP-PMKNLSQ]` - QStringList pathComponents = testResults.split('/'); + QStringList pathComponents = _testResults.split('/'); QString filename = pathComponents[pathComponents.length() - 1]; _resultsFolder = filename.left(filename.length() - 4); - QString resultsPath = tempDirectory + "/" + _resultsFolder + "/"; + QString resultsPath = _workingDirectory + "/" + _resultsFolder + "/"; QDir().mkdir(resultsPath); - QStringList tokens = testResults.split('/'); - QString htmlFilename = resultsPath + tokens[tokens.length() - 1].split('.')[0] + ".html"; + _htmlFilename = resultsPath + HTML_FILENAME; - QFile file(htmlFilename); + QFile file(_htmlFilename); if (!file.open(QIODevice::WriteOnly | QIODevice::Text)) { QMessageBox::critical(0, "Internal error: " + QString(__FILE__) + ":" + QString::number(__LINE__), - "Could not create '" + htmlFilename + "'"); + "Could not create '" + _htmlFilename + "'"); exit(-1); } @@ -52,7 +58,7 @@ void AWSInterface::createHTMLFile(const QString& testResults, const QString& tem startHTMLpage(stream); writeHead(stream); - writeBody(testResults, stream); + writeBody(stream); finishHTMLpage(stream); file.close(); @@ -73,9 +79,9 @@ void AWSInterface::writeHead(QTextStream& stream) { stream << "\t" << "\n"; } -void AWSInterface::writeBody(const QString& testResults, QTextStream& stream) { +void AWSInterface::writeBody(QTextStream& stream) { stream << "\t" << "\n"; - writeTitle(testResults, stream); + writeTitle(stream); writeTable(stream); stream << "\t" << "\n"; } @@ -84,10 +90,10 @@ void AWSInterface::finishHTMLpage(QTextStream& stream) { stream << "\n"; } -void AWSInterface::writeTitle(const QString& testResults, QTextStream& stream) { +void AWSInterface::writeTitle(QTextStream& stream) { // Separate relevant components from the results name // The expected format is as follows: `D:/tt/snapshots/TestResults--2018-10-04_11-09-41(PR14128)[DESKTOP-PMKNLSQ].zip` - QStringList tokens = testResults.split('/'); + QStringList tokens = _testResults.split('/'); // date_buildorPR_hostName will be 2018-10-03_15-35-28(9433)[DESKTOP-PMKNLSQ] QString date_buildorPR_hostName = tokens[tokens.length() - 1].split("--")[1].split(".")[0]; @@ -130,7 +136,7 @@ void AWSInterface::writeTable(QTextStream& stream) { // The fourth and lasts stage creates the HTML entries QStringList originalNames; - QDirIterator it1(_tempDirectory.toStdString().c_str()); + QDirIterator it1(_workingDirectory.toStdString().c_str()); while (it1.hasNext()) { QString nextDirectory = it1.next(); @@ -152,15 +158,14 @@ void AWSInterface::writeTable(QTextStream& stream) { newNames.append(originalNames[i].split("--tests.")[1]); } - QString htmlFolder{ _tempDirectory + "/" + _resultsFolder + "/" + FAILURE_FOLDER }; - QDir().mkdir(htmlFolder); + _htmlFolder = _workingDirectory + "/" + _resultsFolder + "/" + FAILURE_FOLDER; + QDir().mkdir(_htmlFolder); for (int i = 0; i < newNames.length(); ++i) { - QDir dir(originalNames[i]); - dir.rename(originalNames[i], htmlFolder + "/" + newNames[i]); + QDir().rename(originalNames[i], _htmlFolder + "/" + newNames[i]); } - QDirIterator it2((htmlFolder).toStdString().c_str()); + QDirIterator it2((_htmlFolder).toStdString().c_str()); while (it2.hasNext()) { QString nextDirectory = it2.next(); @@ -210,15 +215,61 @@ void AWSInterface::closeTable(QTextStream& stream) { void AWSInterface::createEntry(int index, const QString& testFailure, QTextStream& stream) { stream << "\t\t\t\n"; - stream << "\t\t\t\t\

" << QString::number(index) << "

\n"; + stream << "\t\t\t\t

" << QString::number(index) << "

\n"; // For a test named `D:/t/fgadhcUDHSFaidsfh3478JJJFSDFIUSOEIrf/Failure_1--tests.engine.interaction.pick.collision.many.00000` // we need `Failure_1--tests.engine.interaction.pick.collision.many.00000` QStringList failureNameComponents = testFailure.split('/'); QString failureName = failureNameComponents[failureNameComponents.length() - 1]; - stream << "\t\t\t\t\n"; - stream << "\t\t\t\t\n"; - stream << "\t\t\t\t\n"; + stream << "\t\t\t\t\n"; + stream << "\t\t\t\t\n"; + stream << "\t\t\t\t\n"; stream << "\t\t\t\n"; -} \ No newline at end of file +} + +void AWSInterface::updateAWS() { + QString filename = _workingDirectory + "/updateAWS.py"; + if (QFile::exists(filename)) { + QFile::remove(filename); + } + QFile file(filename); + + if (!file.open(QIODevice::WriteOnly | QIODevice::Text)) { + QMessageBox::critical(0, "Internal error: " + QString(__FILE__) + ":" + QString::number(__LINE__), + "Could not create 'addTestCases.py'"); + exit(-1); + } + + QTextStream stream(&file); + + stream << "import boto3\n"; + stream << "s3 = boto3.resource('s3')\n\n"; + + QDirIterator it1(_htmlFolder.toStdString().c_str()); + while (it1.hasNext()) { + QString nextDirectory = it1.next(); + + // Skip `.` and `..` directories + if (nextDirectory.right(1) == ".") { + continue; + } + + // nextDirectory looks like `D:/t/TestResults--2018-10-02_16-54-11(9426)[DESKTOP-PMKNLSQ]/failures/engine.render.effect.bloom.00000` + // We need to concatenate the last 3 components, to get `TestResults--2018-10-02_16-54-11(9426)[DESKTOP-PMKNLSQ]/failures/engine.render.effect.bloom.00000` + QStringList parts = nextDirectory.split('/'); + QString filename = parts[parts.length() - 3] + "/" + parts[parts.length() - 2] + "/" + parts[parts.length() - 1]; + + const QString imageNames[] { "Actual Image.png", "Expected Image.png", "Difference Image.png" }; + + for (int i = 0; i < 3; ++i) { + stream << "data = open('" << filename << "/" << imageNames[i] << "', 'rb')\n"; + stream << "s3.Bucket('hifi-content').put_object(Bucket='hifi-content', Key='nissim/" << filename << "/" << imageNames[i] << "', Body=data)\n\n"; + } + } + + stream << "data = open('" << _resultsFolder << "/" << HTML_FILENAME << "', 'rb')\n"; + stream << "s3.Bucket('hifi-content').put_object(Bucket='hifi-content', Key='nissim/" << _resultsFolder << "/" << HTML_FILENAME << "', Body=data)\n"; + + file.close(); +} diff --git a/tools/auto-tester/src/AWSInterface.h b/tools/auto-tester/src/AWSInterface.h index 1e2f3be556..0da9eae838 100644 --- a/tools/auto-tester/src/AWSInterface.h +++ b/tools/auto-tester/src/AWSInterface.h @@ -19,27 +19,33 @@ class AWSInterface : public QObject { public: explicit AWSInterface(QObject* parent = 0); - void createWebPageFromResults(const QString& testResults, const QString& tempDirectory); - - void createHTMLFile(const QString& testResults, const QString& tempDirectory); + void createWebPageFromResults(const QString& testResults, const QString& workingDirectory); + void extractTestFailuresFromZippedFolder(); + void createHTMLFile(); void startHTMLpage(QTextStream& stream); void writeHead(QTextStream& stream); - void writeBody(const QString& testResults, QTextStream& stream); + void writeBody(QTextStream& stream); void finishHTMLpage(QTextStream& stream); - void writeTitle(const QString& testResults, QTextStream& stream); + void writeTitle(QTextStream& stream); void writeTable(QTextStream& stream); void openTable(QTextStream& stream); void closeTable(QTextStream& stream); void createEntry(int index, const QString& testFailure, QTextStream& stream); + void updateAWS(); + private: - QString _tempDirectory; + QString _testResults; + QString _workingDirectory; QString _resultsFolder; + QString _htmlFolder; + QString _htmlFilename; const QString FAILURE_FOLDER{ "failures" }; + const QString HTML_FILENAME{ "TestResults.html" }; }; #endif // hifi_AWSInterface_h \ No newline at end of file