Merge pull request #11941 from huffman/feat/atp-baking-process

Move asset server baking to a separate process
This commit is contained in:
divya 2017-12-09 20:48:14 -08:00 committed by GitHub
commit e187852df1
No known key found for this signature in database
GPG key ID: 4AEE18F83AFDEB23
19 changed files with 228 additions and 87 deletions

View file

@ -13,9 +13,25 @@ setup_memory_debugger()
link_hifi_libraries(
audio avatars octree gpu model fbx entities
networking animation recording shared script-engine embedded-webserver
controllers physics plugins midi baking image
controllers physics plugins midi image
)
add_dependencies(${TARGET_NAME} oven)
if (WIN32)
add_custom_command(
TARGET ${TARGET_NAME} POST_BUILD
COMMAND ${CMAKE_COMMAND} -E copy_directory
$<TARGET_FILE_DIR:oven>
$<TARGET_FILE_DIR:${TARGET_NAME}>)
else()
add_custom_command(
TARGET ${TARGET_NAME} POST_BUILD
COMMAND ${CMAKE_COMMAND} -E create_symlink
$<TARGET_FILE:oven>
$<TARGET_FILE_DIR:${TARGET_NAME}>/oven)
endif()
if (WIN32)
package_libraries_for_deployment()
endif()

View file

@ -29,11 +29,10 @@
#include <QtCore/QUrlQuery>
#include <ClientServerUtils.h>
#include <FBXBaker.h>
#include <JSBaker.h>
#include <NodeType.h>
#include <SharedUtil.h>
#include <PathUtils.h>
#include <image/Image.h>
#include "AssetServerLogging.h"
#include "BakeAssetTask.h"
@ -250,7 +249,7 @@ AssetServer::AssetServer(ReceivedMessage& message) :
image::setNormalTexturesCompressionEnabled(true);
image::setCubeTexturesCompressionEnabled(true);
BAKEABLE_TEXTURE_EXTENSIONS = TextureBaker::getSupportedFormats();
BAKEABLE_TEXTURE_EXTENSIONS = image::getSupportedFormats();
qDebug() << "Supported baking texture formats:" << BAKEABLE_MODEL_EXTENSIONS;
// Most of the work will be I/O bound, reading from disk and constructing packet objects,
@ -416,6 +415,9 @@ void AssetServer::completeSetup() {
if (assetsFilesizeLimit != 0 && assetsFilesizeLimit < MAX_UPLOAD_SIZE) {
_filesizeLimit = assetsFilesizeLimit * BITS_PER_MEGABITS;
}
PathUtils::removeTemporaryApplicationDirs();
PathUtils::removeTemporaryApplicationDirs("Oven");
}
void AssetServer::cleanupUnmappedFiles() {

View file

@ -11,11 +11,18 @@
#include "BakeAssetTask.h"
#include <QtCore/QThread>
#include <mutex>
#include <QtCore/QThread>
#include <QCoreApplication>
#include <FBXBaker.h>
#include <PathUtils.h>
#include <JSBaker.h>
static const int OVEN_STATUS_CODE_SUCCESS { 0 };
static const int OVEN_STATUS_CODE_FAIL { 1 };
static const int OVEN_STATUS_CODE_ABORT { 2 };
std::once_flag registerMetaTypesFlag;
BakeAssetTask::BakeAssetTask(const AssetHash& assetHash, const AssetPath& assetPath, const QString& filePath) :
_assetHash(assetHash),
@ -23,6 +30,10 @@ BakeAssetTask::BakeAssetTask(const AssetHash& assetHash, const AssetPath& assetP
_filePath(filePath)
{
std::call_once(registerMetaTypesFlag, []() {
qRegisterMetaType<QProcess::ProcessError>("QProcess::ProcessError");
qRegisterMetaType<QProcess::ExitStatus>("QProcess::ExitStatus");
});
}
void cleanupTempFiles(QString tempOutputDir, std::vector<QString> files) {
@ -41,67 +52,76 @@ void cleanupTempFiles(QString tempOutputDir, std::vector<QString> files) {
};
void BakeAssetTask::run() {
_isBaking.store(true);
qRegisterMetaType<QVector<QString> >("QVector<QString>");
TextureBakerThreadGetter fn = []() -> QThread* { return QThread::currentThread(); };
QString tempOutputDir;
if (_assetPath.endsWith(".fbx")) {
tempOutputDir = PathUtils::generateTemporaryDir();
_baker = std::unique_ptr<FBXBaker> {
new FBXBaker(QUrl("file:///" + _filePath), fn, tempOutputDir)
};
} else if (_assetPath.endsWith(".js", Qt::CaseInsensitive)) {
_baker = std::unique_ptr<JSBaker>{
new JSBaker(QUrl("file:///" + _filePath), PathUtils::generateTemporaryDir())
};
} else {
tempOutputDir = PathUtils::generateTemporaryDir();
_baker = std::unique_ptr<TextureBaker> {
new TextureBaker(QUrl("file:///" + _filePath), image::TextureUsage::CUBE_TEXTURE,
tempOutputDir)
};
if (_isBaking.exchange(true)) {
qWarning() << "Tried to start bake asset task while already baking";
return;
}
QEventLoop loop;
connect(_baker.get(), &Baker::finished, &loop, &QEventLoop::quit);
connect(_baker.get(), &Baker::aborted, &loop, &QEventLoop::quit);
QMetaObject::invokeMethod(_baker.get(), "bake", Qt::QueuedConnection);
loop.exec();
QString tempOutputDir = PathUtils::generateTemporaryDir();
auto base = QFileInfo(QCoreApplication::applicationFilePath()).absoluteDir();
QString path = base.absolutePath() + "/oven";
QString extension = _assetPath.mid(_assetPath.lastIndexOf('.') + 1);
QStringList args {
"-i", _filePath,
"-o", tempOutputDir,
"-t", extension,
};
if (_baker->wasAborted()) {
qDebug() << "Aborted baking: " << _assetHash << _assetPath;
_ovenProcess.reset(new QProcess());
_wasAborted.store(true);
connect(_ovenProcess.get(), static_cast<void(QProcess::*)(int, QProcess::ExitStatus)>(&QProcess::finished),
this, [this, tempOutputDir](int exitCode, QProcess::ExitStatus exitStatus) {
qDebug() << "Baking process finished: " << exitCode << exitStatus;
cleanupTempFiles(tempOutputDir, _baker->getOutputFiles());
if (exitStatus == QProcess::CrashExit) {
if (_wasAborted) {
emit bakeAborted(_assetHash, _assetPath);
} else {
QString errors = "Fatal error occurred while baking";
emit bakeFailed(_assetHash, _assetPath, errors);
}
} else if (exitCode == OVEN_STATUS_CODE_SUCCESS) {
QDir outputDir = tempOutputDir;
auto files = outputDir.entryInfoList(QDir::Files);
QVector<QString> outputFiles;
for (auto& file : files) {
outputFiles.push_back(file.absoluteFilePath());
}
emit bakeAborted(_assetHash, _assetPath);
} else if (_baker->hasErrors()) {
qDebug() << "Failed to bake: " << _assetHash << _assetPath << _baker->getErrors();
emit bakeComplete(_assetHash, _assetPath, tempOutputDir, outputFiles);
} else if (exitStatus == QProcess::NormalExit && exitCode == OVEN_STATUS_CODE_ABORT) {
_wasAborted.store(true);
emit bakeAborted(_assetHash, _assetPath);
} else {
QString errors;
if (exitCode == OVEN_STATUS_CODE_FAIL) {
QDir outputDir = tempOutputDir;
auto errorFilePath = outputDir.absoluteFilePath("errors.txt");
QFile errorFile { errorFilePath };
if (errorFile.open(QIODevice::ReadOnly)) {
errors = errorFile.readAll();
errorFile.close();
} else {
errors = "Unknown error occurred while baking";
}
}
emit bakeFailed(_assetHash, _assetPath, errors);
}
auto errors = _baker->getErrors().join('\n'); // Join error list into a single string for convenience
_didFinish.store(true);
cleanupTempFiles(tempOutputDir, _baker->getOutputFiles());
});
qDebug() << "Starting oven for " << _assetPath;
_ovenProcess->start(path, args, QIODevice::ReadOnly);
if (!_ovenProcess->waitForStarted(-1)) {
QString errors = "Oven process failed to start";
emit bakeFailed(_assetHash, _assetPath, errors);
} else {
auto vectorOutputFiles = QVector<QString>::fromStdVector(_baker->getOutputFiles());
qDebug() << "Finished baking: " << _assetHash << _assetPath << vectorOutputFiles;
_didFinish.store(true);
emit bakeComplete(_assetHash, _assetPath, tempOutputDir, vectorOutputFiles);
return;
}
_ovenProcess->waitForFinished();
}
void BakeAssetTask::abort() {
if (_baker) {
_baker->abort();
if (!_wasAborted.exchange(true)) {
_ovenProcess->terminate();
}
}

View file

@ -17,9 +17,10 @@
#include <QtCore/QDebug>
#include <QtCore/QObject>
#include <QtCore/QRunnable>
#include <QDir>
#include <QProcess>
#include <AssetUtils.h>
#include <Baker.h>
class BakeAssetTask : public QObject, public QRunnable {
Q_OBJECT
@ -32,7 +33,6 @@ public:
void abort();
bool wasAborted() const { return _wasAborted.load(); }
bool didFinish() const { return _didFinish.load(); }
signals:
void bakeComplete(QString assetHash, QString assetPath, QString tempOutputDir, QVector<QString> outputFiles);
@ -44,9 +44,8 @@ private:
AssetHash _assetHash;
AssetPath _assetPath;
QString _filePath;
std::unique_ptr<Baker> _baker;
std::unique_ptr<QProcess> _ovenProcess { nullptr };
std::atomic<bool> _wasAborted { false };
std::atomic<bool> _didFinish { false };
};
#endif // hifi_BakeAssetTask_h

View file

@ -607,7 +607,7 @@ void FBXBaker::rewriteAndBakeSceneTextures() {
return;
}
if (!TextureBaker::getSupportedFormats().contains(textureFileInfo.suffix())) {
if (!image::getSupportedFormats().contains(textureFileInfo.suffix())) {
// this is a texture format we don't bake, skip it
handleWarning(fbxTextureFileName + " is not a bakeable texture format");
continue;

View file

@ -61,14 +61,6 @@ void TextureBaker::abort() {
_abortProcessing.store(true);
}
const QStringList TextureBaker::getSupportedFormats() {
auto formats = QImageReader::supportedImageFormats();
QStringList stringFormats;
std::transform(formats.begin(), formats.end(), std::back_inserter(stringFormats),
[](QByteArray& format) -> QString { return format; });
return stringFormats;
}
void TextureBaker::loadTexture() {
// check if the texture is local or first needs to be downloaded
if (_textureURL.isLocalFile()) {

View file

@ -31,8 +31,6 @@ public:
const QDir& outputDirectory, const QString& bakedFilename = QString(),
const QByteArray& textureContent = QByteArray());
static const QStringList getSupportedFormats();
const QByteArray& getOriginalTexture() const { return _originalTexture; }
QUrl getTextureURL() const { return _textureURL; }

View file

@ -1149,7 +1149,7 @@ bool ModelEntityRenderer::needsRenderUpdateFromTypedEntity(const TypedEntityPoin
if (model && model->isLoaded()) {
if (!entity->_dimensionsInitialized || entity->_needsInitialSimulation) {
return true;
}
}
// Check to see if we need to update the model bounds
if (entity->needsUpdateModelBounds()) {

View file

@ -75,6 +75,14 @@ glm::uvec2 rectifyToSparseSize(const glm::uvec2& size) {
namespace image {
const QStringList getSupportedFormats() {
auto formats = QImageReader::supportedImageFormats();
QStringList stringFormats;
std::transform(formats.begin(), formats.end(), std::back_inserter(stringFormats),
[](QByteArray& format) -> QString { return format; });
return stringFormats;
}
QImage::Format QIMAGE_HDR_FORMAT = QImage::Format_RGB30;
TextureUsage::TextureLoader TextureUsage::getTextureLoaderForType(Type type, const QVariantMap& options) {

View file

@ -80,6 +80,8 @@ gpu::TexturePointer processCubeTextureColorFromImage(const QImage& srcImage, con
} // namespace TextureUsage
const QStringList getSupportedFormats();
bool isColorTexturesCompressionEnabled();
bool isNormalTexturesCompressionEnabled();
bool isGrayscaleTexturesCompressionEnabled();

View file

@ -19,8 +19,14 @@
#include <QDir>
#include <QUrl>
#include <QtCore/QStandardPaths>
#include <QRegularExpression>
#include <mutex> // std::once
#include "shared/GlobalAppProperties.h"
#include "SharedUtil.h"
// Format: AppName-PID-Timestamp
// Example: ...
QString TEMP_DIR_FORMAT { "%1-%2-%3" };
const QString& PathUtils::resourcesPath() {
#ifdef Q_OS_MAC
@ -60,7 +66,8 @@ QString PathUtils::generateTemporaryDir() {
QString appName = qApp->applicationName();
for (auto i = 0; i < 64; ++i) {
auto now = std::chrono::system_clock::now().time_since_epoch().count();
QDir tempDir = rootTempDir.filePath(appName + "-" + QString::number(now));
auto dirName = TEMP_DIR_FORMAT.arg(appName).arg(qApp->applicationPid()).arg(now);
QDir tempDir = rootTempDir.filePath(dirName);
if (tempDir.mkpath(".")) {
return tempDir.absolutePath();
}
@ -68,6 +75,39 @@ QString PathUtils::generateTemporaryDir() {
return "";
}
// Delete all temporary directories for an application
int PathUtils::removeTemporaryApplicationDirs(QString appName) {
if (appName.isNull()) {
appName = qApp->applicationName();
}
auto dirName = TEMP_DIR_FORMAT.arg(appName).arg("*").arg("*");
QDir rootTempDir = QDir::tempPath();
auto dirs = rootTempDir.entryInfoList({ dirName }, QDir::Dirs);
int removed = 0;
for (auto& dir : dirs) {
auto dirName = dir.fileName();
auto absoluteDirPath = QDir(dir.absoluteFilePath());
QRegularExpression re { "^" + QRegularExpression::escape(appName) + "\\-(?<pid>\\d+)\\-(?<timestamp>\\d+)$" };
auto match = re.match(dirName);
if (match.hasMatch()) {
auto pid = match.capturedRef("pid").toLongLong();
auto timestamp = match.capturedRef("timestamp");
if (!processIsRunning(pid)) {
qDebug() << " Removing old temporary directory: " << dir.absoluteFilePath();
absoluteDirPath.removeRecursively();
removed++;
} else {
qDebug() << " Not removing (process is running): " << dir.absoluteFilePath();
}
}
}
return removed;
}
QString fileNameWithoutExtension(const QString& fileName, const QVector<QString> possibleExtensions) {
QString fileNameLowered = fileName.toLower();
foreach (const QString possibleExtension, possibleExtensions) {

View file

@ -39,6 +39,8 @@ public:
static QString generateTemporaryDir();
static int removeTemporaryApplicationDirs(QString appName = QString::null);
static Qt::CaseSensitivity getFSCaseSensitivity();
static QString stripFilename(const QUrl& url);
// note: this is FS-case-sensitive version of parentURL.isParentOf(childURL)

View file

@ -44,6 +44,12 @@ extern "C" FILE * __cdecl __iob_func(void) {
#include <CoreFoundation/CoreFoundation.h>
#endif
#if defined(Q_OS_LINUX) || defined(Q_OS_MAC)
#include <signal.h>
#include <cerrno>
#endif
#include <QtCore/QDebug>
#include <QDateTime>
#include <QElapsedTimer>
@ -1078,6 +1084,24 @@ void setMaxCores(uint8_t maxCores) {
#endif
}
bool processIsRunning(int64_t pid) {
#ifdef Q_OS_WIN
HANDLE process = OpenProcess(PROCESS_QUERY_LIMITED_INFORMATION, FALSE, pid);
if (process) {
DWORD exitCode;
if (GetExitCodeProcess(process, &exitCode) != 0) {
return exitCode == STILL_ACTIVE;
}
}
return false;
#else
if (kill(pid, 0) == -1) {
return errno != ESRCH;
}
return true;
#endif
}
void quitWithParentProcess() {
if (qApp) {
qDebug() << "Parent process died, quitting";

View file

@ -238,6 +238,8 @@ void setMaxCores(uint8_t maxCores);
const QString PARENT_PID_OPTION = "parent-pid";
void watchParentProcess(int parentPID);
bool processIsRunning(int64_t pid);
#ifdef Q_OS_WIN
void* createProcessGroup();

View file

@ -17,4 +17,4 @@ if (UNIX)
endif()
endif ()
set_target_properties(${TARGET_NAME} PROPERTIES EXCLUDE_FROM_ALL TRUE EXCLUDE_FROM_DEFAULT_BUILD TRUE)
install_beside_console()

View file

@ -12,6 +12,7 @@
#include <QObject>
#include <QImageReader>
#include <QtCore/QDebug>
#include <QFile>
#include "ModelBakingLoggingCategory.h"
#include "Oven.h"
@ -22,22 +23,30 @@
BakerCLI::BakerCLI(Oven* parent) : QObject(parent) {
}
void BakerCLI::bakeFile(QUrl inputUrl, const QString outputPath) {
void BakerCLI::bakeFile(QUrl inputUrl, const QString& outputPath, const QString& type) {
// if the URL doesn't have a scheme, assume it is a local file
if (inputUrl.scheme() != "http" && inputUrl.scheme() != "https" && inputUrl.scheme() != "ftp") {
inputUrl.setScheme("file");
}
static const QString MODEL_EXTENSION { ".fbx" };
qDebug() << "Baking file type: " << type;
static const QString MODEL_EXTENSION { "fbx" };
QString extension = type;
if (extension.isNull()) {
auto url = inputUrl.toDisplayString();
extension = url.mid(url.lastIndexOf('.'));
}
// check what kind of baker we should be creating
bool isFBX = inputUrl.toDisplayString().endsWith(MODEL_EXTENSION, Qt::CaseInsensitive);
bool isSupportedImage = false;
bool isFBX = extension == MODEL_EXTENSION;
for (QByteArray format : QImageReader::supportedImageFormats()) {
isSupportedImage |= inputUrl.toDisplayString().endsWith(format, Qt::CaseInsensitive);
}
bool isSupportedImage = QImageReader::supportedImageFormats().contains(extension.toLatin1());
_outputPath = outputPath;
// create our appropiate baker
if (isFBX) {
@ -48,7 +57,7 @@ void BakerCLI::bakeFile(QUrl inputUrl, const QString outputPath) {
_baker->moveToThread(qApp->getNextWorkerThread());
} else {
qCDebug(model_baking) << "Failed to determine baker type for file" << inputUrl;
QApplication::exit(1);
QApplication::exit(OVEN_STATUS_CODE_FAIL);
}
// invoke the bake method on the baker thread
@ -60,5 +69,17 @@ void BakerCLI::bakeFile(QUrl inputUrl, const QString outputPath) {
void BakerCLI::handleFinishedBaker() {
qCDebug(model_baking) << "Finished baking file.";
QApplication::exit(_baker.get()->hasErrors());
int exitCode = OVEN_STATUS_CODE_SUCCESS;
// Do we need this?
if (_baker->wasAborted()) {
exitCode = OVEN_STATUS_CODE_ABORT;
} else if (_baker->hasErrors()) {
exitCode = OVEN_STATUS_CODE_FAIL;
QFile errorFile { _outputPath.absoluteFilePath(OVEN_ERROR_FILENAME) };
if (errorFile.open(QFile::WriteOnly)) {
errorFile.write(_baker->getErrors().join('\n').toUtf8());
errorFile.close();
}
}
QApplication::exit(exitCode);
}

View file

@ -13,22 +13,32 @@
#define hifi_BakerCLI_h
#include <QtCore/QObject>
#include <QDir>
#include <memory>
#include "Baker.h"
#include "Oven.h"
static const int OVEN_STATUS_CODE_SUCCESS { 0 };
static const int OVEN_STATUS_CODE_FAIL { 1 };
static const int OVEN_STATUS_CODE_ABORT { 2 };
static const QString OVEN_ERROR_FILENAME = "errors.txt";
class BakerCLI : public QObject {
Q_OBJECT
public:
BakerCLI(Oven* parent);
void bakeFile(QUrl inputUrl, const QString outputPath);
void bakeFile(QUrl inputUrl, const QString& outputPath, const QString& type = QString::null);
private slots:
void handleFinishedBaker();
private:
QDir _outputPath;
std::unique_ptr<Baker> _baker;
};
#endif // hifi_BakerCLI_h
#endif // hifi_BakerCLI_h

View file

@ -24,6 +24,7 @@ static const QString OUTPUT_FOLDER = "/Users/birarda/code/hifi/lod/test-oven/exp
static const QString CLI_INPUT_PARAMETER = "i";
static const QString CLI_OUTPUT_PARAMETER = "o";
static const QString CLI_TYPE_PARAMETER = "t";
Oven::Oven(int argc, char* argv[]) :
QApplication(argc, argv)
@ -39,7 +40,8 @@ Oven::Oven(int argc, char* argv[]) :
parser.addOptions({
{ CLI_INPUT_PARAMETER, "Path to file that you would like to bake.", "input" },
{ CLI_OUTPUT_PARAMETER, "Path to folder that will be used as output.", "output" }
{ CLI_OUTPUT_PARAMETER, "Path to folder that will be used as output.", "output" },
{ CLI_TYPE_PARAMETER, "Type of asset.", "type" }
});
parser.addHelpOption();
parser.process(*this);
@ -59,7 +61,8 @@ Oven::Oven(int argc, char* argv[]) :
BakerCLI* cli = new BakerCLI(this);
QUrl inputUrl(QDir::fromNativeSeparators(parser.value(CLI_INPUT_PARAMETER)));
QUrl outputUrl(QDir::fromNativeSeparators(parser.value(CLI_OUTPUT_PARAMETER)));
cli->bakeFile(inputUrl, outputUrl.toString());
QString type = parser.isSet(CLI_TYPE_PARAMETER) ? parser.value(CLI_TYPE_PARAMETER) : QString::null;
cli->bakeFile(inputUrl, outputUrl.toString(), type);
} else {
parser.showHelp();
QApplication::quit();

View file

@ -14,6 +14,8 @@
#include <QtWidgets/QWidget>
#include <memory>
#include <Baker.h>
class BakeWidget : public QWidget {