Mirrors/overte
3
0
Fork 0
mirror of https://github.com/overte-org/overte.git synced 2025-04-20 18:23:54 +02:00
Code Issues Projects Releases Packages Wiki Activity Actions 2

Merge pull request #15443 from Atlante45/fix/asset-server-baking-master

Browse source 
Case 22333: Master: Fix asset server auto-baking
This commit is contained in:
Shannon Romano 2019-04-24 16:30:14 -07:00 committed by GitHub
commit 70c2b8b8c4
No known key found for this signature in database
GPG key ID: 4AEE18F83AFDEB23
10 changed files with 407 additions and 368 deletions
Show stats Download patch file Download diff file Expand all files Collapse all files

334
assignment-client/src/assets/AssetServer.cpp
View file

@ -107,6 +107,10 @@ BakeVersion currentBakeVersionForAssetType(BakedAssetType type) {
}
}
QString getBakeMapping(const AssetUtils::AssetHash& hash, const QString& relativeFilePath) {
return AssetUtils::HIDDEN_BAKED_CONTENT_FOLDER + hash + "/" + relativeFilePath;
}
const QString ASSET_SERVER_LOGGING_TARGET_NAME = "asset-server";
void AssetServer::bakeAsset(const AssetUtils::AssetHash& assetHash, const AssetUtils::AssetPath& assetPath, const QString& filePath) {
@ -141,26 +145,27 @@ std::pair<AssetUtils::BakingStatus, QString> AssetServer::getAssetStatus(const A
return { AssetUtils::Baked, "" };
}
auto dotIndex = path.lastIndexOf(".");
if (dotIndex == -1) {
BakedAssetType type = assetTypeForFilename(path);
if (type == BakedAssetType::Undefined) {
return { AssetUtils::Irrelevant, "" };
}
auto extension = path.mid(dotIndex + 1);
bool loaded;
AssetMeta meta;
std::tie(loaded, meta) = readMetaFile(hash);
QString bakedFilename;
if (BAKEABLE_MODEL_EXTENSIONS.contains(extension)) {
bakedFilename = BAKED_MODEL_SIMPLE_NAME;
} else if (BAKEABLE_TEXTURE_EXTENSIONS.contains(extension.toLocal8Bit()) && hasMetaFile(hash)) {
bakedFilename = BAKED_TEXTURE_SIMPLE_NAME;
} else if (BAKEABLE_SCRIPT_EXTENSIONS.contains(extension)) {
bakedFilename = BAKED_SCRIPT_SIMPLE_NAME;
} else {
// We create a meta file for Skyboxes at runtime when they get requested
// Otherwise, textures don't get baked by themselves.
if (type == BakedAssetType::Texture && !loaded) {
return { AssetUtils::Irrelevant, "" };
}
auto bakedPath = AssetUtils::HIDDEN_BAKED_CONTENT_FOLDER + hash + "/" + bakedFilename;
QString bakedFilename = bakedFilenameForAssetType(type);
auto bakedPath = getBakeMapping(hash, bakedFilename);
if (loaded && !meta.redirectTarget.isEmpty()) {
bakedPath = meta.redirectTarget;
}
auto jt = _fileMappings.find(bakedPath);
if (jt != _fileMappings.end()) {
if (jt->second == hash) {
@ -168,14 +173,8 @@ std::pair<AssetUtils::BakingStatus, QString> AssetServer::getAssetStatus(const A
} else {
return { AssetUtils::Baked, "" };
}
} else {
bool loaded;
AssetMeta meta;
std::tie(loaded, meta) = readMetaFile(hash);
if (loaded && meta.failedLastBake) {
return { AssetUtils::Error, meta.lastBakeErrors };
}
} else if (loaded && meta.failedLastBake) {
return { AssetUtils::Error, meta.lastBakeErrors };
}
return { AssetUtils::Pending, "" };
@ -227,8 +226,16 @@ bool AssetServer::needsToBeBaked(const AssetUtils::AssetPath& path, const AssetU
return false;
}
bool loaded;
AssetMeta meta;
std::tie(loaded, meta) = readMetaFile(assetHash);
QString bakedFilename = bakedFilenameForAssetType(type);
auto bakedPath = AssetUtils::HIDDEN_BAKED_CONTENT_FOLDER + assetHash + "/" + bakedFilename;
auto bakedPath = getBakeMapping(assetHash, bakedFilename);
if (loaded && !meta.redirectTarget.isEmpty()) {
bakedPath = meta.redirectTarget;
}
auto mappingIt = _fileMappings.find(bakedPath);
bool bakedMappingExists = mappingIt != _fileMappings.end();
@ -238,10 +245,8 @@ bool AssetServer::needsToBeBaked(const AssetUtils::AssetPath& path, const AssetU
return false;
}
bool loaded;
AssetMeta meta;
std::tie(loaded, meta) = readMetaFile(assetHash);
// We create a meta file for Skyboxes at runtime when they get requested
// Otherwise, textures don't get baked by themselves.
if (type == BakedAssetType::Texture && !loaded) {
return false;
}
@ -633,36 +638,33 @@ void AssetServer::handleGetMappingOperation(ReceivedMessage& message, NLPacketLi
if (it != _fileMappings.end()) {
// check if we should re-direct to a baked asset
// first, figure out from the mapping extension what type of file this is
auto assetPathExtension = assetPath.mid(assetPath.lastIndexOf('.') + 1).toLower();
auto type = assetTypeForFilename(assetPath);
QString bakedRootFile = bakedFilenameForAssetType(type);
auto originalAssetHash = it->second;
QString redirectedAssetHash;
QString bakedAssetPath;
quint8 wasRedirected = false;
bool bakingDisabled = false;
if (!bakedRootFile.isEmpty()) {
// we ran into an asset for which we could have a baked version, let's check if it's ready
bakedAssetPath = AssetUtils::HIDDEN_BAKED_CONTENT_FOLDER + originalAssetHash + "/" + bakedRootFile;
auto bakedIt = _fileMappings.find(bakedAssetPath);
bool loaded;
AssetMeta meta;
std::tie(loaded, meta) = readMetaFile(originalAssetHash);
if (bakedIt != _fileMappings.end()) {
if (bakedIt->second != originalAssetHash) {
qDebug() << "Did find baked version for: " << originalAssetHash << assetPath;
// we found a baked version of the requested asset to serve, redirect to that
redirectedAssetHash = bakedIt->second;
wasRedirected = true;
} else {
qDebug() << "Did not find baked version for: " << originalAssetHash << assetPath << " (disabled)";
bakingDisabled = true;
}
auto type = assetTypeForFilename(assetPath);
QString bakedRootFile = bakedFilenameForAssetType(type);
QString bakedAssetPath = getBakeMapping(originalAssetHash, bakedRootFile);
if (loaded && !meta.redirectTarget.isEmpty()) {
bakedAssetPath = meta.redirectTarget;
}
auto bakedIt = _fileMappings.find(bakedAssetPath);
if (bakedIt != _fileMappings.end()) {
if (bakedIt->second != originalAssetHash) {
qDebug() << "Did find baked version for: " << originalAssetHash << assetPath;
// we found a baked version of the requested asset to serve, redirect to that
redirectedAssetHash = bakedIt->second;
wasRedirected = true;
} else {
qDebug() << "Did not find baked version for: " << originalAssetHash << assetPath;
qDebug() << "Did not find baked version for: " << originalAssetHash << assetPath << " (disabled)";
bakingDisabled = true;
}
}
@ -684,20 +686,13 @@ void AssetServer::handleGetMappingOperation(ReceivedMessage& message, NLPacketLi
auto query = QUrlQuery(url.query());
bool isSkybox = query.hasQueryItem("skybox");
if (isSkybox) {
bool loaded;
AssetMeta meta;
std::tie(loaded, meta) = readMetaFile(originalAssetHash);
if (!loaded) {
AssetMeta needsBakingMeta;
needsBakingMeta.bakeVersion = NEEDS_BAKING_BAKE_VERSION;
writeMetaFile(originalAssetHash, needsBakingMeta);
if (!bakingDisabled) {
maybeBake(assetPath, originalAssetHash);
}
if (isSkybox && !loaded) {
AssetMeta needsBakingMeta;
needsBakingMeta.bakeVersion = NEEDS_BAKING_BAKE_VERSION;
writeMetaFile(originalAssetHash, needsBakingMeta);
if (!bakingDisabled) {
maybeBake(assetPath, originalAssetHash);
}
}
}
@ -1297,14 +1292,6 @@ bool AssetServer::renameMapping(AssetUtils::AssetPath oldPath, AssetUtils::Asset
}
}
static const QString BAKED_ASSET_SIMPLE_FBX_NAME = "asset.fbx";
static const QString BAKED_ASSET_SIMPLE_TEXTURE_NAME = "texture.ktx";
static const QString BAKED_ASSET_SIMPLE_JS_NAME = "asset.js";
QString getBakeMapping(const AssetUtils::AssetHash& hash, const QString& relativeFilePath) {
return AssetUtils::HIDDEN_BAKED_CONTENT_FOLDER + hash + "/" + relativeFilePath;
}
void AssetServer::handleFailedBake(QString originalAssetHash, QString assetPath, QString errors) {
qDebug() << "Failed to bake: " << originalAssetHash << assetPath << "(" << errors << ")";
@ -1326,12 +1313,78 @@ void AssetServer::handleFailedBake(QString originalAssetHash, QString assetPath,
}
void AssetServer::handleCompletedBake(QString originalAssetHash, QString originalAssetPath,
QString bakedTempOutputDir, QVector<QString> bakedFilePaths) {
QString bakedTempOutputDir) {
auto reportCompletion = [this, originalAssetPath, originalAssetHash](bool errorCompletingBake,
QString errorReason,
QString redirectTarget) {
auto type = assetTypeForFilename(originalAssetPath);
auto currentTypeVersion = currentBakeVersionForAssetType(type);
AssetMeta meta;
meta.bakeVersion = currentTypeVersion;
meta.failedLastBake = errorCompletingBake;
meta.redirectTarget = redirectTarget;
if (errorCompletingBake) {
qWarning() << "Could not complete bake for" << originalAssetHash;
meta.lastBakeErrors = errorReason;
}
writeMetaFile(originalAssetHash, meta);
_pendingBakes.remove(originalAssetHash);
};
bool errorCompletingBake { false };
QString errorReason;
QString redirectTarget;
qDebug() << "Completing bake for " << originalAssetHash;
// Find the directory containing the baked content
QDir outputDir(bakedTempOutputDir);
QString outputDirName = outputDir.dirName();
auto directories = outputDir.entryList(QDir::Dirs | QDir::NoDotAndDotDot);
QString bakedDirectoryPath;
for (const auto& dirName : directories) {
outputDir.cd(dirName);
if (outputDir.exists("baked") && outputDir.exists("original")) {
bakedDirectoryPath = outputDir.filePath("baked");
break;
}
outputDir.cdUp();
}
if (bakedDirectoryPath.isEmpty()) {
errorCompletingBake = true;
errorReason = "Failed to find baking output";
// Cleanup temporary output directory
PathUtils::deleteMyTemporaryDir(outputDirName);
reportCompletion(errorCompletingBake, errorReason, redirectTarget);
return;
}
// Compile list of all the baked files
QDirIterator it(bakedDirectoryPath, QDirIterator::Subdirectories);
QVector<QString> bakedFilePaths;
while (it.hasNext()) {
it.next();
if (it.fileInfo().isFile()) {
bakedFilePaths.push_back(it.filePath());
}
}
if (bakedFilePaths.isEmpty()) {
errorCompletingBake = true;
errorReason = "Baking output has no files";
// Cleanup temporary output directory
PathUtils::deleteMyTemporaryDir(outputDirName);
reportCompletion(errorCompletingBake, errorReason, redirectTarget);
return;
}
QDir bakedDirectory(bakedDirectoryPath);
for (auto& filePath : bakedFilePaths) {
// figure out the hash for the contents of this file
QFile file(filePath);
@ -1340,89 +1393,72 @@ void AssetServer::handleCompletedBake(QString originalAssetHash, QString origina
AssetUtils::AssetHash bakedFileHash;
if (file.open(QIODevice::ReadOnly)) {
QCryptographicHash hasher(QCryptographicHash::Sha256);
if (hasher.addData(&file)) {
bakedFileHash = hasher.result().toHex();
} else {
// stop handling this bake, couldn't hash the contents of the file
errorCompletingBake = true;
errorReason = "Failed to finalize bake";
break;
}
// first check that we don't already have this bake file in our list
auto bakeFileDestination = _filesDirectory.absoluteFilePath(bakedFileHash);
if (!QFile::exists(bakeFileDestination)) {
// copy each to our files folder (with the hash as their filename)
if (!file.copy(_filesDirectory.absoluteFilePath(bakedFileHash))) {
// stop handling this bake, couldn't copy the bake file into our files directory
errorCompletingBake = true;
errorReason = "Failed to copy baked assets to asset server";
break;
}
}
// setup the mapping for this bake file
auto relativeFilePath = QUrl(filePath).fileName();
qDebug() << "Relative file path is: " << relativeFilePath;
if (relativeFilePath.endsWith(".fbx", Qt::CaseInsensitive)) {
// for an FBX file, we replace the filename with the simple name
// (to handle the case where two mapped assets have the same hash but different names)
relativeFilePath = BAKED_ASSET_SIMPLE_FBX_NAME;
} else if (relativeFilePath.endsWith(".js", Qt::CaseInsensitive)) {
relativeFilePath = BAKED_ASSET_SIMPLE_JS_NAME;
} else if (!originalAssetPath.endsWith(".fbx", Qt::CaseInsensitive)) {
relativeFilePath = BAKED_ASSET_SIMPLE_TEXTURE_NAME;
}
QString bakeMapping = getBakeMapping(originalAssetHash, relativeFilePath);
// add a mapping (under the hidden baked folder) for this file resulting from the bake
if (setMapping(bakeMapping, bakedFileHash)) {
qDebug() << "Added" << bakeMapping << "for bake file" << bakedFileHash << "from bake of" << originalAssetHash;
} else {
qDebug() << "Failed to set mapping";
// stop handling this bake, couldn't add a mapping for this bake file
errorCompletingBake = true;
errorReason = "Failed to finalize bake";
break;
}
} else {
if (!file.open(QIODevice::ReadOnly)) {
qDebug() << "Failed to open baked file: " << filePath;
// stop handling this bake, we couldn't open one of the files for reading
errorCompletingBake = true;
errorReason = "Failed to finalize bake";
errorReason = "Could not open baked file " + file.fileName();
break;
}
}
for (auto& filePath : bakedFilePaths) {
QFile file(filePath);
if (!file.remove()) {
qWarning() << "Failed to remove temporary file:" << filePath;
QCryptographicHash hasher(QCryptographicHash::Sha256);
if (!hasher.addData(&file)) {
// stop handling this bake, couldn't hash the contents of the file
errorCompletingBake = true;
errorReason = "Could not hash data for " + file.fileName();
break;
}
}
if (!QDir(bakedTempOutputDir).rmdir(".")) {
qWarning() << "Failed to remove temporary directory:" << bakedTempOutputDir;
bakedFileHash = hasher.result().toHex();
// first check that we don't already have this bake file in our list
auto bakeFileDestination = _filesDirectory.absoluteFilePath(bakedFileHash);
if (!QFile::exists(bakeFileDestination)) {
// copy each to our files folder (with the hash as their filename)
if (!file.copy(_filesDirectory.absoluteFilePath(bakedFileHash))) {
// stop handling this bake, couldn't copy the bake file into our files directory
errorCompletingBake = true;
errorReason = "Failed to copy baked assets to asset server";
break;
}
}
// setup the mapping for this bake file
auto relativeFilePath = bakedDirectory.relativeFilePath(filePath);
QString bakeMapping = getBakeMapping(originalAssetHash, relativeFilePath);
// Check if this is the file we should redirect to when someone asks for the original asset
if ((relativeFilePath.endsWith(".baked.fst", Qt::CaseInsensitive) && originalAssetPath.endsWith(".fbx")) ||
(relativeFilePath.endsWith(".texmeta.json", Qt::CaseInsensitive) && !originalAssetPath.endsWith(".fbx"))) {
if (!redirectTarget.isEmpty()) {
qWarning() << "Found multiple baked redirect target for" << originalAssetPath;
}
redirectTarget = bakeMapping;
}
// add a mapping (under the hidden baked folder) for this file resulting from the bake
if (!setMapping(bakeMapping, bakedFileHash)) {
qDebug() << "Failed to set mapping";
// stop handling this bake, couldn't add a mapping for this bake file
errorCompletingBake = true;
errorReason = "Failed to set mapping for baked file " + file.fileName();
break;
}
qDebug() << "Added" << bakeMapping << "for bake file" << bakedFileHash << "from bake of" << originalAssetHash;
}
auto type = assetTypeForFilename(originalAssetPath);
auto currentTypeVersion = currentBakeVersionForAssetType(type);
AssetMeta meta;
meta.bakeVersion = currentTypeVersion;
meta.failedLastBake = errorCompletingBake;
if (errorCompletingBake) {
qWarning() << "Could not complete bake for" << originalAssetHash;
meta.lastBakeErrors = errorReason;
if (redirectTarget.isEmpty()) {
errorCompletingBake = true;
errorReason = "Could not find root file for baked output";
}
writeMetaFile(originalAssetHash, meta);
_pendingBakes.remove(originalAssetHash);
// Cleanup temporary output directory
PathUtils::deleteMyTemporaryDir(outputDirName);
reportCompletion(errorCompletingBake, errorReason, redirectTarget);
}
void AssetServer::handleAbortedBake(QString originalAssetHash, QString assetPath) {
@ -1435,6 +1471,7 @@ void AssetServer::handleAbortedBake(QString originalAssetHash, QString assetPath
static const QString BAKE_VERSION_KEY = "bake_version";
static const QString FAILED_LAST_BAKE_KEY = "failed_last_bake";
static const QString LAST_BAKE_ERRORS_KEY = "last_bake_errors";
static const QString REDIRECT_TARGET_KEY = "redirect_target";
std::pair<bool, AssetMeta> AssetServer::readMetaFile(AssetUtils::AssetHash hash) {
auto metaFilePath = AssetUtils::HIDDEN_BAKED_CONTENT_FOLDER + hash + "/" + "meta.json";
@ -1461,6 +1498,7 @@ std::pair<bool, AssetMeta> AssetServer::readMetaFile(AssetUtils::AssetHash hash)
auto bakeVersion = root[BAKE_VERSION_KEY];
auto failedLastBake = root[FAILED_LAST_BAKE_KEY];
auto lastBakeErrors = root[LAST_BAKE_ERRORS_KEY];
auto redirectTarget = root[REDIRECT_TARGET_KEY];
if (bakeVersion.isDouble()
&& failedLastBake.isBool()
@ -1470,6 +1508,7 @@ std::pair<bool, AssetMeta> AssetServer::readMetaFile(AssetUtils::AssetHash hash)
meta.bakeVersion = bakeVersion.toInt();
meta.failedLastBake = failedLastBake.toBool();
meta.lastBakeErrors = lastBakeErrors.toString();
meta.redirectTarget = redirectTarget.toString();
return { true, meta };
} else {
@ -1488,6 +1527,7 @@ bool AssetServer::writeMetaFile(AssetUtils::AssetHash originalAssetHash, const A
metaFileObject[BAKE_VERSION_KEY] = (int)meta.bakeVersion;
metaFileObject[FAILED_LAST_BAKE_KEY] = meta.failedLastBake;
metaFileObject[LAST_BAKE_ERRORS_KEY] = meta.lastBakeErrors;
metaFileObject[REDIRECT_TARGET_KEY] = meta.redirectTarget;
QJsonDocument metaFileDoc;
metaFileDoc.setObject(metaFileObject);
@ -1521,10 +1561,18 @@ bool AssetServer::setBakingEnabled(const AssetUtils::AssetPathList& paths, bool
if (type == BakedAssetType::Undefined) {
continue;
}
QString bakedFilename = bakedFilenameForAssetType(type);
auto hash = it->second;
bool loaded;
AssetMeta meta;
std::tie(loaded, meta) = readMetaFile(hash);
QString bakedFilename = bakedFilenameForAssetType(type);
auto bakedMapping = getBakeMapping(hash, bakedFilename);
if (loaded && !meta.redirectTarget.isEmpty()) {
bakedMapping = meta.redirectTarget;
}
auto it = _fileMappings.find(bakedMapping);
bool currentlyDisabled = (it != _fileMappings.end() && it->second == hash);

7
assignment-client/src/assets/AssetServer.h
View file

@ -62,12 +62,10 @@ enum class ScriptBakeVersion : BakeVersion {
};
struct AssetMeta {
AssetMeta() {
}
BakeVersion bakeVersion { INITIAL_BAKE_VERSION };
bool failedLastBake { false };
QString lastBakeErrors;
QString redirectTarget;
};
class BakeAssetTask;
@ -139,8 +137,7 @@ private:
void bakeAsset(const AssetUtils::AssetHash& assetHash, const AssetUtils::AssetPath& assetPath, const QString& filePath);
/// Move baked content for asset to baked directory and update baked status
void handleCompletedBake(QString originalAssetHash, QString assetPath, QString bakedTempOutputDir,
QVector<QString> bakedFilePaths);
void handleCompletedBake(QString originalAssetHash, QString assetPath, QString bakedTempOutputDir);
void handleFailedBake(QString originalAssetHash, QString assetPath, QString errors);
void handleAbortedBake(QString originalAssetHash, QString assetPath);

56
assignment-client/src/assets/BakeAssetTask.cpp
View file

@ -36,33 +36,38 @@ BakeAssetTask::BakeAssetTask(const AssetUtils::AssetHash& assetHash, const Asset
});
}
void cleanupTempFiles(QString tempOutputDir, std::vector<QString> files) {
for (const auto& filename : files) {
QFile f { filename };
if (!f.remove()) {
qDebug() << "Failed to remove:" << filename;
}
}
if (!tempOutputDir.isEmpty()) {
QDir dir { tempOutputDir };
if (!dir.rmdir(".")) {
qDebug() << "Failed to remove temporary directory:" << tempOutputDir;
}
}
};
void BakeAssetTask::run() {
if (_isBaking.exchange(true)) {
qWarning() << "Tried to start bake asset task while already baking";
return;
}
// Make a new temporary directory for the Oven to work in
QString tempOutputDir = PathUtils::generateTemporaryDir();
QString tempOutputDirName = QDir(tempOutputDir).dirName();
if (tempOutputDir.isEmpty()) {
QString errors = "Could not create temporary working directory";
emit bakeFailed(_assetHash, _assetPath, errors);
PathUtils::deleteMyTemporaryDir(tempOutputDirName);
return;
}
// Copy file to bake the temporary dir and give a name the oven can work with
auto assetName = _assetPath.split("/").last();
auto tempAssetPath = tempOutputDir + "/" + assetName;
auto success = QFile::copy(_filePath, tempAssetPath);
if (!success) {
QString errors = "Couldn't copy file to bake to temporary directory";
emit bakeFailed(_assetHash, _assetPath, errors);
PathUtils::deleteMyTemporaryDir(tempOutputDirName);
return;
}
auto base = QFileInfo(QCoreApplication::applicationFilePath()).absoluteDir();
QString path = base.absolutePath() + "/oven";
QString extension = _assetPath.mid(_assetPath.lastIndexOf('.') + 1);
QStringList args {
"-i", _filePath,
"-i", tempAssetPath,
"-o", tempOutputDir,
"-t", extension,
};
@ -72,10 +77,11 @@ void BakeAssetTask::run() {
QEventLoop loop;
connect(_ovenProcess.get(), static_cast<void(QProcess::*)(int, QProcess::ExitStatus)>(&QProcess::finished),
this, [&loop, this, tempOutputDir](int exitCode, QProcess::ExitStatus exitStatus) {
this, [&loop, this, tempOutputDir, tempAssetPath, tempOutputDirName](int exitCode, QProcess::ExitStatus exitStatus) {
qDebug() << "Baking process finished: " << exitCode << exitStatus;
if (exitStatus == QProcess::CrashExit) {
PathUtils::deleteMyTemporaryDir(tempOutputDirName);
if (_wasAborted) {
emit bakeAborted(_assetHash, _assetPath);
} else {
@ -83,16 +89,10 @@ void BakeAssetTask::run() {
emit bakeFailed(_assetHash, _assetPath, errors);
}
} else if (exitCode == OVEN_STATUS_CODE_SUCCESS) {
QDir outputDir = tempOutputDir;
auto files = outputDir.entryInfoList(QDir::Files);
QVector<QString> outputFiles;
for (auto& file : files) {
outputFiles.push_back(file.absoluteFilePath());
}
emit bakeComplete(_assetHash, _assetPath, tempOutputDir, outputFiles);
emit bakeComplete(_assetHash, _assetPath, tempOutputDir);
} else if (exitStatus == QProcess::NormalExit && exitCode == OVEN_STATUS_CODE_ABORT) {
_wasAborted.store(true);
PathUtils::deleteMyTemporaryDir(tempOutputDirName);
emit bakeAborted(_assetHash, _assetPath);
} else {
QString errors;
@ -107,6 +107,7 @@ void BakeAssetTask::run() {
errors = "Unknown error occurred while baking";
}
}
PathUtils::deleteMyTemporaryDir(tempOutputDirName);
emit bakeFailed(_assetHash, _assetPath, errors);
}
@ -115,7 +116,10 @@ void BakeAssetTask::run() {
qDebug() << "Starting oven for " << _assetPath;
_ovenProcess->start(path, args, QIODevice::ReadOnly);
if (!_ovenProcess->waitForStarted(-1)) {
qDebug() << "Running:" << path << args;
if (!_ovenProcess->waitForStarted()) {
PathUtils::deleteMyTemporaryDir(tempOutputDirName);
QString errors = "Oven process failed to start";
emit bakeFailed(_assetHash, _assetPath, errors);
return;

2
assignment-client/src/assets/BakeAssetTask.h
View file

@ -37,7 +37,7 @@ public slots:
void abort();
signals:
void bakeComplete(QString assetHash, QString assetPath, QString tempOutputDir, QVector<QString> outputFiles);
void bakeComplete(QString assetHash, QString assetPath, QString tempOutputDir);
void bakeFailed(QString assetHash, QString assetPath, QString errors);
void bakeAborted(QString assetHash, QString assetPath);

6
libraries/material-networking/src/material-networking/TextureCache.cpp
View file

@ -635,11 +635,9 @@ void NetworkTexture::makeLocalRequest() {
}
bool NetworkTexture::handleFailedRequest(ResourceRequest::Result result) {
if (_currentlyLoadingResourceType != ResourceType::KTX
&& result == ResourceRequest::Result::RedirectFail) {
if (_shouldFailOnRedirect && result == ResourceRequest::Result::RedirectFail) {
auto newPath = _request->getRelativePathUrl();
if (newPath.fileName().endsWith(".ktx")) {
if (newPath.fileName().toLower().endsWith(".ktx")) {
_currentlyLoadingResourceType = ResourceType::KTX;
_activeUrl = newPath;
_shouldFailOnRedirect = false;

302
libraries/model-networking/src/model-networking/ModelCache.cpp
View file

@ -31,8 +31,6 @@
Q_LOGGING_CATEGORY(trace_resource_parse_geometry, "trace.resource.parse.geometry")
class GeometryReader;
class GeometryExtra {
public:
const GeometryMappingPair& mapping;
@ -87,113 +85,6 @@ namespace std {
};
}
QUrl resolveTextureBaseUrl(const QUrl& url, const QUrl& textureBaseUrl) {
return textureBaseUrl.isValid() ? textureBaseUrl : url;
}
class GeometryMappingResource : public GeometryResource {
Q_OBJECT
public:
GeometryMappingResource(const QUrl& url) : GeometryResource(url) {};
QString getType() const override { return "GeometryMapping"; }
virtual void downloadFinished(const QByteArray& data) override;
private slots:
void onGeometryMappingLoaded(bool success);
private:
GeometryResource::Pointer _geometryResource;
QMetaObject::Connection _connection;
};
void GeometryMappingResource::downloadFinished(const QByteArray& data) {
PROFILE_ASYNC_BEGIN(resource_parse_geometry, "GeometryMappingResource::downloadFinished", _url.toString(),
{ { "url", _url.toString() } });
// store parsed contents of FST file
_mapping = FSTReader::readMapping(data);
QString filename = _mapping.value("filename").toString();
if (filename.isNull()) {
finishedLoading(false);
} else {
const QString baseURL = _mapping.value("baseURL").toString();
const QUrl base = _effectiveBaseURL.resolved(baseURL);
QUrl url = base.resolved(filename);
QString texdir = _mapping.value(TEXDIR_FIELD).toString();
if (!texdir.isNull()) {
if (!texdir.endsWith('/')) {
texdir += '/';
}
_textureBaseUrl = resolveTextureBaseUrl(url, base.resolved(texdir));
} else {
_textureBaseUrl = url.resolved(QUrl("."));
}
auto scripts = FSTReader::getScripts(base, _mapping);
if (scripts.size() > 0) {
_mapping.remove(SCRIPT_FIELD);
for (auto &scriptPath : scripts) {
_mapping.insertMulti(SCRIPT_FIELD, scriptPath);
}
}
auto animGraphVariant = _mapping.value("animGraphUrl");
if (animGraphVariant.isValid()) {
QUrl fstUrl(animGraphVariant.toString());
if (fstUrl.isValid()) {
_animGraphOverrideUrl = base.resolved(fstUrl);
} else {
_animGraphOverrideUrl = QUrl();
}
} else {
_animGraphOverrideUrl = QUrl();
}
auto modelCache = DependencyManager::get<ModelCache>();
GeometryExtra extra { GeometryMappingPair(base, _mapping), _textureBaseUrl, false };
// Get the raw GeometryResource
_geometryResource = modelCache->getResource(url, QUrl(), &extra, std::hash<GeometryExtra>()(extra)).staticCast<GeometryResource>();
// Avoid caching nested resources - their references will be held by the parent
_geometryResource->_isCacheable = false;
if (_geometryResource->isLoaded()) {
onGeometryMappingLoaded(!_geometryResource->getURL().isEmpty());
} else {
if (_connection) {
disconnect(_connection);
}
_connection = connect(_geometryResource.data(), &Resource::finished,
this, &GeometryMappingResource::onGeometryMappingLoaded);
}
}
}
void GeometryMappingResource::onGeometryMappingLoaded(bool success) {
if (success && _geometryResource) {
_hfmModel = _geometryResource->_hfmModel;
_materialMapping = _geometryResource->_materialMapping;
_meshParts = _geometryResource->_meshParts;
_meshes = _geometryResource->_meshes;
_materials = _geometryResource->_materials;
// Avoid holding onto extra references
_geometryResource.reset();
// Make sure connection will not trigger again
disconnect(_connection); // FIXME Should not have to do this
}
PROFILE_ASYNC_END(resource_parse_geometry, "GeometryMappingResource::downloadFinished", _url.toString());
finishedLoading(success);
}
class GeometryReader : public QRunnable {
public:
GeometryReader(const ModelLoader& modelLoader, QWeakPointer<Resource>& resource, const QUrl& url, const GeometryMappingPair& mapping,
@ -308,47 +199,124 @@ void GeometryReader::run() {
}
}
class GeometryDefinitionResource : public GeometryResource {
Q_OBJECT
public:
GeometryDefinitionResource(const ModelLoader& modelLoader, const QUrl& url) : GeometryResource(url), _modelLoader(modelLoader) {}
GeometryDefinitionResource(const GeometryDefinitionResource& other) :
GeometryResource(other),
_modelLoader(other._modelLoader),
_mapping(other._mapping),
_combineParts(other._combineParts) {}
QUrl resolveTextureBaseUrl(const QUrl& url, const QUrl& textureBaseUrl) {
return textureBaseUrl.isValid() ? textureBaseUrl : url;
}
QString getType() const override { return "GeometryDefinition"; }
GeometryResource::GeometryResource(const GeometryResource& other) :
Resource(other),
Geometry(other),
_modelLoader(other._modelLoader),
_mappingPair(other._mappingPair),
_textureBaseURL(other._textureBaseURL),
_combineParts(other._combineParts),
_isCacheable(other._isCacheable)
{
if (other._geometryResource) {
_startedLoading = false;
}
}
virtual void downloadFinished(const QByteArray& data) override;
void GeometryResource::downloadFinished(const QByteArray& data) {
if (_effectiveBaseURL.fileName().toLower().endsWith(".fst")) {
PROFILE_ASYNC_BEGIN(resource_parse_geometry, "GeometryResource::downloadFinished", _url.toString(), { { "url", _url.toString() } });
void setExtra(void* extra) override;
// store parsed contents of FST file
_mapping = FSTReader::readMapping(data);
protected:
Q_INVOKABLE void setGeometryDefinition(HFMModel::Pointer hfmModel, const MaterialMapping& materialMapping);
QString filename = _mapping.value("filename").toString();
private:
ModelLoader _modelLoader;
GeometryMappingPair _mapping;
bool _combineParts;
};
if (filename.isNull()) {
finishedLoading(false);
} else {
const QString baseURL = _mapping.value("baseURL").toString();
const QUrl base = _effectiveBaseURL.resolved(baseURL);
QUrl url = base.resolved(filename);
void GeometryDefinitionResource::setExtra(void* extra) {
QString texdir = _mapping.value(TEXDIR_FIELD).toString();
if (!texdir.isNull()) {
if (!texdir.endsWith('/')) {
texdir += '/';
}
_textureBaseURL = resolveTextureBaseUrl(url, base.resolved(texdir));
} else {
_textureBaseURL = url.resolved(QUrl("."));
}
auto scripts = FSTReader::getScripts(base, _mapping);
if (scripts.size() > 0) {
_mapping.remove(SCRIPT_FIELD);
for (auto &scriptPath : scripts) {
_mapping.insertMulti(SCRIPT_FIELD, scriptPath);
}
}
auto animGraphVariant = _mapping.value("animGraphUrl");
if (animGraphVariant.isValid()) {
QUrl fstUrl(animGraphVariant.toString());
if (fstUrl.isValid()) {
_animGraphOverrideUrl = base.resolved(fstUrl);
} else {
_animGraphOverrideUrl = QUrl();
}
} else {
_animGraphOverrideUrl = QUrl();
}
auto modelCache = DependencyManager::get<ModelCache>();
GeometryExtra extra { GeometryMappingPair(base, _mapping), _textureBaseURL, false };
// Get the raw GeometryResource
_geometryResource = modelCache->getResource(url, QUrl(), &extra, std::hash<GeometryExtra>()(extra)).staticCast<GeometryResource>();
// Avoid caching nested resources - their references will be held by the parent
_geometryResource->_isCacheable = false;
if (_geometryResource->isLoaded()) {
onGeometryMappingLoaded(!_geometryResource->getURL().isEmpty());
} else {
if (_connection) {
disconnect(_connection);
}
_connection = connect(_geometryResource.data(), &Resource::finished, this, &GeometryResource::onGeometryMappingLoaded);
}
}
} else {
if (_url != _effectiveBaseURL) {
_url = _effectiveBaseURL;
_textureBaseURL = _effectiveBaseURL;
}
QThreadPool::globalInstance()->start(new GeometryReader(_modelLoader, _self, _effectiveBaseURL, _mappingPair, data, _combineParts, _request->getWebMediaType()));
}
}
void GeometryResource::onGeometryMappingLoaded(bool success) {
if (success && _geometryResource) {
_hfmModel = _geometryResource->_hfmModel;
_materialMapping = _geometryResource->_materialMapping;
_meshParts = _geometryResource->_meshParts;
_meshes = _geometryResource->_meshes;
_materials = _geometryResource->_materials;
// Avoid holding onto extra references
_geometryResource.reset();
// Make sure connection will not trigger again
disconnect(_connection); // FIXME Should not have to do this
}
PROFILE_ASYNC_END(resource_parse_geometry, "GeometryResource::downloadFinished", _url.toString());
finishedLoading(success);
}
void GeometryResource::setExtra(void* extra) {
const GeometryExtra* geometryExtra = static_cast<const GeometryExtra*>(extra);
_mapping = geometryExtra ? geometryExtra->mapping : GeometryMappingPair(QUrl(), QVariantHash());
_textureBaseUrl = geometryExtra ? resolveTextureBaseUrl(_url, geometryExtra->textureBaseUrl) : QUrl();
_mappingPair = geometryExtra ? geometryExtra->mapping : GeometryMappingPair(QUrl(), QVariantHash());
_textureBaseURL = geometryExtra ? resolveTextureBaseUrl(_url, geometryExtra->textureBaseUrl) : QUrl();
_combineParts = geometryExtra ? geometryExtra->combineParts : true;
}
void GeometryDefinitionResource::downloadFinished(const QByteArray& data) {
if (_url != _effectiveBaseURL) {
_url = _effectiveBaseURL;
_textureBaseUrl = _effectiveBaseURL;
}
QThreadPool::globalInstance()->start(new GeometryReader(_modelLoader, _self, _effectiveBaseURL, _mapping, data, _combineParts, _request->getWebMediaType()));
}
void GeometryDefinitionResource::setGeometryDefinition(HFMModel::Pointer hfmModel, const MaterialMapping& materialMapping) {
void GeometryResource::setGeometryDefinition(HFMModel::Pointer hfmModel, const MaterialMapping& materialMapping) {
// Assume ownership of the processed HFMModel
_hfmModel = hfmModel;
_materialMapping = materialMapping;
@ -357,7 +325,7 @@ void GeometryDefinitionResource::setGeometryDefinition(HFMModel::Pointer hfmMode
QHash<QString, size_t> materialIDAtlas;
for (const HFMMaterial& material : _hfmModel->materials) {
materialIDAtlas[material.materialID] = _materials.size();
_materials.push_back(std::make_shared<NetworkMaterial>(material, _textureBaseUrl));
_materials.push_back(std::make_shared<NetworkMaterial>(material, _textureBaseURL));
}
std::shared_ptr<GeometryMeshes> meshes = std::make_shared<GeometryMeshes>();
@ -380,6 +348,23 @@ void GeometryDefinitionResource::setGeometryDefinition(HFMModel::Pointer hfmMode
finishedLoading(true);
}
void GeometryResource::deleter() {
resetTextures();
Resource::deleter();
}
void GeometryResource::setTextures() {
if (_hfmModel) {
for (const HFMMaterial& material : _hfmModel->materials) {
_materials.push_back(std::make_shared<NetworkMaterial>(material, _textureBaseURL));
}
}
}
void GeometryResource::resetTextures() {
_materials.clear();
}
ModelCache::ModelCache() {
const qint64 GEOMETRY_DEFAULT_UNUSED_MAX_SIZE = DEFAULT_UNUSED_MAX_SIZE;
setUnusedResourceCacheSize(GEOMETRY_DEFAULT_UNUSED_MAX_SIZE);
@ -392,26 +377,14 @@ ModelCache::ModelCache() {
}
QSharedPointer<Resource> ModelCache::createResource(const QUrl& url) {
Resource* resource = nullptr;
if (url.path().toLower().endsWith(".fst")) {
resource = new GeometryMappingResource(url);
} else {
resource = new GeometryDefinitionResource(_modelLoader, url);
}
return QSharedPointer<Resource>(resource, &Resource::deleter);
return QSharedPointer<Resource>(new GeometryResource(url, _modelLoader), &GeometryResource::deleter);
}
QSharedPointer<Resource> ModelCache::createResourceCopy(const QSharedPointer<Resource>& resource) {
if (resource->getURL().path().toLower().endsWith(".fst")) {
return QSharedPointer<Resource>(new GeometryMappingResource(*resource.staticCast<GeometryMappingResource>()), &Resource::deleter);
} else {
return QSharedPointer<Resource>(new GeometryDefinitionResource(*resource.staticCast<GeometryDefinitionResource>()), &Resource::deleter);
}
return QSharedPointer<Resource>(new GeometryResource(*resource.staticCast<GeometryResource>()), &GeometryResource::deleter);
}
GeometryResource::Pointer ModelCache::getGeometryResource(const QUrl& url,
const GeometryMappingPair& mapping, const QUrl& textureBaseUrl) {
GeometryResource::Pointer ModelCache::getGeometryResource(const QUrl& url, const GeometryMappingPair& mapping, const QUrl& textureBaseUrl) {
bool combineParts = true;
GeometryExtra geometryExtra = { mapping, textureBaseUrl, combineParts };
GeometryResource::Pointer resource = getResource(url, QUrl(), &geometryExtra, std::hash<GeometryExtra>()(geometryExtra)).staticCast<GeometryResource>();
@ -531,23 +504,6 @@ const std::shared_ptr<NetworkMaterial> Geometry::getShapeMaterial(int partID) co
return nullptr;
}
void GeometryResource::deleter() {
resetTextures();
Resource::deleter();
}
void GeometryResource::setTextures() {
if (_hfmModel) {
for (const HFMMaterial& material : _hfmModel->materials) {
_materials.push_back(std::make_shared<NetworkMaterial>(material, _textureBaseUrl));
}
}
}
void GeometryResource::resetTextures() {
_materials.clear();
}
void GeometryResourceWatcher::startWatching() {
connect(_resource.data(), &Resource::finished, this, &GeometryResourceWatcher::resourceFinished);
connect(_resource.data(), &Resource::onRefresh, this, &GeometryResourceWatcher::resourceRefreshed);

42
libraries/model-networking/src/model-networking/ModelCache.h
View file

@ -24,8 +24,6 @@
class MeshPart;
class GeometryMappingResource;
using GeometryMappingPair = std::pair<QUrl, QVariantHash>;
Q_DECLARE_METATYPE(GeometryMappingPair)
@ -60,8 +58,6 @@ public:
const QVariantHash& getMapping() const { return _mapping; }
protected:
friend class GeometryMappingResource;
// Shared across all geometries, constant throughout lifetime
std::shared_ptr<const HFMModel> _hfmModel;
MaterialMapping _materialMapping;
@ -80,23 +76,29 @@ private:
/// A geometry loaded from the network.
class GeometryResource : public Resource, public Geometry {
Q_OBJECT
public:
using Pointer = QSharedPointer<GeometryResource>;
GeometryResource(const QUrl& url) : Resource(url) {}
GeometryResource(const GeometryResource& other) :
Resource(other),
Geometry(other),
_textureBaseUrl(other._textureBaseUrl),
_isCacheable(other._isCacheable) {}
GeometryResource(const QUrl& url, const ModelLoader& modelLoader) : Resource(url), _modelLoader(modelLoader) {}
GeometryResource(const GeometryResource& other);
virtual bool areTexturesLoaded() const override { return isLoaded() && Geometry::areTexturesLoaded(); }
QString getType() const override { return "Geometry"; }
virtual void deleter() override;
virtual void downloadFinished(const QByteArray& data) override;
void setExtra(void* extra) override;
virtual bool areTexturesLoaded() const override { return isLoaded() && Geometry::areTexturesLoaded(); }
private slots:
void onGeometryMappingLoaded(bool success);
protected:
friend class ModelCache;
friend class GeometryMappingResource;
Q_INVOKABLE void setGeometryDefinition(HFMModel::Pointer hfmModel, const MaterialMapping& materialMapping);
// Geometries may not hold onto textures while cached - that is for the texture cache
// Instead, these methods clear and reset textures from the geometry when caching/loading
@ -104,10 +106,18 @@ protected:
void setTextures();
void resetTextures();
QUrl _textureBaseUrl;
virtual bool isCacheable() const override { return _loaded && _isCacheable; }
bool _isCacheable { true };
private:
ModelLoader _modelLoader;
GeometryMappingPair _mappingPair;
QUrl _textureBaseURL;
bool _combineParts;
GeometryResource::Pointer _geometryResource;
QMetaObject::Connection _connection;
bool _isCacheable{ true };
};
class GeometryResourceWatcher : public QObject {
@ -158,7 +168,7 @@ public:
const QUrl& textureBaseUrl = QUrl());
protected:
friend class GeometryMappingResource;
friend class GeometryResource;
virtual QSharedPointer<Resource> createResource(const QUrl& url) override;
QSharedPointer<Resource> createResourceCopy(const QSharedPointer<Resource>& resource) override;

1
libraries/networking/src/ResourceCache.cpp
View file

@ -576,6 +576,7 @@ Resource::Resource(const Resource& other) :
Resource::Resource(const QUrl& url) :
_url(url),
_effectiveBaseURL(url),
_activeUrl(url),
_requestID(++requestID) {
init();

24
libraries/shared/src/PathUtils.cpp
View file

@ -185,6 +185,30 @@ QString PathUtils::generateTemporaryDir() {
return "";
}
bool PathUtils::deleteMyTemporaryDir(QString dirName) {
QDir rootTempDir = QDir::tempPath();
QString appName = qApp->applicationName();
QRegularExpression re { "^" + QRegularExpression::escape(appName) + "\\-(?<pid>\\d+)\\-(?<timestamp>\\d+)$" };
auto match = re.match(dirName);
auto pid = match.capturedRef("pid").toLongLong();
if (match.hasMatch() && rootTempDir.exists(dirName) && pid == qApp->applicationPid()) {
auto absoluteDirPath = QDir(rootTempDir.absoluteFilePath(dirName));
bool success = absoluteDirPath.removeRecursively();
if (success) {
qDebug() << " Removing temporary directory: " << absoluteDirPath.absolutePath();
} else {
qDebug() << " Failed to remove temporary directory: " << absoluteDirPath.absolutePath();
}
return success;
}
return false;
}
// Delete all temporary directories for an application
int PathUtils::removeTemporaryApplicationDirs(QString appName) {
if (appName.isNull()) {

1
libraries/shared/src/PathUtils.h
View file

@ -56,6 +56,7 @@ public:
static QString getAppLocalDataFilePath(const QString& filename);
static QString generateTemporaryDir();
static bool deleteMyTemporaryDir(QString dirName);
static int removeTemporaryApplicationDirs(QString appName = QString::null);