Mirrors/overte-thingvellir
3
0
Fork 0
mirror of https://thingvellir.net/git/overte synced 2025-03-27 23:52:03 +01:00
Code Issues Projects Releases Packages Wiki Activity Actions

Merge pull request #11872 from huffman/feat/version-wo

Browse source 
Add asset server baking versioning
This commit is contained in:
Ryan Huffman 2018-03-23 08:27:37 -07:00 committed by GitHub
commit e90c051744
No known key found for this signature in database
GPG key ID: 4AEE18F83AFDEB23
5 changed files with 162 additions and 60 deletions
Show stats Download patch file Download diff file Expand all files Collapse all files

168
assignment-client/src/assets/AssetServer.cpp
View file

@ -46,15 +46,68 @@ static const uint8_t CPU_AFFINITY_COUNT_LOW = 1;
static const int INTERFACE_RUNNING_CHECK_FREQUENCY_MS = 1000;
#endif
const QString ASSET_SERVER_LOGGING_TARGET_NAME = "asset-server";
static const QStringList BAKEABLE_MODEL_EXTENSIONS = { "fbx" };
static QStringList BAKEABLE_TEXTURE_EXTENSIONS;
static const QStringList BAKEABLE_SCRIPT_EXTENSIONS = {};
static const QStringList BAKEABLE_SCRIPT_EXTENSIONS = { };
static const QString BAKED_MODEL_SIMPLE_NAME = "asset.fbx";
static const QString BAKED_TEXTURE_SIMPLE_NAME = "texture.ktx";
static const QString BAKED_SCRIPT_SIMPLE_NAME = "asset.js";
static const ModelBakeVersion CURRENT_MODEL_BAKE_VERSION = (ModelBakeVersion)((BakeVersion)ModelBakeVersion::COUNT - 1);
static const TextureBakeVersion CURRENT_TEXTURE_BAKE_VERSION = (TextureBakeVersion)((BakeVersion)TextureBakeVersion::COUNT - 1);
static const ScriptBakeVersion CURRENT_SCRIPT_BAKE_VERSION = (ScriptBakeVersion)((BakeVersion)ScriptBakeVersion::COUNT - 1);
BakedAssetType assetTypeForExtension(const QString& extension) {
auto extensionLower = extension.toLower();
if (BAKEABLE_MODEL_EXTENSIONS.contains(extensionLower)) {
return Model;
} else if (BAKEABLE_TEXTURE_EXTENSIONS.contains(extensionLower.toLocal8Bit())) {
return Texture;
} else if (BAKEABLE_SCRIPT_EXTENSIONS.contains(extensionLower)) {
return Script;
}
return Undefined;
}
BakedAssetType assetTypeForFilename(const QString& filename) {
auto dotIndex = filename.lastIndexOf(".");
if (dotIndex == -1) {
return BakedAssetType::Undefined;
}
auto extension = filename.mid(dotIndex + 1);
return assetTypeForExtension(extension);
}
QString bakedFilenameForAssetType(BakedAssetType type) {
switch (type) {
case Model:
return BAKED_MODEL_SIMPLE_NAME;
case Texture:
return BAKED_TEXTURE_SIMPLE_NAME;
case Script:
return BAKED_SCRIPT_SIMPLE_NAME;
default:
return "";
}
}
BakeVersion currentBakeVersionForAssetType(BakedAssetType type) {
switch (type) {
case Model:
return (BakeVersion)CURRENT_MODEL_BAKE_VERSION;
case Texture:
return (BakeVersion)CURRENT_TEXTURE_BAKE_VERSION;
case Script:
return (BakeVersion)CURRENT_SCRIPT_BAKE_VERSION;
default:
return 0;
}
}
const QString ASSET_SERVER_LOGGING_TARGET_NAME = "asset-server";
void AssetServer::bakeAsset(const AssetUtils::AssetHash& assetHash, const AssetUtils::AssetPath& assetPath, const QString& filePath) {
qDebug() << "Starting bake for: " << assetPath << assetHash;
auto it = _pendingBakes.find(assetHash);
@ -167,36 +220,38 @@ bool AssetServer::needsToBeBaked(const AssetUtils::AssetPath& path, const AssetU
return false;
}
auto dotIndex = path.lastIndexOf(".");
if (dotIndex == -1) {
BakedAssetType type = assetTypeForFilename(path);
if (type == Undefined) {
return false;
}
auto extension = path.mid(dotIndex + 1);
QString bakedFilename = bakedFilenameForAssetType(type);
auto bakedPath = AssetUtils::HIDDEN_BAKED_CONTENT_FOLDER + assetHash + "/" + bakedFilename;
auto mappingIt = _fileMappings.find(bakedPath);
bool bakedMappingExists = mappingIt != _fileMappings.end();
QString bakedFilename;
// If the path is mapped to the original file's hash, baking has been disabled for this
// asset
if (bakedMappingExists && mappingIt->second == assetHash) {
return false;
}
bool loaded;
AssetMeta meta;
std::tie(loaded, meta) = readMetaFile(assetHash);
// TODO: Allow failed bakes that happened on old versions to be re-baked
if (loaded && meta.failedLastBake) {
if (type == Texture && !loaded) {
return false;
}
if (BAKEABLE_MODEL_EXTENSIONS.contains(extension)) {
bakedFilename = BAKED_MODEL_SIMPLE_NAME;
} else if (loaded && BAKEABLE_TEXTURE_EXTENSIONS.contains(extension.toLocal8Bit())) {
bakedFilename = BAKED_TEXTURE_SIMPLE_NAME;
} else if (BAKEABLE_SCRIPT_EXTENSIONS.contains(extension)) {
bakedFilename = BAKED_SCRIPT_SIMPLE_NAME;
} else {
auto currentVersion = currentBakeVersionForAssetType(type);
if (loaded && (meta.failedLastBake && meta.bakeVersion >= currentVersion)) {
return false;
}
auto bakedPath = AssetUtils::HIDDEN_BAKED_CONTENT_FOLDER + assetHash + "/" + bakedFilename;
return _fileMappings.find(bakedPath) == _fileMappings.end();
return !bakedMappingExists || (meta.bakeVersion < currentVersion);
}
bool interfaceRunning() {
@ -598,15 +653,9 @@ void AssetServer::handleGetMappingOperation(ReceivedMessage& message, NLPacketLi
// first, figure out from the mapping extension what type of file this is
auto assetPathExtension = assetPath.mid(assetPath.lastIndexOf('.') + 1).toLower();
QString bakedRootFile;
if (BAKEABLE_MODEL_EXTENSIONS.contains(assetPathExtension)) {
bakedRootFile = BAKED_MODEL_SIMPLE_NAME;
} else if (BAKEABLE_TEXTURE_EXTENSIONS.contains(assetPathExtension.toLocal8Bit())) {
bakedRootFile = BAKED_TEXTURE_SIMPLE_NAME;
} else if (BAKEABLE_SCRIPT_EXTENSIONS.contains(assetPathExtension)) {
bakedRootFile = BAKED_SCRIPT_SIMPLE_NAME;
}
auto type = assetTypeForFilename(assetPath);
QString bakedRootFile = bakedFilenameForAssetType(type);
auto originalAssetHash = it->second;
QString redirectedAssetHash;
@ -653,9 +702,19 @@ void AssetServer::handleGetMappingOperation(ReceivedMessage& message, NLPacketLi
auto query = QUrlQuery(url.query());
bool isSkybox = query.hasQueryItem("skybox");
if (isSkybox) {
writeMetaFile(originalAssetHash);
if (!bakingDisabled) {
maybeBake(assetPath, originalAssetHash);
bool loaded;
AssetMeta meta;
std::tie(loaded, meta) = readMetaFile(originalAssetHash);
if (!loaded) {
AssetMeta needsBakingMeta;
needsBakingMeta.bakeVersion = NEEDS_BAKING_BAKE_VERSION;
writeMetaFile(originalAssetHash, needsBakingMeta);
if (!bakingDisabled) {
maybeBake(assetPath, originalAssetHash);
}
}
}
}
@ -1275,15 +1334,19 @@ QString getBakeMapping(const AssetUtils::AssetHash& hash, const QString& relativ
}
void AssetServer::handleFailedBake(QString originalAssetHash, QString assetPath, QString errors) {
qDebug() << "Failed: " << originalAssetHash << assetPath << errors;
qDebug() << "Failed to bake: " << originalAssetHash << assetPath << "(" << errors << ")";
bool loaded;
AssetMeta meta;
std::tie(loaded, meta) = readMetaFile(originalAssetHash);
auto type = assetTypeForFilename(assetPath);
auto currentTypeVersion = currentBakeVersionForAssetType(type);
meta.failedLastBake = true;
meta.lastBakeErrors = errors;
meta.bakeVersion = currentTypeVersion;
writeMetaFile(originalAssetHash, meta);
@ -1373,17 +1436,20 @@ void AssetServer::handleCompletedBake(QString originalAssetHash, QString origina
qWarning() << "Failed to remove temporary directory:" << bakedTempOutputDir;
}
if (!errorCompletingBake) {
// create the meta file to store which version of the baking process we just completed
writeMetaFile(originalAssetHash);
} else {
auto type = assetTypeForFilename(originalAssetPath);
auto currentTypeVersion = currentBakeVersionForAssetType(type);
AssetMeta meta;
meta.bakeVersion = currentTypeVersion;
meta.failedLastBake = errorCompletingBake;
if (errorCompletingBake) {
qWarning() << "Could not complete bake for" << originalAssetHash;
AssetMeta meta;
meta.failedLastBake = true;
meta.lastBakeErrors = errorReason;
writeMetaFile(originalAssetHash, meta);
}
writeMetaFile(originalAssetHash, meta);
_pendingBakes.remove(originalAssetHash);
}
@ -1447,7 +1513,7 @@ bool AssetServer::writeMetaFile(AssetUtils::AssetHash originalAssetHash, const A
// construct the JSON that will be in the meta file
QJsonObject metaFileObject;
metaFileObject[BAKE_VERSION_KEY] = meta.bakeVersion;
metaFileObject[BAKE_VERSION_KEY] = (int)meta.bakeVersion;
metaFileObject[FAILED_LAST_BAKE_KEY] = meta.failedLastBake;
metaFileObject[LAST_BAKE_ERRORS_KEY] = meta.lastBakeErrors;
@ -1479,27 +1545,13 @@ bool AssetServer::setBakingEnabled(const AssetUtils::AssetPathList& paths, bool
for (const auto& path : paths) {
auto it = _fileMappings.find(path);
if (it != _fileMappings.end()) {
auto type = assetTypeForFilename(path);
if (type == Undefined) {
continue;
}
QString bakedFilename = bakedFilenameForAssetType(type);
auto hash = it->second;
auto dotIndex = path.lastIndexOf(".");
if (dotIndex == -1) {
continue;
}
auto extension = path.mid(dotIndex + 1);
QString bakedFilename;
if (BAKEABLE_MODEL_EXTENSIONS.contains(extension)) {
bakedFilename = BAKED_MODEL_SIMPLE_NAME;
} else if (BAKEABLE_TEXTURE_EXTENSIONS.contains(extension.toLocal8Bit()) && hasMetaFile(hash)) {
bakedFilename = BAKED_TEXTURE_SIMPLE_NAME;
} else if (BAKEABLE_SCRIPT_EXTENSIONS.contains(extension)) {
bakedFilename = BAKED_SCRIPT_SIMPLE_NAME;
} else {
continue;
}
auto bakedMapping = getBakeMapping(hash, bakedFilename);
auto it = _fileMappings.find(bakedMapping);

41
assignment-client/src/assets/AssetServer.h
View file

@ -23,8 +23,47 @@
#include "RegisteredMetaTypes.h"
using BakeVersion = int;
static const BakeVersion INITIAL_BAKE_VERSION = 0;
static const BakeVersion NEEDS_BAKING_BAKE_VERSION = -1;
enum BakedAssetType : int {
Model = 0,
Texture,
Script,
NUM_ASSET_TYPES,
Undefined
};
// ATTENTION! If you change the current version for an asset type, you will also
// need to update the function currentBakeVersionForAssetType() inside of AssetServer.cpp.
enum class ModelBakeVersion : BakeVersion {
Initial = INITIAL_BAKE_VERSION,
COUNT
};
// ATTENTION! See above.
enum class TextureBakeVersion : BakeVersion {
Initial = INITIAL_BAKE_VERSION,
COUNT
};
// ATTENTION! See above.
enum class ScriptBakeVersion : BakeVersion {
Initial = INITIAL_BAKE_VERSION,
FixEmptyScripts,
COUNT
};
struct AssetMeta {
int bakeVersion { 0 };
AssetMeta() {
}
BakeVersion bakeVersion;
bool failedLastBake { false };
QString lastBakeErrors;
};

7
tools/oven/src/BakerCLI.cpp
View file

@ -18,6 +18,7 @@
#include "ModelBakingLoggingCategory.h"
#include "BakerCLI.h"
#include "FBXBaker.h"
#include "JSBaker.h"
#include "TextureBaker.h"
BakerCLI::BakerCLI(OvenCLIApplication* parent) : QObject(parent) {
@ -34,6 +35,7 @@ void BakerCLI::bakeFile(QUrl inputUrl, const QString& outputPath, const QString&
qDebug() << "Baking file type: " << type;
static const QString MODEL_EXTENSION { "fbx" };
static const QString SCRIPT_EXTENSION { "js" };
QString extension = type;
@ -44,6 +46,7 @@ void BakerCLI::bakeFile(QUrl inputUrl, const QString& outputPath, const QString&
// check what kind of baker we should be creating
bool isFBX = extension == MODEL_EXTENSION;
bool isScript = extension == SCRIPT_EXTENSION;
bool isSupportedImage = QImageReader::supportedImageFormats().contains(extension.toLatin1());
@ -57,12 +60,16 @@ void BakerCLI::bakeFile(QUrl inputUrl, const QString& outputPath, const QString&
outputPath)
};
_baker->moveToThread(Oven::instance().getNextWorkerThread());
} else if (isScript) {
_baker = std::unique_ptr<Baker> { new JSBaker(inputUrl, outputPath) };
_baker->moveToThread(Oven::instance().getNextWorkerThread());
} else if (isSupportedImage) {
_baker = std::unique_ptr<Baker> { new TextureBaker(inputUrl, image::TextureUsage::CUBE_TEXTURE, outputPath) };
_baker->moveToThread(Oven::instance().getNextWorkerThread());
} else {
qCDebug(model_baking) << "Failed to determine baker type for file" << inputUrl;
QCoreApplication::exit(OVEN_STATUS_CODE_FAIL);
return;
}
// invoke the bake method on the baker thread

3
tools/oven/src/BakerCLI.h
View file

@ -14,6 +14,7 @@
#include <QtCore/QObject>
#include <QDir>
#include <QUrl>
#include <memory>
@ -31,6 +32,8 @@ class BakerCLI : public QObject {
public:
BakerCLI(OvenCLIApplication* parent);
public slots:
void bakeFile(QUrl inputUrl, const QString& outputPath, const QString& type = QString::null);
private slots:

3
tools/oven/src/OvenCLIApplication.cpp
View file

@ -40,7 +40,8 @@ OvenCLIApplication::OvenCLIApplication(int argc, char* argv[]) :
QUrl inputUrl(QDir::fromNativeSeparators(parser.value(CLI_INPUT_PARAMETER)));
QUrl outputUrl(QDir::fromNativeSeparators(parser.value(CLI_OUTPUT_PARAMETER)));
QString type = parser.isSet(CLI_TYPE_PARAMETER) ? parser.value(CLI_TYPE_PARAMETER) : QString::null;
cli->bakeFile(inputUrl, outputUrl.toString(), type);
QMetaObject::invokeMethod(cli, "bakeFile", Qt::QueuedConnection, Q_ARG(QUrl, inputUrl),
Q_ARG(QString, outputUrl.toString()), Q_ARG(QString, type));
} else {
parser.showHelp();
QCoreApplication::quit();