Merge pull request #11872 from huffman/feat/version-wo

Add asset server baking versioning

assignment-client/src/assets/AssetServer.cpp

@@ -46,15 +46,68 @@ static const uint8_t CPU_AFFINITY_COUNT_LOW = 1;
 static const int INTERFACE_RUNNING_CHECK_FREQUENCY_MS = 1000;
 #endif
 
-const QString ASSET_SERVER_LOGGING_TARGET_NAME = "asset-server";
-
 static const QStringList BAKEABLE_MODEL_EXTENSIONS = { "fbx" };
 static QStringList BAKEABLE_TEXTURE_EXTENSIONS;
-static const QStringList BAKEABLE_SCRIPT_EXTENSIONS = {};
+static const QStringList BAKEABLE_SCRIPT_EXTENSIONS = { };
+
 static const QString BAKED_MODEL_SIMPLE_NAME = "asset.fbx";
 static const QString BAKED_TEXTURE_SIMPLE_NAME = "texture.ktx";
 static const QString BAKED_SCRIPT_SIMPLE_NAME = "asset.js";
 
+static const ModelBakeVersion CURRENT_MODEL_BAKE_VERSION = (ModelBakeVersion)((BakeVersion)ModelBakeVersion::COUNT - 1);
+static const TextureBakeVersion CURRENT_TEXTURE_BAKE_VERSION = (TextureBakeVersion)((BakeVersion)TextureBakeVersion::COUNT - 1);
+static const ScriptBakeVersion CURRENT_SCRIPT_BAKE_VERSION = (ScriptBakeVersion)((BakeVersion)ScriptBakeVersion::COUNT - 1);
+
+BakedAssetType assetTypeForExtension(const QString& extension) {
+    auto extensionLower = extension.toLower();
+    if (BAKEABLE_MODEL_EXTENSIONS.contains(extensionLower)) {
+        return Model;
+    } else if (BAKEABLE_TEXTURE_EXTENSIONS.contains(extensionLower.toLocal8Bit())) {
+        return Texture;
+    } else if (BAKEABLE_SCRIPT_EXTENSIONS.contains(extensionLower)) {
+        return Script;
+    }
+    return Undefined;
+}
+
+BakedAssetType assetTypeForFilename(const QString& filename) {
+    auto dotIndex = filename.lastIndexOf(".");
+    if (dotIndex == -1) {
+        return BakedAssetType::Undefined;
+    }
+
+    auto extension = filename.mid(dotIndex + 1);
+    return assetTypeForExtension(extension);
+}
+
+QString bakedFilenameForAssetType(BakedAssetType type) {
+    switch (type) {
+        case Model:
+            return BAKED_MODEL_SIMPLE_NAME;
+        case Texture:
+            return BAKED_TEXTURE_SIMPLE_NAME;
+        case Script:
+            return BAKED_SCRIPT_SIMPLE_NAME;
+        default:
+            return "";
+    }
+}
+
+BakeVersion currentBakeVersionForAssetType(BakedAssetType type) {
+    switch (type) {
+        case Model:
+            return (BakeVersion)CURRENT_MODEL_BAKE_VERSION;
+        case Texture:
+            return (BakeVersion)CURRENT_TEXTURE_BAKE_VERSION;
+        case Script:
+            return (BakeVersion)CURRENT_SCRIPT_BAKE_VERSION;
+        default:
+            return 0;
+    }
+}
+
+const QString ASSET_SERVER_LOGGING_TARGET_NAME = "asset-server";
+
 void AssetServer::bakeAsset(const AssetUtils::AssetHash& assetHash, const AssetUtils::AssetPath& assetPath, const QString& filePath) {
     qDebug() << "Starting bake for: " << assetPath << assetHash;
     auto it = _pendingBakes.find(assetHash);
@@ -167,36 +220,38 @@ bool AssetServer::needsToBeBaked(const AssetUtils::AssetPath& path, const AssetU
         return false;
     }
 
-    auto dotIndex = path.lastIndexOf(".");
+    BakedAssetType type = assetTypeForFilename(path);
-    if (dotIndex == -1) {
+
+    if (type == Undefined) {
         return false;
     }
 
-    auto extension = path.mid(dotIndex + 1);
+    QString bakedFilename = bakedFilenameForAssetType(type);
+    auto bakedPath = AssetUtils::HIDDEN_BAKED_CONTENT_FOLDER + assetHash + "/" + bakedFilename;
+    auto mappingIt = _fileMappings.find(bakedPath);
+    bool bakedMappingExists = mappingIt != _fileMappings.end();
 
-    QString bakedFilename;
+    // If the path is mapped to the original file's hash, baking has been disabled for this
+    // asset
+    if (bakedMappingExists && mappingIt->second == assetHash) {
+        return false;
+    }
 
     bool loaded;
     AssetMeta meta;
     std::tie(loaded, meta) = readMetaFile(assetHash);
 
-    // TODO: Allow failed bakes that happened on old versions to be re-baked
+    if (type == Texture && !loaded) {
-    if (loaded && meta.failedLastBake) {
         return false;
     }
 
-    if (BAKEABLE_MODEL_EXTENSIONS.contains(extension)) {
+    auto currentVersion = currentBakeVersionForAssetType(type);
-        bakedFilename = BAKED_MODEL_SIMPLE_NAME;
+
-    } else if (loaded && BAKEABLE_TEXTURE_EXTENSIONS.contains(extension.toLocal8Bit())) {
+    if (loaded && (meta.failedLastBake && meta.bakeVersion >= currentVersion)) {
-        bakedFilename = BAKED_TEXTURE_SIMPLE_NAME;
-    } else if (BAKEABLE_SCRIPT_EXTENSIONS.contains(extension)) {
-        bakedFilename = BAKED_SCRIPT_SIMPLE_NAME;
-    } else {
         return false;
     }
 
-    auto bakedPath = AssetUtils::HIDDEN_BAKED_CONTENT_FOLDER + assetHash + "/" + bakedFilename;
+    return !bakedMappingExists || (meta.bakeVersion < currentVersion);
-    return _fileMappings.find(bakedPath) == _fileMappings.end();
 }
 
 bool interfaceRunning() {
@@ -598,15 +653,9 @@ void AssetServer::handleGetMappingOperation(ReceivedMessage& message, NLPacketLi
 
         // first, figure out from the mapping extension what type of file this is
         auto assetPathExtension = assetPath.mid(assetPath.lastIndexOf('.') + 1).toLower();
-        QString bakedRootFile;
 
-        if (BAKEABLE_MODEL_EXTENSIONS.contains(assetPathExtension)) {
+        auto type = assetTypeForFilename(assetPath);
-            bakedRootFile = BAKED_MODEL_SIMPLE_NAME;
+        QString bakedRootFile = bakedFilenameForAssetType(type);
-        } else if (BAKEABLE_TEXTURE_EXTENSIONS.contains(assetPathExtension.toLocal8Bit())) {
-            bakedRootFile = BAKED_TEXTURE_SIMPLE_NAME;
-        } else if (BAKEABLE_SCRIPT_EXTENSIONS.contains(assetPathExtension)) {
-            bakedRootFile = BAKED_SCRIPT_SIMPLE_NAME;
-        }
         
         auto originalAssetHash = it->second;
         QString redirectedAssetHash;
@@ -653,9 +702,19 @@ void AssetServer::handleGetMappingOperation(ReceivedMessage& message, NLPacketLi
             auto query = QUrlQuery(url.query());
             bool isSkybox = query.hasQueryItem("skybox");
             if (isSkybox) {
-                writeMetaFile(originalAssetHash);
+                bool loaded;
-                if (!bakingDisabled) {
+                AssetMeta meta;
-                    maybeBake(assetPath, originalAssetHash);
+                std::tie(loaded, meta) = readMetaFile(originalAssetHash);
+
+                if (!loaded) {
+                    AssetMeta needsBakingMeta;
+                    needsBakingMeta.bakeVersion = NEEDS_BAKING_BAKE_VERSION;
+
+                    writeMetaFile(originalAssetHash, needsBakingMeta);
+                    if (!bakingDisabled) {
+                        maybeBake(assetPath, originalAssetHash);
+                    }
+
                 }
             }
         }
@@ -1275,15 +1334,19 @@ QString getBakeMapping(const AssetUtils::AssetHash& hash, const QString& relativ
 }
 
 void AssetServer::handleFailedBake(QString originalAssetHash, QString assetPath, QString errors) {
-    qDebug() << "Failed: " << originalAssetHash << assetPath << errors;
+    qDebug() << "Failed to bake: " << originalAssetHash << assetPath << "(" << errors << ")";
 
     bool loaded;
     AssetMeta meta;
 
     std::tie(loaded, meta) = readMetaFile(originalAssetHash);
 
+    auto type = assetTypeForFilename(assetPath);
+    auto currentTypeVersion = currentBakeVersionForAssetType(type);
+
     meta.failedLastBake = true;
     meta.lastBakeErrors = errors;
+    meta.bakeVersion = currentTypeVersion;
 
     writeMetaFile(originalAssetHash, meta);
 
@@ -1373,17 +1436,20 @@ void AssetServer::handleCompletedBake(QString originalAssetHash, QString origina
         qWarning() << "Failed to remove temporary directory:" << bakedTempOutputDir;
     }
 
-    if (!errorCompletingBake) {
+    auto type = assetTypeForFilename(originalAssetPath);
-        // create the meta file to store which version of the baking process we just completed
+    auto currentTypeVersion = currentBakeVersionForAssetType(type);
-        writeMetaFile(originalAssetHash);
+
-    } else {
+    AssetMeta meta;
+    meta.bakeVersion = currentTypeVersion;
+    meta.failedLastBake = errorCompletingBake;
+
+    if (errorCompletingBake) {
         qWarning() << "Could not complete bake for" << originalAssetHash;
-        AssetMeta meta;
-        meta.failedLastBake = true;
         meta.lastBakeErrors = errorReason;
-        writeMetaFile(originalAssetHash, meta);
     }
 
+    writeMetaFile(originalAssetHash, meta);
+
     _pendingBakes.remove(originalAssetHash);
 }
 
@@ -1447,7 +1513,7 @@ bool AssetServer::writeMetaFile(AssetUtils::AssetHash originalAssetHash, const A
     // construct the JSON that will be in the meta file
     QJsonObject metaFileObject;
 
-    metaFileObject[BAKE_VERSION_KEY] = meta.bakeVersion;
+    metaFileObject[BAKE_VERSION_KEY] = (int)meta.bakeVersion;
     metaFileObject[FAILED_LAST_BAKE_KEY] = meta.failedLastBake;
     metaFileObject[LAST_BAKE_ERRORS_KEY] = meta.lastBakeErrors;
 
@@ -1479,27 +1545,13 @@ bool AssetServer::setBakingEnabled(const AssetUtils::AssetPathList& paths, bool
     for (const auto& path : paths) {
         auto it = _fileMappings.find(path);
         if (it != _fileMappings.end()) {
+            auto type = assetTypeForFilename(path);
+            if (type == Undefined) {
+                continue;
+            }
+            QString bakedFilename = bakedFilenameForAssetType(type);
+
             auto hash = it->second;
-
-            auto dotIndex = path.lastIndexOf(".");
-            if (dotIndex == -1) {
-                continue;
-            }
-
-            auto extension = path.mid(dotIndex + 1);
-
-            QString bakedFilename;
-    
-            if (BAKEABLE_MODEL_EXTENSIONS.contains(extension)) {
-                bakedFilename = BAKED_MODEL_SIMPLE_NAME;
-            } else if (BAKEABLE_TEXTURE_EXTENSIONS.contains(extension.toLocal8Bit()) && hasMetaFile(hash)) {
-                bakedFilename = BAKED_TEXTURE_SIMPLE_NAME;
-            } else if (BAKEABLE_SCRIPT_EXTENSIONS.contains(extension)) {
-                bakedFilename = BAKED_SCRIPT_SIMPLE_NAME;
-            } else {
-                continue;
-            }
-
             auto bakedMapping = getBakeMapping(hash, bakedFilename);
 
             auto it = _fileMappings.find(bakedMapping);

assignment-client/src/assets/AssetServer.h

@@ -23,8 +23,47 @@
 
 #include "RegisteredMetaTypes.h"
 
+using BakeVersion = int;
+static const BakeVersion INITIAL_BAKE_VERSION = 0;
+static const BakeVersion NEEDS_BAKING_BAKE_VERSION = -1;
+
+enum BakedAssetType : int {
+    Model = 0,
+    Texture,
+    Script,
+
+    NUM_ASSET_TYPES,
+    Undefined
+};
+
+// ATTENTION! If you change the current version for an asset type, you will also
+// need to update the function currentBakeVersionForAssetType() inside of AssetServer.cpp.
+enum class ModelBakeVersion : BakeVersion {
+    Initial = INITIAL_BAKE_VERSION,
+
+    COUNT
+};
+
+// ATTENTION! See above.
+enum class TextureBakeVersion : BakeVersion {
+    Initial = INITIAL_BAKE_VERSION,
+
+    COUNT
+};
+
+// ATTENTION! See above.
+enum class ScriptBakeVersion : BakeVersion {
+    Initial = INITIAL_BAKE_VERSION,
+    FixEmptyScripts,
+
+    COUNT
+};
+
 struct AssetMeta {
-    int bakeVersion { 0 };
+    AssetMeta() {
+    }
+
+    BakeVersion bakeVersion;
     bool failedLastBake { false };
     QString lastBakeErrors;
 };

tools/oven/src/BakerCLI.cpp

@@ -18,6 +18,7 @@
 #include "ModelBakingLoggingCategory.h"
 #include "BakerCLI.h"
 #include "FBXBaker.h"
+#include "JSBaker.h"
 #include "TextureBaker.h"
 
 BakerCLI::BakerCLI(OvenCLIApplication* parent) : QObject(parent) {
@@ -34,6 +35,7 @@ void BakerCLI::bakeFile(QUrl inputUrl, const QString& outputPath, const QString&
     qDebug() << "Baking file type: " << type;
 
     static const QString MODEL_EXTENSION { "fbx" };
+    static const QString SCRIPT_EXTENSION { "js" };
 
     QString extension = type;
 
@@ -44,6 +46,7 @@ void BakerCLI::bakeFile(QUrl inputUrl, const QString& outputPath, const QString&
 
     // check what kind of baker we should be creating
     bool isFBX = extension == MODEL_EXTENSION;
+    bool isScript = extension == SCRIPT_EXTENSION;
 
     bool isSupportedImage = QImageReader::supportedImageFormats().contains(extension.toLatin1());
 
@@ -57,12 +60,16 @@ void BakerCLI::bakeFile(QUrl inputUrl, const QString& outputPath, const QString&
                          outputPath)
         };
         _baker->moveToThread(Oven::instance().getNextWorkerThread());
+    } else if (isScript) {
+        _baker = std::unique_ptr<Baker> { new JSBaker(inputUrl, outputPath) };
+        _baker->moveToThread(Oven::instance().getNextWorkerThread());
     } else if (isSupportedImage) {
         _baker = std::unique_ptr<Baker> { new TextureBaker(inputUrl, image::TextureUsage::CUBE_TEXTURE, outputPath) };
         _baker->moveToThread(Oven::instance().getNextWorkerThread());
     } else {
         qCDebug(model_baking) << "Failed to determine baker type for file" << inputUrl;
         QCoreApplication::exit(OVEN_STATUS_CODE_FAIL);
+        return;
     }
 
     // invoke the bake method on the baker thread

tools/oven/src/BakerCLI.h

@@ -14,6 +14,7 @@
 
 #include <QtCore/QObject>
 #include <QDir>
+#include <QUrl>
 
 #include <memory>
 
@@ -31,6 +32,8 @@ class BakerCLI : public QObject {
 
 public:
     BakerCLI(OvenCLIApplication* parent);
+
+public slots:
     void bakeFile(QUrl inputUrl, const QString& outputPath, const QString& type = QString::null);
 
 private slots:

tools/oven/src/OvenCLIApplication.cpp

@@ -40,7 +40,8 @@ OvenCLIApplication::OvenCLIApplication(int argc, char* argv[]) :
         QUrl inputUrl(QDir::fromNativeSeparators(parser.value(CLI_INPUT_PARAMETER)));
         QUrl outputUrl(QDir::fromNativeSeparators(parser.value(CLI_OUTPUT_PARAMETER)));
         QString type = parser.isSet(CLI_TYPE_PARAMETER) ? parser.value(CLI_TYPE_PARAMETER) : QString::null;
-        cli->bakeFile(inputUrl, outputUrl.toString(), type);
+        QMetaObject::invokeMethod(cli, "bakeFile", Qt::QueuedConnection, Q_ARG(QUrl, inputUrl),
+                                    Q_ARG(QString, outputUrl.toString()), Q_ARG(QString, type));
     } else {
         parser.showHelp();
         QCoreApplication::quit();