Merge pull request #15439 from SamGondelman/textureUsage

Case 22322: Oven handles TextureBakers with same URL but different usage + fix domain baking

libraries/baking/src/FBXBaker.cpp

@@ -31,7 +31,6 @@
 #include <FBXWriter.h>
 
 #include "ModelBakingLoggingCategory.h"
-#include "TextureBaker.h"
 
 FBXBaker::FBXBaker(const QUrl& inputModelURL, const QString& bakedOutputDirectory, const QString& originalOutputDirectory, bool hasBeenBaked) :
         ModelBaker(inputModelURL, bakedOutputDirectory, originalOutputDirectory, hasBeenBaked) {

libraries/baking/src/FBXBaker.h

@@ -18,15 +18,11 @@
 #include <QtNetwork/QNetworkReply>
 
 #include "Baker.h"
-#include "TextureBaker.h"
 #include "ModelBaker.h"
 #include "ModelBakingLoggingCategory.h"
 
-#include <gpu/Texture.h> 
-
 #include <FBX.h>
 
-using TextureBakerThreadGetter = std::function<QThread*()>;
 
 class FBXBaker : public ModelBaker {
     Q_OBJECT

libraries/baking/src/MaterialBaker.cpp

@@ -133,12 +133,12 @@ void MaterialBaker::processMaterial() {
                     QString extension = idx >= 0 ? cleanURL.mid(idx + 1).toLower() : "";
 
                     if (QImageReader::supportedImageFormats().contains(extension.toLatin1())) {
-                        QPair<QUrl, image::TextureUsage::Type> textureKey(textureURL, type);
+                        TextureKey textureKey(textureURL, type);
                         if (!_textureBakers.contains(textureKey)) {
                             auto baseTextureFileName = _textureFileNamer.createBaseTextureFileName(textureURL.fileName(), type);
 
                             QSharedPointer<TextureBaker> textureBaker {
-                                new TextureBaker(textureURL, type, _textureOutputDir, "", baseTextureFileName, content),
+                                new TextureBaker(textureURL, type, _textureOutputDir, baseTextureFileName, content),
                                 &TextureBaker::deleteLater
                             };
                             textureBaker->setMapChannel(mapChannel);
@@ -170,7 +170,7 @@ void MaterialBaker::handleFinishedTextureBaker() {
     auto baker = qobject_cast<TextureBaker*>(sender());
 
     if (baker) {
-        QPair<QUrl, image::TextureUsage::Type> textureKey = { baker->getTextureURL(), baker->getTextureType() };
+        TextureKey textureKey = { baker->getTextureURL(), baker->getTextureType() };
         if (!baker->hasErrors()) {
             // this TextureBaker is done and everything went according to plan
             qCDebug(material_baking) << "Re-writing texture references to" << baker->getTextureURL();

libraries/baking/src/MaterialBaker.h

@@ -21,6 +21,8 @@
 
 static const QString BAKED_MATERIAL_EXTENSION = ".baked.json";
 
+using TextureKey = QPair<QUrl, image::TextureUsage::Type>;
+
 class MaterialBaker : public Baker {
     Q_OBJECT
 public:
@@ -58,8 +60,8 @@ private:
 
     NetworkMaterialResourcePointer _materialResource;
 
-    QHash<QPair<QUrl, image::TextureUsage::Type>, QSharedPointer<TextureBaker>> _textureBakers;
+    QHash<TextureKey, QSharedPointer<TextureBaker>> _textureBakers;
-    QMultiHash<QPair<QUrl, image::TextureUsage::Type>, std::shared_ptr<NetworkMaterial>> _materialsNeedingRewrite;
+    QMultiHash<TextureKey, std::shared_ptr<NetworkMaterial>> _materialsNeedingRewrite;
 
     QString _bakedOutputDir;
     QString _textureOutputDir;

libraries/baking/src/OBJBaker.h

@@ -13,13 +13,9 @@
 #define hifi_OBJBaker_h
 
 #include "Baker.h"
-#include "TextureBaker.h"
 #include "ModelBaker.h"
-
 #include "ModelBakingLoggingCategory.h"
 
-using TextureBakerThreadGetter = std::function<QThread*()>;
-
 using NodeID = qlonglong;
 
 class OBJBaker : public ModelBaker {

libraries/baking/src/TextureBaker.cpp

@@ -33,14 +33,13 @@ const QString BAKED_META_TEXTURE_SUFFIX = ".texmeta.json";
 bool TextureBaker::_compressionEnabled = true;
 
 TextureBaker::TextureBaker(const QUrl& textureURL, image::TextureUsage::Type textureType,
-                           const QDir& outputDirectory, const QString& metaTexturePathPrefix,
+                           const QDir& outputDirectory, const QString& baseFilename,
-                           const QString& baseFilename, const QByteArray& textureContent) :
+                           const QByteArray& textureContent) :
     _textureURL(textureURL),
     _originalTexture(textureContent),
     _textureType(textureType),
     _baseFilename(baseFilename),
-    _outputDirectory(outputDirectory),
+    _outputDirectory(outputDirectory)
-    _metaTexturePathPrefix(metaTexturePathPrefix)
 {
     if (baseFilename.isEmpty()) {
         // figure out the baked texture filename
@@ -151,7 +150,7 @@ void TextureBaker::processTexture() {
         // IMPORTANT: _originalTexture is empty past this point
         _originalTexture.clear();
         _outputFiles.push_back(originalCopyFilePath);
-        meta.original = _metaTexturePathPrefix + _originalCopyFilePath.fileName();
+        meta.original = _originalCopyFilePath.fileName();
     }
 
     // Load the copy of the original file from the baked output directory. New images will be created using the original as the source data.
@@ -204,7 +203,7 @@ void TextureBaker::processTexture() {
                 return;
             }
             _outputFiles.push_back(filePath);
-            meta.availableTextureTypes[memKTX->_header.getGLInternaFormat()] = _metaTexturePathPrefix + fileName;
+            meta.availableTextureTypes[memKTX->_header.getGLInternaFormat()] = fileName;
         }
     }
 
@@ -240,7 +239,7 @@ void TextureBaker::processTexture() {
             return;
         }
         _outputFiles.push_back(filePath);
-        meta.uncompressed = _metaTexturePathPrefix + fileName;
+        meta.uncompressed = fileName;
     } else {
         buffer.reset();
     }

libraries/baking/src/TextureBaker.h

@@ -32,8 +32,8 @@ class TextureBaker : public Baker {
 
 public:
     TextureBaker(const QUrl& textureURL, image::TextureUsage::Type textureType,
-                 const QDir& outputDirectory, const QString& metaTexturePathPrefix = "",
+                 const QDir& outputDirectory, const QString& baseFilename = QString(),
-                 const QString& baseFilename = QString(), const QByteArray& textureContent = QByteArray());
+                 const QByteArray& textureContent = QByteArray());
 
     const QByteArray& getOriginalTexture() const { return _originalTexture; }
 
@@ -74,7 +74,6 @@ private:
     QString _baseFilename;
     QDir _outputDirectory;
     QString _metaTextureFileName;
-    QString _metaTexturePathPrefix;
     QUrl _originalCopyFilePath;
 
     std::atomic<bool> _abortProcessing { false };

libraries/gpu/src/gpu/Texture.cpp

@@ -37,7 +37,6 @@ ContextMetricCount Texture::_textureCPUCount;
 ContextMetricSize Texture::_textureCPUMemSize;
 std::atomic<Texture::Size> Texture::_allowedCPUMemoryUsage { 0 };
 
-
 #define MIN_CORES_FOR_INCREMENTAL_TEXTURES 5
 bool recommendedSparseTextures = (QThread::idealThreadCount() >= MIN_CORES_FOR_INCREMENTAL_TEXTURES);
 

libraries/gpu/src/gpu/Texture.h

@@ -550,7 +550,7 @@ public:
     void setUsage(const Usage& usage) { _usage = usage; }
     Usage getUsage() const { return _usage; }
 
-    // For Cube Texture, it's possible to generate the irradiance spherical harmonics and make them availalbe with the texture
+    // For Cube Texture, it's possible to generate the irradiance spherical harmonics and make them available with the texture
     bool generateIrradiance(gpu::BackendTarget target);
     const SHPointer& getIrradiance(uint16 slice = 0) const { return _irradiance; }
     void overrideIrradiance(SHPointer irradiance) { _irradiance = irradiance; }

libraries/image/src/image/TextureProcessing.cpp

@@ -103,7 +103,7 @@ gpu::Element getHDRTextureFormatForTarget(BackendTarget target, bool compressed)
     }
 }
 
-TextureUsage::TextureLoader TextureUsage::getTextureLoaderForType(Type type, const QVariantMap& options) {
+TextureUsage::TextureLoader TextureUsage::getTextureLoaderForType(Type type) {
     switch (type) {
         case ALBEDO_TEXTURE:
             return image::TextureUsage::createAlbedoTextureFromImage;
@@ -114,11 +114,7 @@ TextureUsage::TextureLoader TextureUsage::getTextureLoaderForType(Type type, con
         case SKY_TEXTURE:
             return image::TextureUsage::createCubeTextureFromImage;
         case AMBIENT_TEXTURE:
-            if (options.value("generateIrradiance", true).toBool()) {
+            return image::TextureUsage::createAmbientCubeTextureAndIrradianceFromImage;
-                return image::TextureUsage::createAmbientCubeTextureAndIrradianceFromImage;
-            } else {
-                return image::TextureUsage::createAmbientCubeTextureFromImage;
-            }
         case BUMP_TEXTURE:
             return image::TextureUsage::createNormalTextureFromBumpImage;
         case NORMAL_TEXTURE:
@@ -188,21 +184,11 @@ gpu::TexturePointer TextureUsage::createMetallicTextureFromImage(Image&& srcImag
     return process2DTextureGrayscaleFromImage(std::move(srcImage), srcImageName, compress, target, false, abortProcessing);
 }
 
-gpu::TexturePointer TextureUsage::createCubeTextureAndIrradianceFromImage(Image&& srcImage, const std::string& srcImageName,
-                                                             bool compress, BackendTarget target, const std::atomic<bool>& abortProcessing) {
-    return processCubeTextureColorFromImage(std::move(srcImage), srcImageName, compress, target, CUBE_GENERATE_IRRADIANCE, abortProcessing);
-}
-
 gpu::TexturePointer TextureUsage::createCubeTextureFromImage(Image&& srcImage, const std::string& srcImageName,
                                                              bool compress, BackendTarget target, const std::atomic<bool>& abortProcessing) {
     return processCubeTextureColorFromImage(std::move(srcImage), srcImageName, compress, target, CUBE_DEFAULT, abortProcessing);
 }
 
-gpu::TexturePointer TextureUsage::createAmbientCubeTextureFromImage(Image&& image, const std::string& srcImageName,
-                                                           bool compress, gpu::BackendTarget target, const std::atomic<bool>& abortProcessing) {
-    return processCubeTextureColorFromImage(std::move(image), srcImageName, compress, target, CUBE_GGX_CONVOLVE, abortProcessing);
-}
-
 gpu::TexturePointer TextureUsage::createAmbientCubeTextureAndIrradianceFromImage(Image&& image, const std::string& srcImageName,
                                                                         bool compress, gpu::BackendTarget target, const std::atomic<bool>& abortProcessing) {
     return processCubeTextureColorFromImage(std::move(image), srcImageName, compress, target, CUBE_GENERATE_IRRADIANCE | CUBE_GGX_CONVOLVE, abortProcessing);
@@ -388,7 +374,7 @@ gpu::TexturePointer processImage(std::shared_ptr<QIODevice> content, const std::
     if (sourceChannel != ColorChannel::NONE) {
         mapToRedChannel(image, sourceChannel);
     }
-    
+
     auto loader = TextureUsage::getTextureLoaderForType(textureType);
     auto texture = loader(std::move(image), filename, compress, target, abortProcessing);
 

libraries/image/src/image/TextureProcessing.h

@@ -76,7 +76,7 @@ enum Type {
 };
 
 using TextureLoader = std::function<gpu::TexturePointer(Image&&, const std::string&, bool, gpu::BackendTarget, const std::atomic<bool>&)>;
-TextureLoader getTextureLoaderForType(Type type, const QVariantMap& options = QVariantMap());
+TextureLoader getTextureLoaderForType(Type type);
 
 gpu::TexturePointer create2DTextureFromImage(Image&& image, const std::string& srcImageName,
                                              bool compress, gpu::BackendTarget target, const std::atomic<bool>& abortProcessing);
@@ -98,10 +98,6 @@ gpu::TexturePointer createMetallicTextureFromImage(Image&& image, const std::str
                                                    bool compress, gpu::BackendTarget target, const std::atomic<bool>& abortProcessing);
 gpu::TexturePointer createCubeTextureFromImage(Image&& image, const std::string& srcImageName,
                                                bool compress, gpu::BackendTarget target, const std::atomic<bool>& abortProcessing);
-gpu::TexturePointer createCubeTextureAndIrradianceFromImage(Image&& image, const std::string& srcImageName,
-                                                            bool compress, gpu::BackendTarget target, const std::atomic<bool>& abortProcessing);
-gpu::TexturePointer createAmbientCubeTextureFromImage(Image&& image, const std::string& srcImageName,
-                                                      bool compress, gpu::BackendTarget target, const std::atomic<bool>& abortProcessing);
 gpu::TexturePointer createAmbientCubeTextureAndIrradianceFromImage(Image&& image, const std::string& srcImageName,
                                                                    bool compress, gpu::BackendTarget target, const std::atomic<bool>& abortProcessing);
 gpu::TexturePointer createLightmapTextureFromImage(Image&& image, const std::string& srcImageName,

libraries/material-networking/src/material-networking/TextureCache.cpp

@@ -311,13 +311,13 @@ gpu::BackendTarget getBackendTarget() {
 }
 
 /// Returns a texture version of an image file
-gpu::TexturePointer TextureCache::getImageTexture(const QString& path, image::TextureUsage::Type type, QVariantMap options) {
+gpu::TexturePointer TextureCache::getImageTexture(const QString& path, image::TextureUsage::Type type) {
     QImage image = QImage(path);
     if (image.isNull()) {
         qCWarning(networking) << "Unable to load required resource texture" << path;
         return nullptr;
     }
-    auto loader = image::TextureUsage::getTextureLoaderForType(type, options);
+    auto loader = image::TextureUsage::getTextureLoaderForType(type);
 
 #ifdef USE_GLES
     constexpr bool shouldCompress = true;

libraries/material-networking/src/material-networking/TextureCache.h

@@ -176,7 +176,7 @@ public:
     const gpu::TexturePointer& getBlackTexture();
 
     /// Returns a texture version of an image file
-    static gpu::TexturePointer getImageTexture(const QString& path, image::TextureUsage::Type type = image::TextureUsage::DEFAULT_TEXTURE, QVariantMap options = QVariantMap());
+    static gpu::TexturePointer getImageTexture(const QString& path, image::TextureUsage::Type type = image::TextureUsage::DEFAULT_TEXTURE);
 
     /// Loads a texture from the specified URL.
     NetworkTexturePointer getTexture(const QUrl& url, image::TextureUsage::Type type = image::TextureUsage::DEFAULT_TEXTURE,

tools/oven/src/DomainBaker.cpp

@@ -188,18 +188,20 @@ void DomainBaker::addModelBaker(const QString& property, const QString& url, con
 void DomainBaker::addTextureBaker(const QString& property, const QString& url, image::TextureUsage::Type type, const QJsonValueRef& jsonRef) {
     QString cleanURL = QUrl(url).adjusted(QUrl::RemoveQuery | QUrl::RemoveFragment).toDisplayString();
     auto idx = cleanURL.lastIndexOf('.');
-    auto extension = idx >= 0 ? url.mid(idx + 1).toLower() : "";
+    auto extension = idx >= 0 ? cleanURL.mid(idx + 1).toLower() : "";
 
     if (QImageReader::supportedImageFormats().contains(extension.toLatin1())) {
         // grab a clean version of the URL without a query or fragment
         QUrl textureURL = QUrl(url).adjusted(QUrl::RemoveQuery | QUrl::RemoveFragment);
+        TextureKey key = { textureURL, type };
 
         // setup a texture baker for this URL, as long as we aren't baking a texture already
-        if (!_textureBakers.contains(textureURL)) {
+        if (!_textureBakers.contains(key)) {
+            auto baseTextureFileName = _textureFileNamer.createBaseTextureFileName(textureURL.fileName(), type);
 
             // setup a baker for this texture
             QSharedPointer<TextureBaker> textureBaker {
-                new TextureBaker(textureURL, type, _contentOutputPath),
+                new TextureBaker(textureURL, type, _contentOutputPath, baseTextureFileName),
                 &TextureBaker::deleteLater
             };
 
@@ -207,7 +209,7 @@ void DomainBaker::addTextureBaker(const QString& property, const QString& url, i
             connect(textureBaker.data(), &TextureBaker::finished, this, &DomainBaker::handleFinishedTextureBaker);
 
             // insert it into our bakers hash so we hold a strong pointer to it
-            _textureBakers.insert(textureURL, textureBaker);
+            _textureBakers.insert(key, textureBaker);
 
             // move the baker to a worker thread and kickoff the bake
             textureBaker->moveToThread(Oven::instance().getNextWorkerThread());
@@ -219,7 +221,8 @@ void DomainBaker::addTextureBaker(const QString& property, const QString& url, i
 
         // add this QJsonValueRef to our multi hash so that it can re-write the texture URL
         // to the baked version once the baker is complete
-        _entitiesNeedingRewrite.insert(textureURL, { property, jsonRef });
+        // it doesn't really matter what this key is as long as it's consistent
+        _entitiesNeedingRewrite.insert(textureURL.toDisplayString() + "^" + QString::number(type), { property, jsonRef });
     } else {
         qDebug() << "Texture extension not supported: " << extension;
     }
@@ -499,9 +502,11 @@ void DomainBaker::handleFinishedTextureBaker() {
     auto baker = qobject_cast<TextureBaker*>(sender());
 
     if (baker) {
+        QUrl rewriteKey = baker->getTextureURL().toDisplayString() + "^" + QString::number(baker->getTextureType());
+
         if (!baker->hasErrors()) {
             // this TextureBaker is done and everything went according to plan
-            qDebug() << "Re-writing entity references to" << baker->getTextureURL();
+            qDebug() << "Re-writing entity references to" << baker->getTextureURL() << "with usage" << baker->getTextureType();
 
             // setup a new URL using the prefix we were passed
             auto relativeTextureFilePath = QDir(_contentOutputPath).relativeFilePath(baker->getMetaTextureFileName());
@@ -512,7 +517,7 @@ void DomainBaker::handleFinishedTextureBaker() {
 
             // enumerate the QJsonRef values for the URL of this texture from our multi hash of
             // entity objects needing a URL re-write
-            for (auto propertyEntityPair : _entitiesNeedingRewrite.values(baker->getTextureURL())) {
+            for (auto propertyEntityPair : _entitiesNeedingRewrite.values(rewriteKey)) {
                 QString property = propertyEntityPair.first;
                 // convert the entity QJsonValueRef to a QJsonObject so we can modify its URL
                 auto entity = propertyEntityPair.second.toObject();
@@ -556,10 +561,10 @@ void DomainBaker::handleFinishedTextureBaker() {
         }
 
         // remove the baked URL from the multi hash of entities needing a re-write
-        _entitiesNeedingRewrite.remove(baker->getTextureURL());
+        _entitiesNeedingRewrite.remove(rewriteKey);
 
         // drop our shared pointer to this baker so that it gets cleaned up
-        _textureBakers.remove(baker->getTextureURL());
+        _textureBakers.remove({ baker->getTextureURL(), baker->getTextureType() });
 
         // emit progress to tell listeners how many textures we have baked
         emit bakeProgress(++_completedSubBakes, _totalNumberOfSubBakes);
@@ -751,10 +756,11 @@ void DomainBaker::writeNewEntitiesFile() {
     // time to write out a main models.json.gz file
 
     // first setup a document with the entities array below the entities key
-    _json.object()[ENTITIES_OBJECT_KEY] = _entities;
+    QJsonObject json = _json.object();
+    json[ENTITIES_OBJECT_KEY] = _entities;
 
     // turn that QJsonDocument into a byte array ready for compression
-    QByteArray jsonByteArray = _json.toJson();
+    QByteArray jsonByteArray = QJsonDocument(json).toJson();
 
     // compress the json byte array using gzip
     QByteArray compressedJson;

tools/oven/src/DomainBaker.h

@@ -64,7 +64,8 @@ private:
     QJsonArray _entities;
 
     QHash<QUrl, QSharedPointer<ModelBaker>> _modelBakers;
-    QHash<QUrl, QSharedPointer<TextureBaker>> _textureBakers;
+    QHash<TextureKey, QSharedPointer<TextureBaker>> _textureBakers;
+    TextureFileNamer _textureFileNamer;
     QHash<QUrl, QSharedPointer<JSBaker>> _scriptBakers;
     QHash<QUrl, QSharedPointer<MaterialBaker>> _materialBakers;
     

tools/oven/src/ui/SkyboxBakeWidget.cpp

@@ -202,7 +202,7 @@ void SkyboxBakeWidget::bakeButtonClicked() {
             ambientMapBaseFilename = QUrl(urlParts.front()).fileName();
 
             // we need to bake the corresponding ambient map too
-            addBaker(new TextureBaker(skyboxToBakeURL, image::TextureUsage::AMBIENT_TEXTURE, outputDirectory.absolutePath(), QString(), ambientMapBaseFilename),
+            addBaker(new TextureBaker(skyboxToBakeURL, image::TextureUsage::AMBIENT_TEXTURE, outputDirectory.absolutePath(), ambientMapBaseFilename),
                      outputDirectory);
         }
     }