add writing of new entities file during domain bake

libraries/model-baking/src/FBXBaker.cpp

@@ -140,7 +140,10 @@ void FBXBaker::handleFBXNetworkReply() {
         // kick off the bake process now that everything is ready to go
         bake();
     } else {
-        qDebug() << "ERROR DOWNLOADING FBX" << requestReply->errorString();
+        // add an error to our list stating that the FBX could not be downloaded
+
+
+        emit finished();
     }
 }
 
@@ -155,6 +158,12 @@ void FBXBaker::bake() {
 
     removeEmbeddedMediaFolder();
     possiblyCleanupOriginals();
+
+    // at this point we are sure that we've finished everything that does not relate to textures
+    // so set that flag now
+    _finishedNonTextureOperations = true;
+
+    checkIfFinished();
 }
 
 bool FBXBaker::importScene() {
@@ -226,6 +235,8 @@ QString FBXBaker::createBakedTextureFileName(const QFileInfo& textureFileInfo) {
 QUrl FBXBaker::getTextureURL(const QFileInfo& textureFileInfo, FbxFileTexture* fileTexture) {
     QUrl urlToTexture;
 
+    qDebug() << "Looking at" << textureFileInfo.absoluteFilePath();
+
     if (textureFileInfo.exists() && textureFileInfo.isFile()) {
         // set the texture URL to the local texture that we have confirmed exists
         urlToTexture = QUrl::fromLocalFile(textureFileInfo.absoluteFilePath());
@@ -355,8 +366,9 @@ bool FBXBaker::rewriteAndBakeSceneTextures() {
                             // use QFileInfo to easily split up the existing texture filename into its components
                             QFileInfo textureFileInfo { fileTexture->GetFileName() };
 
-                            // make sure this texture points to something
+                            // make sure this texture points to something and isn't one we've already re-mapped
-                            if (!textureFileInfo.filePath().isEmpty()) {
+                            if (!textureFileInfo.filePath().isEmpty()
+                                && textureFileInfo.completeSuffix() != BAKED_TEXTURE_EXT.mid(1)) {
 
                                 // construct the new baked texture file name and file path
                                 // ensuring that the baked texture will have a unique name
@@ -438,15 +450,25 @@ void FBXBaker::handleBakedTexture() {
             << "for" << _fbxURL;
         }
     }
-}
 
-static const QString BAKED_FBX_EXTENSION = ".baked.fbx";
+    // now that this texture has been baked and handled, we can remove that TextureBaker from our list
+    _unbakedTextures.remove(bakedTexture->getTextureURL());
+
+    // since this could have been the last texture we were waiting for
+    // we should perform a quick check now to see if we are done baking this model
+    checkIfFinished();
+}
 
 bool FBXBaker::exportScene() {
     // setup the exporter
     FbxExporter* exporter = FbxExporter::Create(_sdkManager, "");
 
     auto rewrittenFBXPath = _uniqueOutputPath + BAKED_OUTPUT_SUBFOLDER + _fbxName + BAKED_FBX_EXTENSION;
+
+    // save the relative path to this FBX inside our passed output folder
+    _bakedFBXRelativePath = rewrittenFBXPath;
+    _bakedFBXRelativePath.remove(_baseOutputPath + "/");
+
     bool exportStatus = exporter->Initialize(rewrittenFBXPath.toLocal8Bit().data());
 
     if (!exportStatus) {
@@ -478,3 +500,9 @@ void FBXBaker::possiblyCleanupOriginals() {
         QDir(_uniqueOutputPath + ORIGINAL_OUTPUT_SUBFOLDER).removeRecursively();
     }
 }
+
+void FBXBaker::checkIfFinished() {
+    if (_unbakedTextures.isEmpty() && _finishedNonTextureOperations) {
+        emit finished();
+    }
+}

libraries/model-baking/src/FBXBaker.h

@@ -44,6 +44,8 @@ enum TextureType {
 
 class TextureBaker;
 
+static const QString BAKED_FBX_EXTENSION = ".baked.fbx";
+
 class FBXBaker : public QObject {
     Q_OBJECT
 public:
@@ -52,6 +54,9 @@ public:
 
     void start();
 
+    QUrl getFBXUrl() const { return _fbxURL; }
+    QString getBakedFBXRelativePath() const { return _bakedFBXRelativePath; }
+
 signals:
     void finished();
 
@@ -68,6 +73,7 @@ private:
     bool exportScene();
     void removeEmbeddedMediaFolder();
     void possiblyCleanupOriginals();
+    void checkIfFinished();
 
     QString createBakedTextureFileName(const QFileInfo& textureFileInfo);
     QUrl getTextureURL(const QFileInfo& textureFileInfo, fbxsdk::FbxFileTexture* fileTexture);
@@ -81,6 +87,7 @@ private:
     
     QString _baseOutputPath;
     QString _uniqueOutputPath;
+    QString _bakedFBXRelativePath;
 
     fbxsdk::FbxManager* _sdkManager;
     fbxsdk::FbxScene* _scene { nullptr };
@@ -94,6 +101,8 @@ private:
     std::list<std::unique_ptr<TextureBaker>> _bakingTextures;
 
     bool _copyOriginals { true };
+
+    bool _finishedNonTextureOperations { false };
 };
 
 #endif // hifi_FBXBaker_h

libraries/model-baking/src/TextureBaker.cpp

@@ -73,7 +73,10 @@ void TextureBaker::handleTextureNetworkReply() {
         // kickoff the texture bake now that everything is ready to go
         bake();
     } else {
+        // add an error to our list stating that this texture could not be downloaded
         qCDebug(model_baking) << "Error downloading texture" << requestReply->errorString();
+
+        emit finished();
     }
 }
 

tools/oven/src/DomainBaker.cpp

@@ -20,12 +20,18 @@
 
 #include "DomainBaker.h"
 
-DomainBaker::DomainBaker(const QUrl& localModelFileURL, const QString& domainName, const QString& baseOutputPath) :
+DomainBaker::DomainBaker(const QUrl& localModelFileURL, const QString& domainName,
+                         const QString& baseOutputPath, const QUrl& destinationPath) :
     _localEntitiesFileURL(localModelFileURL),
     _domainName(domainName),
     _baseOutputPath(baseOutputPath)
 {
-    
+    // make sure the destination path has a trailing slash
+    if (!destinationPath.toString().endsWith('/')) {
+        _destinationPath = destinationPath.toString() + '/';
+    } else {
+        _destinationPath = destinationPath;
+    }
 }
 
 void DomainBaker::start() {
@@ -45,9 +51,9 @@ void DomainBaker::setupOutputFolder() {
     QString outputDirectoryName = domainPrefix + timeNow.toString(FOLDER_TIMESTAMP_FORMAT);
 
     //  make sure we can create that directory
-    QDir baseDir { _baseOutputPath };
+    QDir outputDir { _baseOutputPath };
 
-    if (!baseDir.mkpath(outputDirectoryName)) {
+    if (!outputDir.mkpath(outputDirectoryName)) {
 
         // add an error to specify that the output directory could not be created
 
@@ -55,10 +61,22 @@ void DomainBaker::setupOutputFolder() {
     }
 
     // store the unique output path so we can re-use it when saving baked models
-    baseDir.cd(outputDirectoryName);
+    outputDir.cd(outputDirectoryName);
-    _uniqueOutputPath = baseDir.absolutePath();
+    _uniqueOutputPath = outputDir.absolutePath();
+
+    // add a content folder inside the unique output folder
+    static const QString CONTENT_OUTPUT_FOLDER_NAME = "content";
+    if (!outputDir.mkpath(CONTENT_OUTPUT_FOLDER_NAME)) {
+        // add an error to specify that the content output directory could not be created
+
+        return;
+    }
+
+    _contentOutputPath = outputDir.absoluteFilePath(CONTENT_OUTPUT_FOLDER_NAME);
 }
 
+const QString ENTITIES_OBJECT_KEY = "Entities";
+
 void DomainBaker::loadLocalFile() {
     // load up the local entities file
     QFile modelsFile { _localEntitiesFileURL.toLocalFile() };
@@ -86,7 +104,7 @@ void DomainBaker::loadLocalFile() {
     auto jsonDocument = QJsonDocument::fromJson(fileContents);
 
     // grab the entities object from the root JSON object
-    _entities = jsonDocument.object()["Entities"].toArray();
+    _entities = jsonDocument.object()[ENTITIES_OBJECT_KEY].toArray();
 
     if (_entities.isEmpty()) {
         // add an error to our list stating that the models file was empty
@@ -96,19 +114,20 @@ void DomainBaker::loadLocalFile() {
     }
 }
 
+static const QString ENTITY_MODEL_URL_KEY = "modelURL";
+
 void DomainBaker::enumerateEntities() {
     qDebug() << "Enumerating" << _entities.size() << "entities from domain";
 
-    foreach(QJsonValue entityValue, _entities) {
+    for (auto it = _entities.begin(); it != _entities.end(); ++it) {
         // make sure this is a JSON object
-        if (entityValue.isObject()) {
+        if (it->isObject()) {
-            auto entity = entityValue.toObject();
+            auto entity = it->toObject();
 
             // check if this is an entity with a model URL
-            static const QString ENTITY_MODEL_URL_KEY = "modelURL";
             if (entity.contains(ENTITY_MODEL_URL_KEY)) {
                 // grab a QUrl for the model URL
-                auto modelURL = QUrl(entity[ENTITY_MODEL_URL_KEY].toString());
+                QUrl modelURL { entity[ENTITY_MODEL_URL_KEY].toString() };
 
                 // check if the file pointed to by this URL is a bakeable model, by comparing extensions
                 auto modelFileName = modelURL.fileName();
@@ -118,12 +137,15 @@ void DomainBaker::enumerateEntities() {
 
                 if (BAKEABLE_MODEL_EXTENSIONS.contains(completeLowerExtension)) {
                     // grab a clean version of the URL without a query or fragment
-                    modelURL.setFragment("");
+                    modelURL.setFragment(QString());
-                    modelURL.setQuery("");
+                    modelURL.setQuery(QString());
 
                     // setup an FBXBaker for this URL, as long as we don't already have one
                     if (!_bakers.contains(modelURL)) {
-                        QSharedPointer<FBXBaker> baker { new FBXBaker(modelURL, _uniqueOutputPath) };
+                        QSharedPointer<FBXBaker> baker { new FBXBaker(modelURL, _contentOutputPath) };
+
+                        // make sure our handler is called when the baker is done
+                        connect(baker.data(), &FBXBaker::finished, this, &DomainBaker::handleFinishedBaker);
 
                         // start the baker
                         baker->start();
@@ -131,8 +153,97 @@ void DomainBaker::enumerateEntities() {
                         // insert it into our bakers hash so we hold a strong pointer to it
                         _bakers.insert(modelURL, baker);
                     }
+
+                    // add this QJsonValueRef to our multi hash so that we can easily re-write
+                    // the model URL to the baked version once the baker is complete
+                    _entitiesNeedingRewrite.insert(modelURL, *it);
                 }
             }
         }
     }
+
+    _enumeratedAllEntities = true;
+
+    // check if it's time to write out the final entities file with re-written URLs
+    possiblyOutputEntitiesFile();
 }
+
+void DomainBaker::handleFinishedBaker() {
+    auto baker = qobject_cast<FBXBaker*>(sender());
+
+    if (baker) {
+        // this FBXBaker is done and everything went according to plan
+
+        // enumerate the QJsonRef values for the URL of this FBX from our multi hash of
+        // entity objects needing a URL re-write
+        for (QJsonValueRef entityValue : _entitiesNeedingRewrite.values(baker->getFBXUrl())) {
+
+            // convert the entity QJsonValueRef to a QJsonObject so we can modify its URL
+            auto entity = entityValue.toObject();
+
+            // grab the old URL
+            QUrl oldModelURL { entity[ENTITY_MODEL_URL_KEY].toString() };
+
+            // setup a new URL using the prefix we were passed
+            QUrl newModelURL = _destinationPath.resolved(baker->getBakedFBXRelativePath().mid(1));
+
+            // copy the fragment and query from the old model URL
+            newModelURL.setQuery(oldModelURL.query());
+            newModelURL.setFragment(oldModelURL.fragment());
+
+            // set the new model URL as the value in our temp QJsonObject
+            entity[ENTITY_MODEL_URL_KEY] = newModelURL.toString();
+
+            // replace our temp object with the value referenced by our QJsonValueRef
+            entityValue = entity;
+        }
+
+        // remove the baked URL from the multi hash of entities needing a re-write
+        _entitiesNeedingRewrite.remove(baker->getFBXUrl());
+
+        // check if it's time to write out the final entities file with re-written URLs
+        possiblyOutputEntitiesFile();
+    }
+}
+
+void DomainBaker::possiblyOutputEntitiesFile() {
+    if (_enumeratedAllEntities && _entitiesNeedingRewrite.isEmpty()) {
+        // we've enumerated all of our entities and re-written all the URLs we'll be able to re-write
+        // time to write out a main models.json.gz file
+
+        // first setup a document with the entities array below the entities key
+        QJsonDocument entitiesDocument;
+
+        QJsonObject rootObject;
+        rootObject[ENTITIES_OBJECT_KEY] = _entities;
+
+        entitiesDocument.setObject(rootObject);
+
+        // turn that QJsonDocument into a byte array ready for compression
+        QByteArray jsonByteArray = entitiesDocument.toJson();
+
+        // compress the json byte array using gzip
+        QByteArray compressedJson;
+        gzip(jsonByteArray, compressedJson);
+
+        // write the gzipped json to a new models file
+        static const QString MODELS_FILE_NAME = "models.json.gz";
+
+        auto bakedEntitiesFilePath = QDir(_uniqueOutputPath).filePath(MODELS_FILE_NAME);
+        QFile compressedEntitiesFile { bakedEntitiesFilePath };
+
+        if (!compressedEntitiesFile.open(QIODevice::WriteOnly)
+            || (compressedEntitiesFile.write(compressedJson) == -1)) {
+            qWarning() << "Failed to export baked entities file to" << bakedEntitiesFilePath;
+            // add an error to our list to state that the output models file could not be created or could not be written to
+
+            return;
+        }
+
+        qDebug() << "Exported entities file with baked model URLs to" << bakedEntitiesFilePath;
+
+        // we've now written out our new models file - time to say that we are finished up
+        emit finished();
+    }
+}
+

tools/oven/src/DomainBaker.h

@@ -21,7 +21,8 @@
 class DomainBaker : public QObject {
     Q_OBJECT
 public:
-    DomainBaker(const QUrl& localEntitiesFileURL, const QString& domainName, const QString& baseOutputPath);
+    DomainBaker(const QUrl& localEntitiesFileURL, const QString& domainName,
+                const QString& baseOutputPath, const QUrl& destinationPath);
 
 public slots:
     void start();
@@ -29,19 +30,28 @@ public slots:
 signals:
     void finished();
 
+private slots:
+    void handleFinishedBaker();
+
 private:
     void setupOutputFolder();
     void loadLocalFile();
     void enumerateEntities();
+    void possiblyOutputEntitiesFile();
 
     QUrl _localEntitiesFileURL;
     QString _domainName;
     QString _baseOutputPath;
     QString _uniqueOutputPath;
+    QString _contentOutputPath;
+    QUrl _destinationPath;
 
     QJsonArray _entities;
 
     QHash<QUrl, QSharedPointer<FBXBaker>> _bakers;
+    QMultiHash<QUrl, QJsonValueRef> _entitiesNeedingRewrite;
+
+    bool _enumeratedAllEntities { false };
 };
 
 #endif // hifi_DomainBaker_h

tools/oven/src/ui/DomainBakeWidget.cpp

@@ -21,13 +21,17 @@
 
 #include "DomainBakeWidget.h"
 
+static const QString DOMAIN_NAME_SETTING_KEY = "domain_name";
 static const QString EXPORT_DIR_SETTING_KEY = "domain_export_directory";
 static const QString BROWSE_START_DIR_SETTING_KEY = "domain_search_directory";
+static const QString DESTINATION_PATH_SETTING_KEY = "destination_path";
 
 DomainBakeWidget::DomainBakeWidget(QWidget* parent, Qt::WindowFlags flags) :
     QWidget(parent, flags),
+    _domainNameSetting(DOMAIN_NAME_SETTING_KEY),
     _exportDirectory(EXPORT_DIR_SETTING_KEY),
-    _browseStartDirectory(BROWSE_START_DIR_SETTING_KEY)
+    _browseStartDirectory(BROWSE_START_DIR_SETTING_KEY),
+    _destinationPathSetting(DESTINATION_PATH_SETTING_KEY)
 {
     setupUI();
 }
@@ -44,6 +48,11 @@ void DomainBakeWidget::setupUI() {
     _domainNameLineEdit = new QLineEdit;
     _domainNameLineEdit->setPlaceholderText("welcome");
 
+    // set the text of the domain name from whatever was used during last bake
+    if (!_domainNameSetting.get().isEmpty()) {
+        _domainNameLineEdit->setText(_domainNameSetting.get());
+    }
+
     gridLayout->addWidget(domainNameLabel);
     gridLayout->addWidget(_domainNameLineEdit, rowIndex, 1, 1, -1);
 
@@ -88,6 +97,22 @@ void DomainBakeWidget::setupUI() {
     // start a new row for the next component
     ++rowIndex;
 
+    // setup a section to choose the upload prefix - the URL where baked models will be made available
+    QLabel* uploadPrefixLabel = new QLabel("Destination URL Path");
+
+    _destinationPathLineEdit = new QLineEdit;
+    _destinationPathLineEdit->setPlaceholderText("http://cdn.example.com/baked-domain/");
+
+    if (!_destinationPathSetting.get().isEmpty()) {
+        _destinationPathLineEdit->setText(_destinationPathSetting.get());
+    }
+
+    gridLayout->addWidget(uploadPrefixLabel, rowIndex, 0);
+    gridLayout->addWidget(_destinationPathLineEdit, rowIndex, 1, 1, -1);
+
+    // start a new row for the next component
+    ++rowIndex;
+
     // add a horizontal line to split the bake/cancel buttons off
     QFrame* lineFrame = new QFrame;
     lineFrame->setFrameShape(QFrame::HLine);
@@ -160,6 +185,13 @@ void DomainBakeWidget::outputDirectoryChanged(const QString& newDirectory) {
 }
 
 void DomainBakeWidget::bakeButtonClicked() {
+
+    // save whatever the current domain name is in settings, we'll re-use it next time the widget is shown
+    _domainNameSetting.set(_domainNameLineEdit->text());
+
+    // save whatever the current destination path is in settings, we'll re-use it next time the widget is shown
+    _destinationPathSetting.set(_destinationPathLineEdit->text());
+
     // make sure we have a valid output directory
     QDir outputDirectory(_outputDirLineEdit->text());
 
@@ -172,7 +204,8 @@ void DomainBakeWidget::bakeButtonClicked() {
         // everything seems to be in place, kick off a bake for this entities file now
         auto fileToBakeURL = QUrl::fromLocalFile(_entitiesFileLineEdit->text());
         _baker = std::unique_ptr<DomainBaker> {
-                new DomainBaker(fileToBakeURL, _domainNameLineEdit->text(), outputDirectory.absolutePath())
+                new DomainBaker(fileToBakeURL, _domainNameLineEdit->text(),
+                                outputDirectory.absolutePath(), _destinationPathLineEdit->text())
         };
         _baker->start();
 

tools/oven/src/ui/DomainBakeWidget.h

@@ -42,9 +42,12 @@ private:
     QLineEdit* _domainNameLineEdit;
     QLineEdit* _entitiesFileLineEdit;
     QLineEdit* _outputDirLineEdit;
+    QLineEdit* _destinationPathLineEdit;
 
+    Setting::Handle<QString> _domainNameSetting;
     Setting::Handle<QString> _exportDirectory;
     Setting::Handle<QString> _browseStartDirectory;
+    Setting::Handle<QString> _destinationPathSetting;
 };
 
 #endif // hifi_ModelBakeWidget_h