From bef093b8e53a70fffeb9f7a6254b859dc85ec1b8 Mon Sep 17 00:00:00 2001 From: Stephen Birarda Date: Fri, 11 Aug 2017 13:40:30 -0700 Subject: [PATCH 001/129] add handling for completed bake --- assignment-client/src/assets/AssetServer.cpp | 109 ++++++++++++++++++- assignment-client/src/assets/AssetServer.h | 8 +- 2 files changed, 115 insertions(+), 2 deletions(-) diff --git a/assignment-client/src/assets/AssetServer.cpp b/assignment-client/src/assets/AssetServer.cpp index 3886ff8d92..911a650cf2 100644 --- a/assignment-client/src/assets/AssetServer.cpp +++ b/assignment-client/src/assets/AssetServer.cpp @@ -18,6 +18,7 @@ #include #include #include +#include #include #include #include @@ -198,7 +199,6 @@ void AssetServer::completeSetup() { qCritical() << "Asset Server assignment will not continue because mapping file could not be loaded."; setFinished(true); } - } void AssetServer::cleanupUnmappedFiles() { @@ -785,3 +785,110 @@ bool AssetServer::renameMapping(AssetPath oldPath, AssetPath newPath) { } } } + +static const QString HIDDEN_BAKED_CONTENT_FOLDER = "/.baked/"; + +void AssetServer::handleCompletedBake(AssetHash originalAssetHash, QDir temporaryOutputDir) { + // enumerate the baking result files in the temporary directory + QDirIterator dirIterator(temporaryOutputDir.absolutePath(), QDir::Files, QDirIterator::Subdirectories); + + bool errorCompletingBake { false }; + + while (dirIterator.hasNext()) { + QString filePath = dirIterator.next(); + + // figure out the hash for the contents of this file + QFile file(filePath); + + AssetHash bakedFileHash; + + if (file.open(QIODevice::ReadOnly)) { + QCryptographicHash hasher(QCryptographicHash::Sha256); + + if (hasher.addData(&file)) { + bakedFileHash = hasher.result().toHex(); + } else { + // stop handling this bake, couldn't hash the contents of the file + errorCompletingBake = true; + break; + } + + // first check that we don't already have this bake file in our list + auto bakeFileDestination = _filesDirectory.absoluteFilePath(bakedFileHash); + if (!QFile::exists(bakeFileDestination)) { + // copy each to our files folder (with the hash as their filename) + if (!file.copy(_filesDirectory.absoluteFilePath(bakedFileHash))) { + // stop handling this bake, couldn't copy the bake file into our files directory + errorCompletingBake = true; + break; + } + } + + // setup the mapping for this bake file + auto relativeFilePath = temporaryOutputDir.relativeFilePath(filePath); + + static const QString BAKED_ASSET_SIMPLE_NAME = "asset.fbx"; + + if (relativeFilePath.endsWith(".fbx", Qt::CaseInsensitive)) { + // for an FBX file, we replace the filename with the simple name + // (to handle the case where two mapped assets have the same hash but different names) + relativeFilePath = BAKED_ASSET_SIMPLE_NAME; + } + + QString bakeMapping = HIDDEN_BAKED_CONTENT_FOLDER + originalAssetHash + "/" + relativeFilePath; + + // add a mapping (under the hidden baked folder) for this file resulting from the bake + if (setMapping(bakeMapping , bakedFileHash)) { + qDebug() << "Added" << bakeMapping << "for bake file" << bakedFileHash << "from bake of" << originalAssetHash; + } else { + // stop handling this bake, couldn't add a mapping for this bake file + errorCompletingBake = true; + break; + } + } else { + // stop handling this bake, we couldn't open one of the files for reading + errorCompletingBake = true; + break; + } + } + + if (!errorCompletingBake) { + // create the meta file to store which version of the baking process we just completed + createMetaFile(originalAssetHash); + } else { + qWarning() << "Could not complete bake for" << originalAssetHash; + } +} + +bool AssetServer::createMetaFile(AssetHash originalAssetHash) { + // construct the JSON that will be in the meta file + QJsonObject metaFileObject; + + static const int BAKE_VERSION = 1; + static const QString VERSION_KEY = "version"; + + metaFileObject[VERSION_KEY] = BAKE_VERSION; + + QJsonDocument metaFileDoc; + metaFileDoc.setObject(metaFileObject); + + auto metaFileJSON = metaFileDoc.toJson(); + + // get a hash for the contents of the meta-file + AssetHash metaFileHash = QCryptographicHash::hash(metaFileJSON, QCryptographicHash::Sha256).toHex(); + + // create the meta file in our files folder, named by the hash of its contents + QFile metaFile(_filesDirectory.absoluteFilePath(metaFileHash)); + + if (metaFile.open(QIODevice::WriteOnly)) { + metaFile.write(metaFileJSON); + metaFile.close(); + + // add a mapping to the meta file so it doesn't get deleted because it is unmapped + auto metaFileMapping = HIDDEN_BAKED_CONTENT_FOLDER + originalAssetHash + "/" + "meta.json"; + + return setMapping(metaFileMapping, metaFileHash); + } else { + return false; + } +} diff --git a/assignment-client/src/assets/AssetServer.h b/assignment-client/src/assets/AssetServer.h index 132fb51433..191ad25b1f 100644 --- a/assignment-client/src/assets/AssetServer.h +++ b/assignment-client/src/assets/AssetServer.h @@ -60,9 +60,15 @@ private: /// Rename mapping from `oldPath` to `newPath`. Returns true if successful bool renameMapping(AssetPath oldPath, AssetPath newPath); - // deletes any unmapped files from the local asset directory + /// Delete any unmapped files from the local asset directory void cleanupUnmappedFiles(); + /// Move baked content for asset to baked directory and update baked status + void handleCompletedBake(AssetHash originalAssetHash, QDir temporaryOutputDir); + + /// Create meta file to describe baked content for original asset + bool createMetaFile(AssetHash originalAssetHash); + Mappings _fileMappings; QDir _resourcesDirectory; From db3524a48fcc962ad3ce764d6b1ba2eb20a47a12 Mon Sep 17 00:00:00 2001 From: Stephen Birarda Date: Fri, 11 Aug 2017 14:26:27 -0700 Subject: [PATCH 002/129] add re-direct of get mapping for baked assets --- assignment-client/src/assets/AssetServer.cpp | 53 +++++++++++++++++-- .../networking/src/udt/PacketHeaders.cpp | 2 + libraries/networking/src/udt/PacketHeaders.h | 3 +- 3 files changed, 54 insertions(+), 4 deletions(-) diff --git a/assignment-client/src/assets/AssetServer.cpp b/assignment-client/src/assets/AssetServer.cpp index 911a650cf2..c0729dcf53 100644 --- a/assignment-client/src/assets/AssetServer.cpp +++ b/assignment-client/src/assets/AssetServer.cpp @@ -23,6 +23,7 @@ #include #include #include +#include #include #include @@ -264,14 +265,61 @@ void AssetServer::handleAssetMappingOperation(QSharedPointer me nodeList->sendPacketList(std::move(replyPacket), *senderNode); } +static const QStringList BAKEABLE_MODEL_EXTENSIONS = { ".fbx" }; +static const QString BAKED_MODEL_SIMPLE_NAME = "asset.fbx"; +static const QString BAKED_TEXTURE_SIMPLE_NAME = "texture.ktx"; + void AssetServer::handleGetMappingOperation(ReceivedMessage& message, SharedNodePointer senderNode, NLPacketList& replyPacket) { QString assetPath = message.readString(); auto it = _fileMappings.find(assetPath); if (it != _fileMappings.end()) { - auto assetHash = it->toString(); + + // check if we should re-direct to a baked asset + + // first, figure out from the mapping extension what type of file this is + auto assetPathExtension = assetPath.right(assetPath.lastIndexOf('.')).toLower(); + QString bakedRootFile; + + if (BAKEABLE_MODEL_EXTENSIONS.contains(assetPathExtension)) { + bakedRootFile = BAKED_MODEL_SIMPLE_NAME; + } else if (QImageReader::supportedImageFormats().contains(assetPathExtension.toLocal8Bit())) { + bakedRootFile = BAKED_TEXTURE_SIMPLE_NAME; + } + + auto originalAssetHash = it->toString(); + QString redirectedAssetHash; + QString bakedAssetPath; + quint8 wasRedirected = false; + + if (!bakedRootFile.isEmpty()) { + // we ran into an asset for which we could have a baked version, let's check if it's ready + bakedAssetPath = "/.baked/" + originalAssetHash + "/" + bakedRootFile; + auto bakedIt = _fileMappings.find(bakedAssetPath); + + if (bakedIt != _fileMappings.end()) { + // we found a baked version of the requested asset to serve, redirect to that + redirectedAssetHash = bakedIt->toString(); + wasRedirected = true; + } + } + replyPacket.writePrimitive(AssetServerError::NoError); - replyPacket.write(QByteArray::fromHex(assetHash.toUtf8())); + + if (wasRedirected) { + qDebug() << "Writing re-directed hash for" << originalAssetHash << "to" << redirectedAssetHash; + replyPacket.write(QByteArray::fromHex(redirectedAssetHash.toUtf8())); + + // add a flag saying that this mapping request was redirect + replyPacket.writePrimitive(wasRedirected); + + // include the re-directed path in case the caller needs to make relative path requests for the baked asset + replyPacket.write(bakedAssetPath.toUtf8()); + + } else { + replyPacket.write(QByteArray::fromHex(originalAssetHash.toUtf8())); + replyPacket.writePrimitive(wasRedirected); + } } else { replyPacket.writePrimitive(AssetServerError::AssetNotFound); } @@ -826,7 +874,6 @@ void AssetServer::handleCompletedBake(AssetHash originalAssetHash, QDir temporar // setup the mapping for this bake file auto relativeFilePath = temporaryOutputDir.relativeFilePath(filePath); - static const QString BAKED_ASSET_SIMPLE_NAME = "asset.fbx"; if (relativeFilePath.endsWith(".fbx", Qt::CaseInsensitive)) { diff --git a/libraries/networking/src/udt/PacketHeaders.cpp b/libraries/networking/src/udt/PacketHeaders.cpp index 241ccaf5d6..f453d096b6 100644 --- a/libraries/networking/src/udt/PacketHeaders.cpp +++ b/libraries/networking/src/udt/PacketHeaders.cpp @@ -42,6 +42,8 @@ PacketVersion versionForPacketType(PacketType packetType) { return static_cast(MessageDataVersion::TextOrBinaryData); case PacketType::ICEServerHeartbeat: return 18; // ICE Server Heartbeat signing + case PacketType::AssetMappingOperationReply: + return static_cast(AssetServerPacketVersion::RedirectedMappings); case PacketType::AssetGetInfo: case PacketType::AssetGet: case PacketType::AssetUpload: diff --git a/libraries/networking/src/udt/PacketHeaders.h b/libraries/networking/src/udt/PacketHeaders.h index e2304e62f7..ea831bd415 100644 --- a/libraries/networking/src/udt/PacketHeaders.h +++ b/libraries/networking/src/udt/PacketHeaders.h @@ -266,7 +266,8 @@ enum class EntityQueryPacketVersion: PacketVersion { enum class AssetServerPacketVersion: PacketVersion { VegasCongestionControl = 19, - RangeRequestSupport + RangeRequestSupport, + RedirectedMappings }; enum class AvatarMixerPacketVersion : PacketVersion { From 03e952ec38f549c3a54a053c69ec8573a45dc170 Mon Sep 17 00:00:00 2001 From: Stephen Birarda Date: Mon, 14 Aug 2017 14:09:35 -0700 Subject: [PATCH 003/129] fix extension check for get mapping redirect --- assignment-client/src/assets/AssetServer.cpp | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/assignment-client/src/assets/AssetServer.cpp b/assignment-client/src/assets/AssetServer.cpp index c0729dcf53..f82ef702f0 100644 --- a/assignment-client/src/assets/AssetServer.cpp +++ b/assignment-client/src/assets/AssetServer.cpp @@ -278,7 +278,7 @@ void AssetServer::handleGetMappingOperation(ReceivedMessage& message, SharedNode // check if we should re-direct to a baked asset // first, figure out from the mapping extension what type of file this is - auto assetPathExtension = assetPath.right(assetPath.lastIndexOf('.')).toLower(); + auto assetPathExtension = assetPath.mid(assetPath.lastIndexOf('.')).toLower(); QString bakedRootFile; if (BAKEABLE_MODEL_EXTENSIONS.contains(assetPathExtension)) { @@ -286,7 +286,7 @@ void AssetServer::handleGetMappingOperation(ReceivedMessage& message, SharedNode } else if (QImageReader::supportedImageFormats().contains(assetPathExtension.toLocal8Bit())) { bakedRootFile = BAKED_TEXTURE_SIMPLE_NAME; } - + auto originalAssetHash = it->toString(); QString redirectedAssetHash; QString bakedAssetPath; From 96672becc68dc2fac3db410017aff5d6aea68dad Mon Sep 17 00:00:00 2001 From: Stephen Birarda Date: Tue, 15 Aug 2017 15:19:20 -0700 Subject: [PATCH 004/129] read redirected path info in MappingRequest --- assignment-client/src/assets/AssetServer.cpp | 2 +- libraries/networking/src/MappingRequest.cpp | 11 +++++++++++ libraries/networking/src/MappingRequest.h | 6 ++++++ 3 files changed, 18 insertions(+), 1 deletion(-) diff --git a/assignment-client/src/assets/AssetServer.cpp b/assignment-client/src/assets/AssetServer.cpp index f82ef702f0..2082856f56 100644 --- a/assignment-client/src/assets/AssetServer.cpp +++ b/assignment-client/src/assets/AssetServer.cpp @@ -314,7 +314,7 @@ void AssetServer::handleGetMappingOperation(ReceivedMessage& message, SharedNode replyPacket.writePrimitive(wasRedirected); // include the re-directed path in case the caller needs to make relative path requests for the baked asset - replyPacket.write(bakedAssetPath.toUtf8()); + replyPacket.writeString(bakedAssetPath); } else { replyPacket.write(QByteArray::fromHex(originalAssetHash.toUtf8())); diff --git a/libraries/networking/src/MappingRequest.cpp b/libraries/networking/src/MappingRequest.cpp index 810b5b376d..bd7c05bdf7 100644 --- a/libraries/networking/src/MappingRequest.cpp +++ b/libraries/networking/src/MappingRequest.cpp @@ -87,6 +87,17 @@ void GetMappingRequest::doStart() { if (!_error) { _hash = message->read(SHA256_HASH_LENGTH).toHex(); + + // check the boolean to see if this request got re-directed + quint8 wasRedirected; + message->readPrimitive(&wasRedirected); + _wasRedirected = wasRedirected; + + // if it did grab that re-directed path + if (_wasRedirected) { + _redirectedPath = message->readString(); + } + } emit finished(this); }); diff --git a/libraries/networking/src/MappingRequest.h b/libraries/networking/src/MappingRequest.h index 85b68e2427..e1cd8a39e8 100644 --- a/libraries/networking/src/MappingRequest.h +++ b/libraries/networking/src/MappingRequest.h @@ -53,6 +53,8 @@ public: GetMappingRequest(const AssetPath& path); AssetHash getHash() const { return _hash; } + AssetPath getRedirectedPath() const { return _redirectedPath; } + bool wasRedirected() const { return _wasRedirected; } signals: void finished(GetMappingRequest* thisRequest); @@ -62,6 +64,10 @@ private: AssetPath _path; AssetHash _hash; + + + AssetPath _redirectedPath; + bool _wasRedirected { false }; }; class SetMappingRequest : public MappingRequest { From 100ac659758ef05a914498bef023f8b1f6de1c59 Mon Sep 17 00:00:00 2001 From: Stephen Birarda Date: Tue, 15 Aug 2017 15:27:44 -0700 Subject: [PATCH 005/129] store relative path URL in resource request on redirect --- libraries/networking/src/AssetRequest.h | 2 ++ libraries/networking/src/AssetResourceRequest.cpp | 5 +++++ libraries/networking/src/ResourceRequest.h | 1 + libraries/script-engine/src/AssetScriptingInterface.cpp | 1 - 4 files changed, 8 insertions(+), 1 deletion(-) diff --git a/libraries/networking/src/AssetRequest.h b/libraries/networking/src/AssetRequest.h index df5cf80ecd..a7213a90d7 100644 --- a/libraries/networking/src/AssetRequest.h +++ b/libraries/networking/src/AssetRequest.h @@ -21,6 +21,8 @@ #include "ByteRange.h" +const QString ATP_SCHEME { "atp:" }; + class AssetRequest : public QObject { Q_OBJECT public: diff --git a/libraries/networking/src/AssetResourceRequest.cpp b/libraries/networking/src/AssetResourceRequest.cpp index a41283cc0d..ecaa6efc1e 100644 --- a/libraries/networking/src/AssetResourceRequest.cpp +++ b/libraries/networking/src/AssetResourceRequest.cpp @@ -91,6 +91,11 @@ void AssetResourceRequest::requestMappingForPath(const AssetPath& path) { statTracker->incrementStat(STAT_ATP_MAPPING_REQUEST_SUCCESS); + // if we got a redirected path we need to store that with the resource request as relative path URL + if (request->wasRedirected()) { + _relativePathURL = ATP_SCHEME + request->getRedirectedPath(); + } + break; default: { switch (request->getError()) { diff --git a/libraries/networking/src/ResourceRequest.h b/libraries/networking/src/ResourceRequest.h index 3ee86025a2..bae0882e97 100644 --- a/libraries/networking/src/ResourceRequest.h +++ b/libraries/networking/src/ResourceRequest.h @@ -84,6 +84,7 @@ protected: virtual void doSend() = 0; QUrl _url; + QUrl _relativePathURL; State _state { NotStarted }; Result _result; QByteArray _data; diff --git a/libraries/script-engine/src/AssetScriptingInterface.cpp b/libraries/script-engine/src/AssetScriptingInterface.cpp index 65259987c4..51b3d7ffbe 100644 --- a/libraries/script-engine/src/AssetScriptingInterface.cpp +++ b/libraries/script-engine/src/AssetScriptingInterface.cpp @@ -55,7 +55,6 @@ void AssetScriptingInterface::setMapping(QString path, QString hash, QScriptValu void AssetScriptingInterface::downloadData(QString urlString, QScriptValue callback) { - const QString ATP_SCHEME { "atp:" }; if (!urlString.startsWith(ATP_SCHEME)) { return; From a59103aad5f9fab61c3226dc57d1c72532e53ad3 Mon Sep 17 00:00:00 2001 From: Stephen Birarda Date: Tue, 15 Aug 2017 15:38:47 -0700 Subject: [PATCH 006/129] truncate filename for relative path URL in RR --- libraries/networking/src/AssetResourceRequest.cpp | 3 +++ 1 file changed, 3 insertions(+) diff --git a/libraries/networking/src/AssetResourceRequest.cpp b/libraries/networking/src/AssetResourceRequest.cpp index ecaa6efc1e..e228582ca9 100644 --- a/libraries/networking/src/AssetResourceRequest.cpp +++ b/libraries/networking/src/AssetResourceRequest.cpp @@ -94,6 +94,9 @@ void AssetResourceRequest::requestMappingForPath(const AssetPath& path) { // if we got a redirected path we need to store that with the resource request as relative path URL if (request->wasRedirected()) { _relativePathURL = ATP_SCHEME + request->getRedirectedPath(); + + // truncate the filename for the re-directed asset so we actually have a path + _relativePathURL = _relativePathURL.adjusted(QUrl::RemoveFilename); } break; From 8e73eae341693162b7c8041f1d13b31f07bd8695 Mon Sep 17 00:00:00 2001 From: Ryan Huffman Date: Mon, 14 Aug 2017 15:25:18 -0700 Subject: [PATCH 007/129] Update nvtt to only use a single thread --- libraries/image/src/image/Image.cpp | 11 +++++++++++ 1 file changed, 11 insertions(+) diff --git a/libraries/image/src/image/Image.cpp b/libraries/image/src/image/Image.cpp index f274dc54f8..eaa55d0315 100644 --- a/libraries/image/src/image/Image.cpp +++ b/libraries/image/src/image/Image.cpp @@ -447,7 +447,18 @@ void generateMips(gpu::Texture* texture, QImage& image, int face = -1) { MyErrorHandler errorHandler; outputOptions.setErrorHandler(&errorHandler); + class SequentialTaskDispatcher : public nvtt::TaskDispatcher { + public: + virtual void dispatch(nvtt::Task* task, void* context, int count) { + for (int i = 0; i < count; i++) { + task(context, i); + } + } + }; + + SequentialTaskDispatcher dispatcher; nvtt::Compressor compressor; + compressor.setTaskDispatcher(&dispatcher); compressor.process(inputOptions, compressionOptions, outputOptions); #else texture->autoGenerateMips(-1); From b862336d17e44f7fe457490fc091e17ba38e151a Mon Sep 17 00:00:00 2001 From: Ryan Huffman Date: Mon, 14 Aug 2017 15:25:33 -0700 Subject: [PATCH 008/129] Add PathUtils::generateTemporaryDir() --- libraries/shared/src/PathUtils.cpp | 13 +++++++++++++ libraries/shared/src/PathUtils.h | 3 +++ 2 files changed, 16 insertions(+) diff --git a/libraries/shared/src/PathUtils.cpp b/libraries/shared/src/PathUtils.cpp index 0636411f51..20e30e15e8 100644 --- a/libraries/shared/src/PathUtils.cpp +++ b/libraries/shared/src/PathUtils.cpp @@ -54,6 +54,19 @@ QString PathUtils::getAppLocalDataFilePath(const QString& filename) { return QDir(getAppLocalDataPath()).absoluteFilePath(filename); } +QString PathUtils::generateTemporaryDir() { + QDir rootTempDir = QDir::tempPath(); + QString appName = qApp->applicationName(); + for (auto i = 0; i < 64; ++i) { + auto now = std::chrono::system_clock::now().time_since_epoch().count(); + QDir tempDir = rootTempDir.filePath(appName + "-" + QString::number(now)); + if (tempDir.mkpath(".")) { + return tempDir.absolutePath(); + } + } + return ""; +} + QString fileNameWithoutExtension(const QString& fileName, const QVector possibleExtensions) { QString fileNameLowered = fileName.toLower(); foreach (const QString possibleExtension, possibleExtensions) { diff --git a/libraries/shared/src/PathUtils.h b/libraries/shared/src/PathUtils.h index 3cb3cd3b63..8c4bcf2394 100644 --- a/libraries/shared/src/PathUtils.h +++ b/libraries/shared/src/PathUtils.h @@ -14,6 +14,7 @@ #include #include +#include #include "DependencyManager.h" @@ -36,6 +37,8 @@ public: static QString getAppDataFilePath(const QString& filename); static QString getAppLocalDataFilePath(const QString& filename); + static QString generateTemporaryDir(); + static Qt::CaseSensitivity getFSCaseSensitivity(); static QString stripFilename(const QUrl& url); // note: this is FS-case-sensitive version of parentURL.isParentOf(childURL) From 8d4ab5f751edeed5e57d62adca875bcc7278e720 Mon Sep 17 00:00:00 2001 From: Ryan Huffman Date: Mon, 14 Aug 2017 15:27:02 -0700 Subject: [PATCH 009/129] Update FBXBaker to take output directory and update oven to use it --- tools/oven/src/BakerCLI.cpp | 2 +- tools/oven/src/DomainBaker.cpp | 20 +++- tools/oven/src/DomainBaker.h | 2 + tools/oven/src/FBXBaker.cpp | 150 ++++++++++++++----------- tools/oven/src/FBXBaker.h | 30 ++--- tools/oven/src/ui/ModelBakeWidget.cpp | 25 ++++- tools/oven/src/ui/ResultsWindow.cpp | 1 + tools/oven/src/ui/SkyboxBakeWidget.cpp | 3 + 8 files changed, 143 insertions(+), 90 deletions(-) diff --git a/tools/oven/src/BakerCLI.cpp b/tools/oven/src/BakerCLI.cpp index 14eb9de150..7b65f8bbf0 100644 --- a/tools/oven/src/BakerCLI.cpp +++ b/tools/oven/src/BakerCLI.cpp @@ -41,7 +41,7 @@ void BakerCLI::bakeFile(QUrl inputUrl, const QString outputPath) { // create our appropiate baker if (isFBX) { - _baker = std::unique_ptr { new FBXBaker(inputUrl, outputPath, []() -> QThread* { return qApp->getNextWorkerThread(); }) }; + _baker = std::unique_ptr { new FBXBaker(inputUrl, []() -> QThread* { return qApp->getNextWorkerThread(); }, outputPath) }; _baker->moveToThread(qApp->getFBXBakerThread()); } else if (isSupportedImage) { _baker = std::unique_ptr { new TextureBaker(inputUrl, image::TextureUsage::CUBE_TEXTURE, outputPath) }; diff --git a/tools/oven/src/DomainBaker.cpp b/tools/oven/src/DomainBaker.cpp index 03bc350f42..5dd7c20b2e 100644 --- a/tools/oven/src/DomainBaker.cpp +++ b/tools/oven/src/DomainBaker.cpp @@ -192,10 +192,18 @@ void DomainBaker::enumerateEntities() { // setup an FBXBaker for this URL, as long as we don't already have one if (!_modelBakers.contains(modelURL)) { + auto filename = modelURL.fileName(); + auto baseName = filename.left(filename.lastIndexOf('.')); + auto subDirName = "/" + baseName; + int i = 0; + while (QDir(_contentOutputPath + subDirName).exists()) { + subDirName = "/" + baseName + "-" + i++; + } QSharedPointer baker { - new FBXBaker(modelURL, _contentOutputPath, []() -> QThread* { + new FBXBaker(modelURL, []() -> QThread* { return qApp->getNextWorkerThread(); - }), &FBXBaker::deleteLater + }, _contentOutputPath + subDirName + "/baked", _contentOutputPath + subDirName + "/original"), + &FBXBaker::deleteLater }; // make sure our handler is called when the baker is done @@ -309,7 +317,11 @@ void DomainBaker::handleFinishedModelBaker() { QUrl oldModelURL { entity[ENTITY_MODEL_URL_KEY].toString() }; // setup a new URL using the prefix we were passed - QUrl newModelURL = _destinationPath.resolved(baker->getBakedFBXRelativePath()); + auto relativeFBXFilePath = baker->getBakedFBXFilePath().remove(_contentOutputPath); + if (relativeFBXFilePath.startsWith("/")) { + relativeFBXFilePath = relativeFBXFilePath.right(relativeFBXFilePath.length() - 1); + } + QUrl newModelURL = _destinationPath.resolved(relativeFBXFilePath); // copy the fragment and query, and user info from the old model URL newModelURL.setQuery(oldModelURL.query()); @@ -335,7 +347,7 @@ void DomainBaker::handleFinishedModelBaker() { if (oldAnimationURL.matches(oldModelURL, QUrl::RemoveQuery | QUrl::RemoveFragment)) { // the animation URL matched the old model URL, so make the animation URL point to the baked FBX // with its original query and fragment - auto newAnimationURL = _destinationPath.resolved(baker->getBakedFBXRelativePath()); + auto newAnimationURL = _destinationPath.resolved(relativeFBXFilePath); newAnimationURL.setQuery(oldAnimationURL.query()); newAnimationURL.setFragment(oldAnimationURL.fragment()); newAnimationURL.setUserInfo(oldAnimationURL.userInfo()); diff --git a/tools/oven/src/DomainBaker.h b/tools/oven/src/DomainBaker.h index 34c5e11e63..6426af0710 100644 --- a/tools/oven/src/DomainBaker.h +++ b/tools/oven/src/DomainBaker.h @@ -55,6 +55,8 @@ private: QString _baseOutputPath; QString _uniqueOutputPath; QString _contentOutputPath; + QString _bakedOutputPath; + QString _originalOutputPath; QUrl _destinationPath; QJsonArray _entities; diff --git a/tools/oven/src/FBXBaker.cpp b/tools/oven/src/FBXBaker.cpp index 0259a6baf8..8ece76b6c4 100644 --- a/tools/oven/src/FBXBaker.cpp +++ b/tools/oven/src/FBXBaker.cpp @@ -25,6 +25,8 @@ #include #include +#include + #include "ModelBakingLoggingCategory.h" #include "TextureBaker.h" @@ -33,12 +35,12 @@ std::once_flag onceFlag; FBXSDKManagerUniquePointer FBXBaker::_sdkManager { nullptr }; -FBXBaker::FBXBaker(const QUrl& fbxURL, const QString& baseOutputPath, - TextureBakerThreadGetter textureThreadGetter, bool copyOriginals) : +FBXBaker::FBXBaker(const QUrl& fbxURL, TextureBakerThreadGetter textureThreadGetter, + const QString& bakedOutputDir, const QString& originalOutputDir) : _fbxURL(fbxURL), - _baseOutputPath(baseOutputPath), - _textureThreadGetter(textureThreadGetter), - _copyOriginals(copyOriginals) + _bakedOutputDir(bakedOutputDir), + _originalOutputDir(originalOutputDir), + _textureThreadGetter(textureThreadGetter) { std::call_once(onceFlag, [](){ // create the static FBX SDK manager @@ -46,21 +48,21 @@ FBXBaker::FBXBaker(const QUrl& fbxURL, const QString& baseOutputPath, manager->Destroy(); }); }); - - // grab the name of the FBX from the URL, this is used for folder output names - auto fileName = fbxURL.fileName(); - _fbxName = fileName.left(fileName.lastIndexOf('.')); -} - -static const QString BAKED_OUTPUT_SUBFOLDER = "baked/"; -static const QString ORIGINAL_OUTPUT_SUBFOLDER = "original/"; - -QString FBXBaker::pathToCopyOfOriginal() const { - return _uniqueOutputPath + ORIGINAL_OUTPUT_SUBFOLDER + _fbxURL.fileName(); } void FBXBaker::bake() { - qCDebug(model_baking) << "Baking" << _fbxURL; + auto tempDir = PathUtils::generateTemporaryDir(); + + if (tempDir.isEmpty()) { + handleError("Failed to create a temporary directory."); + return; + } + + _tempDir = tempDir; + + _originalFBXFilePath = _tempDir.filePath(_fbxURL.fileName()); + qDebug() << "Made temporary dir " << _tempDir; + qDebug() << "Origin file path: " << _originalFBXFilePath; // setup the output folder for the results of this bake setupOutputFolder(); @@ -102,29 +104,25 @@ void FBXBaker::bakeSourceCopy() { } void FBXBaker::setupOutputFolder() { - // construct the output path using the name of the fbx and the base output path - _uniqueOutputPath = _baseOutputPath + "/" + _fbxName + "/"; - // make sure there isn't already an output directory using the same name int iteration = 0; - while (QDir(_uniqueOutputPath).exists()) { - _uniqueOutputPath = _baseOutputPath + "/" + _fbxName + "-" + QString::number(++iteration) + "/"; - } + if (QDir(_bakedOutputDir).exists()) { + qWarning() << "Output path" << _bakedOutputDir << "already exists. Continuing."; + //_bakedOutputDir = _baseOutputPath + "/" + _fbxName + "-" + QString::number(++iteration) + "/"; + } else { + qCDebug(model_baking) << "Creating FBX output folder" << _bakedOutputDir; - qCDebug(model_baking) << "Creating FBX output folder" << _uniqueOutputPath; - - // attempt to make the output folder - if (!QDir().mkdir(_uniqueOutputPath)) { - handleError("Failed to create FBX output folder " + _uniqueOutputPath); - return; - } - - // make the baked and original sub-folders used during export - QDir uniqueOutputDir = _uniqueOutputPath; - if (!uniqueOutputDir.mkdir(BAKED_OUTPUT_SUBFOLDER) || !uniqueOutputDir.mkdir(ORIGINAL_OUTPUT_SUBFOLDER)) { - handleError("Failed to create baked/original subfolders in " + _uniqueOutputPath); - return; + // attempt to make the output folder + if (!QDir().mkpath(_bakedOutputDir)) { + handleError("Failed to create FBX output folder " + _bakedOutputDir); + return; + } + // attempt to make the output folder + if (!QDir().mkpath(_originalOutputDir)) { + handleError("Failed to create FBX output folder " + _bakedOutputDir); + return; + } } } @@ -134,8 +132,21 @@ void FBXBaker::loadSourceFBX() { // load up the local file QFile localFBX { _fbxURL.toLocalFile() }; + qDebug() << "Local file url: " << _fbxURL << _fbxURL.toString() << _fbxURL.toLocalFile() << ", copying to: " << _originalFBXFilePath; + + if (!localFBX.exists()) { + //QMessageBox::warning(this, "Could not find " + _fbxURL.toString(), ""); + handleError("Could not find " + _fbxURL.toString()); + return; + } + // make a copy in the output folder - localFBX.copy(pathToCopyOfOriginal()); + if (!_originalOutputDir.isEmpty()) { + qDebug() << "Copying to: " << _originalOutputDir << "/" << _fbxURL.fileName(); + localFBX.copy(_originalOutputDir + "/" + _fbxURL.fileName()); + } + + localFBX.copy(_originalFBXFilePath); // emit our signal to start the import of the FBX source copy emit sourceCopyReadyToLoad(); @@ -167,19 +178,27 @@ void FBXBaker::handleFBXNetworkReply() { qCDebug(model_baking) << "Downloaded" << _fbxURL; // grab the contents of the reply and make a copy in the output folder - QFile copyOfOriginal(pathToCopyOfOriginal()); + QFile copyOfOriginal(_originalFBXFilePath); - qDebug(model_baking) << "Writing copy of original FBX to" << copyOfOriginal.fileName(); + qDebug(model_baking) << "Writing copy of original FBX to" << _originalFBXFilePath << copyOfOriginal.fileName(); - if (!copyOfOriginal.open(QIODevice::WriteOnly) || (copyOfOriginal.write(requestReply->readAll()) == -1)) { + if (!copyOfOriginal.open(QIODevice::WriteOnly)) { // add an error to the error list for this FBX stating that a duplicate of the original FBX could not be made - handleError("Could not create copy of " + _fbxURL.toString()); + handleError("Could not create copy of " + _fbxURL.toString() + " (Failed to open " + _originalFBXFilePath + ")"); + return; + } + if (copyOfOriginal.write(requestReply->readAll()) == -1) { + handleError("Could not create copy of " + _fbxURL.toString() + " (Failed to write)"); return; } // close that file now that we are done writing to it copyOfOriginal.close(); + if (!_originalOutputDir.isEmpty()) { + copyOfOriginal.copy(_originalOutputDir + "/" + _fbxURL.fileName()); + } + // emit our signal to start the import of the FBX source copy emit sourceCopyReadyToLoad(); } else { @@ -192,9 +211,9 @@ void FBXBaker::importScene() { // create an FBX SDK importer FbxImporter* importer = FbxImporter::Create(_sdkManager.get(), ""); + qDebug() << "file path: " << _originalFBXFilePath.toLocal8Bit().data() << QDir(_originalFBXFilePath).exists(); // import the copy of the original FBX file - QString originalCopyPath = pathToCopyOfOriginal(); - bool importStatus = importer->Initialize(originalCopyPath.toLocal8Bit().data()); + bool importStatus = importer->Initialize(_originalFBXFilePath.toLocal8Bit().data()); if (!importStatus) { // failed to initialize importer, print an error and return @@ -366,8 +385,9 @@ void FBXBaker::rewriteAndBakeSceneTextures() { // even if there was another texture with the same name at a different path auto bakedTextureFileName = createBakedTextureFileName(textureFileInfo); QString bakedTextureFilePath { - _uniqueOutputPath + BAKED_OUTPUT_SUBFOLDER + bakedTextureFileName + _bakedOutputDir + "/" + bakedTextureFileName }; + _outputFiles.push_back(bakedTextureFilePath); qCDebug(model_baking).noquote() << "Re-mapping" << fileTexture->GetFileName() << "to" << bakedTextureFilePath; @@ -376,7 +396,7 @@ void FBXBaker::rewriteAndBakeSceneTextures() { auto urlToTexture = getTextureURL(textureFileInfo, fileTexture); // write the new filename into the FBX scene - fileTexture->SetFileName(bakedTextureFilePath.toLocal8Bit()); + fileTexture->SetFileName(bakedTextureFilePath.toUtf8().data()); // write the relative filename to be the baked texture file name since it will // be right beside the FBX @@ -384,7 +404,7 @@ void FBXBaker::rewriteAndBakeSceneTextures() { if (!_bakingTextures.contains(urlToTexture)) { // bake this texture asynchronously - bakeTexture(urlToTexture, textureType, _uniqueOutputPath + BAKED_OUTPUT_SUBFOLDER); + bakeTexture(urlToTexture, textureType, _bakedOutputDir); } } } @@ -422,7 +442,7 @@ void FBXBaker::handleBakedTexture() { if (bakedTexture) { if (!hasErrors()) { if (!bakedTexture->hasErrors()) { - if (_copyOriginals) { + if (!_originalOutputDir.isEmpty()) { // we've been asked to make copies of the originals, so we need to make copies of this if it is a linked texture // use the path to the texture being baked to determine if this was an embedded or a linked texture @@ -430,7 +450,7 @@ void FBXBaker::handleBakedTexture() { // it is embeddded if the texure being baked was inside the original output folder // since that is where the FBX SDK places the .fbm folder it generates when importing the FBX - auto originalOutputFolder = QUrl::fromLocalFile(_uniqueOutputPath + ORIGINAL_OUTPUT_SUBFOLDER); + auto originalOutputFolder = QUrl::fromLocalFile(_originalOutputDir); if (!originalOutputFolder.isParentOf(bakedTexture->getTextureURL())) { // for linked textures we want to save a copy of original texture beside the original FBX @@ -441,7 +461,7 @@ void FBXBaker::handleBakedTexture() { auto relativeTexturePath = texturePathRelativeToFBX(_fbxURL, bakedTexture->getTextureURL()); QFile originalTextureFile { - _uniqueOutputPath + ORIGINAL_OUTPUT_SUBFOLDER + relativeTexturePath + bakedTexture->getTextureURL().fileName() + _originalOutputDir + "/" + relativeTexturePath + bakedTexture->getTextureURL().fileName() }; if (relativeTexturePath.length() > 0) { @@ -491,38 +511,35 @@ void FBXBaker::exportScene() { // setup the exporter FbxExporter* exporter = FbxExporter::Create(_sdkManager.get(), ""); - auto rewrittenFBXPath = _uniqueOutputPath + BAKED_OUTPUT_SUBFOLDER + _fbxName + BAKED_FBX_EXTENSION; - // save the relative path to this FBX inside our passed output folder - _bakedFBXRelativePath = rewrittenFBXPath; - _bakedFBXRelativePath.remove(_baseOutputPath + "/"); - bool exportStatus = exporter->Initialize(rewrittenFBXPath.toLocal8Bit().data()); + auto fileName = _fbxURL.fileName(); + auto baseName = fileName.left(fileName.lastIndexOf('.')); + auto bakedFilename = baseName + BAKED_FBX_EXTENSION; + + _bakedFBXFilePath = _bakedOutputDir + "/" + bakedFilename; + + bool exportStatus = exporter->Initialize(_bakedFBXFilePath.toLocal8Bit().data()); if (!exportStatus) { // failed to initialize exporter, print an error and return - handleError("Failed to export FBX file at " + _fbxURL.toString() + " to " + rewrittenFBXPath + handleError("Failed to export FBX file at " + _fbxURL.toString() + " to " + _bakedFBXFilePath + "- error: " + exporter->GetStatus().GetErrorString()); } + _outputFiles.push_back(_bakedFBXFilePath); + // export the scene exporter->Export(_scene); - qCDebug(model_baking) << "Exported" << _fbxURL << "with re-written paths to" << rewrittenFBXPath; + qCDebug(model_baking) << "Exported" << _fbxURL << "with re-written paths to" << _bakedFBXFilePath; } void FBXBaker::removeEmbeddedMediaFolder() { // now that the bake is complete, remove the embedded media folder produced by the FBX SDK when it imports an FBX - auto embeddedMediaFolderName = _fbxURL.fileName().replace(".fbx", ".fbm"); - QDir(_uniqueOutputPath + ORIGINAL_OUTPUT_SUBFOLDER + embeddedMediaFolderName).removeRecursively(); -} - -void FBXBaker::possiblyCleanupOriginals() { - if (!_copyOriginals) { - // caller did not ask us to keep the original around, so delete the original output folder now - QDir(_uniqueOutputPath + ORIGINAL_OUTPUT_SUBFOLDER).removeRecursively(); - } + //auto embeddedMediaFolderName = _fbxURL.fileName().replace(".fbx", ".fbm"); + //QDir(_bakedOutputDir + ORIGINAL_OUTPUT_SUBFOLDER + embeddedMediaFolderName).removeRecursively(); } void FBXBaker::checkIfTexturesFinished() { @@ -533,9 +550,6 @@ void FBXBaker::checkIfTexturesFinished() { // remove the embedded media folder that the FBX SDK produces when reading the original removeEmbeddedMediaFolder(); - // cleanup the originals if we weren't asked to keep them around - possiblyCleanupOriginals(); - if (hasErrors()) { // if we're checking for completion but we have errors // that means one or more of our texture baking operations failed diff --git a/tools/oven/src/FBXBaker.h b/tools/oven/src/FBXBaker.h index bcfebbe2a8..faf56ed46b 100644 --- a/tools/oven/src/FBXBaker.h +++ b/tools/oven/src/FBXBaker.h @@ -37,11 +37,12 @@ using TextureBakerThreadGetter = std::function; class FBXBaker : public Baker { Q_OBJECT public: - FBXBaker(const QUrl& fbxURL, const QString& baseOutputPath, - TextureBakerThreadGetter textureThreadGetter, bool copyOriginals = true); + FBXBaker(const QUrl& fbxURL, TextureBakerThreadGetter textureThreadGetter, + const QString& bakedOutputDir, const QString& originalOutputDir = ""); QUrl getFBXUrl() const { return _fbxURL; } - QString getBakedFBXRelativePath() const { return _bakedFBXRelativePath; } + QString getBakedFBXFilePath() const { return _bakedFBXFilePath; } + std::vector getOutputFiles() const { return _outputFiles; } public slots: // all calls to FBXBaker::bake for FBXBaker instances must be from the same thread @@ -61,13 +62,10 @@ private: void loadSourceFBX(); - void bakeCopiedFBX(); - void importScene(); void rewriteAndBakeSceneTextures(); void exportScene(); void removeEmbeddedMediaFolder(); - void possiblyCleanupOriginals(); void checkIfTexturesFinished(); @@ -76,14 +74,20 @@ private: void bakeTexture(const QUrl& textureURL, image::TextureUsage::Type textureType, const QDir& outputDir); - QString pathToCopyOfOriginal() const; - QUrl _fbxURL; - QString _fbxName; - QString _baseOutputPath; - QString _uniqueOutputPath; - QString _bakedFBXRelativePath; + QString _bakedFBXFilePath; + + QString _bakedOutputDir; + + // If set, the original FBX and textures will also be copied here + QString _originalOutputDir; + + QDir _tempDir; + QString _originalFBXFilePath; + + // List of baked output files, includes the FBX and textures + std::vector _outputFiles; static FBXSDKManagerUniquePointer _sdkManager; fbxsdk::FbxScene* _scene { nullptr }; @@ -93,8 +97,6 @@ private: TextureBakerThreadGetter _textureThreadGetter; - bool _copyOriginals { true }; - bool _pendingErrorEmission { false }; }; diff --git a/tools/oven/src/ui/ModelBakeWidget.cpp b/tools/oven/src/ui/ModelBakeWidget.cpp index c696fbad26..926de8bae0 100644 --- a/tools/oven/src/ui/ModelBakeWidget.cpp +++ b/tools/oven/src/ui/ModelBakeWidget.cpp @@ -15,6 +15,7 @@ #include #include #include +#include #include #include @@ -158,12 +159,23 @@ void ModelBakeWidget::bakeButtonClicked() { // make sure we have a valid output directory QDir outputDirectory(_outputDirLineEdit->text()); + outputDirectory.mkdir("."); if (!outputDirectory.exists()) { + QMessageBox::warning(this, "Unable to create directory", "Unable to create output directory. Please create it manually or choose a different directory."); return; } + QDir bakedOutputDirectory = outputDirectory.absoluteFilePath("baked"); + QDir originalOutputDirectory = outputDirectory.absoluteFilePath("original"); + + bakedOutputDirectory.mkdir("."); + originalOutputDirectory.mkdir("."); + + + // make sure we have a non empty URL to a model to bake if (_modelLineEdit->text().isEmpty()) { + QMessageBox::warning(this, "Model URL unspecified", "A model file is required."); return; } @@ -175,14 +187,17 @@ void ModelBakeWidget::bakeButtonClicked() { // if the URL doesn't have a scheme, assume it is a local file if (modelToBakeURL.scheme() != "http" && modelToBakeURL.scheme() != "https" && modelToBakeURL.scheme() != "ftp") { - modelToBakeURL.setScheme("file"); + qDebug() << modelToBakeURL.toString(); + qDebug() << modelToBakeURL.scheme(); + modelToBakeURL = QUrl::fromLocalFile(fileURLString); + qDebug() << "New url: " << modelToBakeURL; } // everything seems to be in place, kick off a bake for this model now auto baker = std::unique_ptr { - new FBXBaker(modelToBakeURL, outputDirectory.absolutePath(), []() -> QThread* { + new FBXBaker(modelToBakeURL, []() -> QThread* { return qApp->getNextWorkerThread(); - }, false) + }, bakedOutputDirectory.absolutePath(), originalOutputDirectory.absolutePath()) }; // move the baker to the FBX baker thread @@ -211,6 +226,10 @@ void ModelBakeWidget::handleFinishedBaker() { return value.first.get() == baker; }); + for (auto& file : baker->getOutputFiles()) { + qDebug() << "Baked file: " << file; + } + if (it != _bakers.end()) { auto resultRow = it->second; auto resultsWindow = qApp->getMainWindow()->showResultsWindow(); diff --git a/tools/oven/src/ui/ResultsWindow.cpp b/tools/oven/src/ui/ResultsWindow.cpp index 35b5160f9b..3a37a328de 100644 --- a/tools/oven/src/ui/ResultsWindow.cpp +++ b/tools/oven/src/ui/ResultsWindow.cpp @@ -50,6 +50,7 @@ void ResultsWindow::setupUI() { // strech the last column of the table (that holds the results) to fill up the remaining available size _resultsTable->horizontalHeader()->resizeSection(0, 0.25 * FIXED_WINDOW_WIDTH); _resultsTable->horizontalHeader()->setSectionResizeMode(1, QHeaderView::ResizeToContents); + _resultsTable->setSizePolicy(QSizePolicy::Expanding, QSizePolicy::Expanding); // make sure we hear about cell clicks so that we can show the output directory for the given row connect(_resultsTable, &QTableWidget::cellClicked, this, &ResultsWindow::handleCellClicked); diff --git a/tools/oven/src/ui/SkyboxBakeWidget.cpp b/tools/oven/src/ui/SkyboxBakeWidget.cpp index d5c280aebd..cbaaa5ec0a 100644 --- a/tools/oven/src/ui/SkyboxBakeWidget.cpp +++ b/tools/oven/src/ui/SkyboxBakeWidget.cpp @@ -13,6 +13,7 @@ #include #include #include +#include #include #include @@ -155,7 +156,9 @@ void SkyboxBakeWidget::bakeButtonClicked() { // make sure we have a valid output directory QDir outputDirectory(_outputDirLineEdit->text()); + outputDirectory.mkdir("."); if (!outputDirectory.exists()) { + QMessageBox::warning(this, "Unable to create directory", "Unable to create output directory. Please create it manually or choose a different directory."); return; } From 39f04adc8db1d9861815313af960e33599e77531 Mon Sep 17 00:00:00 2001 From: Ryan Huffman Date: Wed, 16 Aug 2017 13:51:40 -0700 Subject: [PATCH 010/129] Improve baking interface and add automatic baking to asset server --- assignment-client/CMakeLists.txt | 2 +- assignment-client/src/assets/AssetServer.cpp | 190 +++++- assignment-client/src/assets/AssetServer.h | 35 +- cmake/modules/FindFBX.cmake | 2 +- libraries/baking/CMakeLists.txt | 17 + .../oven => libraries/baking}/src/Baker.cpp | 0 {tools/oven => libraries/baking}/src/Baker.h | 0 libraries/baking/src/FBXBaker.cpp | 568 ++++++++++++++++++ libraries/baking/src/FBXBaker.h | 105 ++++ .../src/ModelBakingLoggingCategory.cpp | 0 .../baking}/src/ModelBakingLoggingCategory.h | 0 libraries/baking/src/TextureBaker.cpp | 131 ++++ libraries/baking/src/TextureBaker.h | 59 ++ libraries/fbx/CMakeLists.txt | 7 +- libraries/fbx/src/FBXReader.cpp | 4 + .../src/model-networking/ModelCache.cpp | 11 +- .../src/model-networking/ModelCache.h | 1 + .../src/model-networking/TextureCache.cpp | 10 +- .../networking/src/AssetResourceRequest.cpp | 6 +- libraries/networking/src/MappingRequest.cpp | 3 + libraries/networking/src/ResourceCache.cpp | 7 + libraries/networking/src/ResourceCache.h | 1 + libraries/networking/src/ResourceRequest.h | 1 + libraries/networking/src/udt/PacketHeaders.h | 15 +- tools/oven/CMakeLists.txt | 2 +- 25 files changed, 1139 insertions(+), 38 deletions(-) create mode 100644 libraries/baking/CMakeLists.txt rename {tools/oven => libraries/baking}/src/Baker.cpp (100%) rename {tools/oven => libraries/baking}/src/Baker.h (100%) create mode 100644 libraries/baking/src/FBXBaker.cpp create mode 100644 libraries/baking/src/FBXBaker.h rename {tools/oven => libraries/baking}/src/ModelBakingLoggingCategory.cpp (100%) rename {tools/oven => libraries/baking}/src/ModelBakingLoggingCategory.h (100%) create mode 100644 libraries/baking/src/TextureBaker.cpp create mode 100644 libraries/baking/src/TextureBaker.h diff --git a/assignment-client/CMakeLists.txt b/assignment-client/CMakeLists.txt index 1a27ddd479..0421195612 100644 --- a/assignment-client/CMakeLists.txt +++ b/assignment-client/CMakeLists.txt @@ -13,7 +13,7 @@ setup_memory_debugger() link_hifi_libraries( audio avatars octree gpu model fbx entities networking animation recording shared script-engine embedded-webserver - controllers physics plugins midi + controllers physics plugins midi baking image ) if (WIN32) diff --git a/assignment-client/src/assets/AssetServer.cpp b/assignment-client/src/assets/AssetServer.cpp index 2082856f56..6e3195f570 100644 --- a/assignment-client/src/assets/AssetServer.cpp +++ b/assignment-client/src/assets/AssetServer.cpp @@ -24,6 +24,8 @@ #include #include #include +#include +#include #include #include @@ -33,6 +35,9 @@ #include "SendAssetTask.h" #include "UploadAssetTask.h" #include +#include + +#include static const uint8_t MIN_CORES_FOR_MULTICORE = 4; static const uint8_t CPU_AFFINITY_COUNT_HIGH = 2; @@ -43,6 +48,128 @@ static const int INTERFACE_RUNNING_CHECK_FREQUENCY_MS = 1000; const QString ASSET_SERVER_LOGGING_TARGET_NAME = "asset-server"; +static const QStringList BAKEABLE_MODEL_EXTENSIONS = { "fbx" }; +static const QList BAKEABLE_TEXTURE_EXTENSIONS = QImageReader::supportedImageFormats(); +static const QString BAKED_MODEL_SIMPLE_NAME = "asset.fbx"; +static const QString BAKED_TEXTURE_SIMPLE_NAME = "texture.ktx"; + +BakeAssetTask::BakeAssetTask(const QString& assetHash, const QString& assetPath, const QString& filePath) + : _assetHash(assetHash), _assetPath(assetPath), _filePath(filePath) { +} + +void BakeAssetTask::run() { + qRegisterMetaType >("QVector"); + TextureBakerThreadGetter fn = []() -> QThread* { return QThread::currentThread(); }; + + if (_filePath.endsWith(".fbx")) { + FBXBaker baker(QUrl("file:///" + _filePath), fn, PathUtils::generateTemporaryDir()); + + QEventLoop loop; + connect(&baker, &Baker::finished, &loop, &QEventLoop::quit); + QMetaObject::invokeMethod(&baker, "bake", Qt::QueuedConnection); + qDebug() << "Running the bake!"; + loop.exec(); + + qDebug() << "Finished baking: " << _assetHash << _assetPath << baker.getOutputFiles(); + emit bakeComplete(_assetHash, _assetPath, QVector::fromStdVector(baker.getOutputFiles())); + } else { + TextureBaker baker(QUrl("file:///" + _filePath), image::TextureUsage::CUBE_TEXTURE, PathUtils::generateTemporaryDir()); + + QEventLoop loop; + connect(&baker, &Baker::finished, &loop, &QEventLoop::quit); + QMetaObject::invokeMethod(&baker, "bake", Qt::QueuedConnection); + qDebug() << "Running the bake!"; + loop.exec(); + + qDebug() << "Finished baking: " << _assetHash << _assetPath << baker.getBakedTextureFileName(); + emit bakeComplete(_assetHash, _assetPath, { baker.getDestinationFilePath() }); + } +} + +void AssetServer::bakeAsset(const QString& assetHash, const QString& assetPath, const QString& filePath) { + qDebug() << "Starting bake for: " << assetPath << assetHash; + auto it = _pendingBakes.find(assetHash); + if (it == _pendingBakes.end()) { + auto task = std::make_shared(assetHash, assetPath, filePath); + task->setAutoDelete(false); + _pendingBakes[assetHash] = task; + + connect(task.get(), &BakeAssetTask::bakeComplete, this, [this, assetPath](QString assetHash, QString assetPath, QVector outputFiles) { + handleCompletedBake(assetPath, assetHash, outputFiles); + }); + + _bakingTaskPool.start(task.get()); + } else { + qDebug() << "Already in queue"; + } +} + +QString AssetServer::getPathToAssetHash(const AssetHash& assetHash) { + return _filesDirectory.absoluteFilePath(assetHash); +} + +void AssetServer::bakeAssets() { + auto it = _fileMappings.cbegin(); + for (; it != _fileMappings.cend(); ++it) { + auto path = it.key(); + auto hash = it.value().toString(); + maybeBake(path, hash); + } +} + +void AssetServer::maybeBake(const AssetPath& path, const AssetHash& hash) { + if (needsToBeBaked(path, hash)) { + qDebug() << "Queuing bake of: " << path; + bakeAsset(hash, path, getPathToAssetHash(hash)); + } +} + +void AssetServer::createEmptyMetaFile(const AssetHash& hash) { + QString metaFilePath = "atp:/" + hash + "/meta.json"; + QFile metaFile { metaFilePath }; + + if (!metaFile.exists()) { + qDebug() << "Creating metfaile for " << hash; + if (metaFile.open(QFile::WriteOnly)) { + qDebug() << "Created metfaile for " << hash; + metaFile.write("{}"); + } + } +} + +bool AssetServer::hasMetaFile(const AssetHash& hash) { + QString metaFilePath = "/.baked/" + hash + "/meta.json"; + qDebug() << "in mappings?" << metaFilePath; + + return _fileMappings.contains(metaFilePath); +} + +bool AssetServer::needsToBeBaked(const AssetPath& path, const AssetHash& assetHash) { + if (path.startsWith("/.baked/")) { + return false; + } + + auto dotIndex = path.lastIndexOf("."); + if (dotIndex == -1) { + return false; + } + + auto extension = path.mid(dotIndex + 1); + + QString bakedFilename; + + if (BAKEABLE_MODEL_EXTENSIONS.contains(extension)) { + bakedFilename = BAKED_MODEL_SIMPLE_NAME; + } else if (BAKEABLE_TEXTURE_EXTENSIONS.contains(extension.toLocal8Bit()) && hasMetaFile(assetHash)) { + bakedFilename = BAKED_TEXTURE_SIMPLE_NAME; + } else { + return false; + } + + auto bakedPath = "/.baked/" + assetHash + "/" + bakedFilename; + return !_fileMappings.contains(bakedPath); +} + bool interfaceRunning() { bool result = false; @@ -76,13 +203,15 @@ void updateConsumedCores() { AssetServer::AssetServer(ReceivedMessage& message) : ThreadedAssignment(message), - _taskPool(this) + _transferTaskPool(this), + _bakingTaskPool(this) { // Most of the work will be I/O bound, reading from disk and constructing packet objects, // so the ideal is greater than the number of cores on the system. static const int TASK_POOL_THREAD_COUNT = 50; - _taskPool.setMaxThreadCount(TASK_POOL_THREAD_COUNT); + _transferTaskPool.setMaxThreadCount(TASK_POOL_THREAD_COUNT); + _bakingTaskPool.setMaxThreadCount(1); auto& packetReceiver = DependencyManager::get()->getPacketReceiver(); packetReceiver.registerListener(PacketType::AssetGet, this, "handleAssetGet"); @@ -196,6 +325,8 @@ void AssetServer::completeSetup() { } nodeList->addSetOfNodeTypesToNodeInterestSet({ NodeType::Agent, NodeType::EntityScriptServer }); + + bakeAssets(); } else { qCritical() << "Asset Server assignment will not continue because mapping file could not be loaded."; setFinished(true); @@ -265,27 +396,27 @@ void AssetServer::handleAssetMappingOperation(QSharedPointer me nodeList->sendPacketList(std::move(replyPacket), *senderNode); } -static const QStringList BAKEABLE_MODEL_EXTENSIONS = { ".fbx" }; -static const QString BAKED_MODEL_SIMPLE_NAME = "asset.fbx"; -static const QString BAKED_TEXTURE_SIMPLE_NAME = "texture.ktx"; - void AssetServer::handleGetMappingOperation(ReceivedMessage& message, SharedNodePointer senderNode, NLPacketList& replyPacket) { QString assetPath = message.readString(); + QUrl url { assetPath }; + assetPath = url.path(); + auto it = _fileMappings.find(assetPath); if (it != _fileMappings.end()) { // check if we should re-direct to a baked asset // first, figure out from the mapping extension what type of file this is - auto assetPathExtension = assetPath.mid(assetPath.lastIndexOf('.')).toLower(); + auto assetPathExtension = assetPath.mid(assetPath.lastIndexOf('.') + 1).toLower(); QString bakedRootFile; if (BAKEABLE_MODEL_EXTENSIONS.contains(assetPathExtension)) { bakedRootFile = BAKED_MODEL_SIMPLE_NAME; - } else if (QImageReader::supportedImageFormats().contains(assetPathExtension.toLocal8Bit())) { + } else if (BAKEABLE_TEXTURE_EXTENSIONS.contains(assetPathExtension.toLocal8Bit())) { bakedRootFile = BAKED_TEXTURE_SIMPLE_NAME; } + qDebug() << bakedRootFile << assetPathExtension; auto originalAssetHash = it->toString(); QString redirectedAssetHash; @@ -298,9 +429,12 @@ void AssetServer::handleGetMappingOperation(ReceivedMessage& message, SharedNode auto bakedIt = _fileMappings.find(bakedAssetPath); if (bakedIt != _fileMappings.end()) { + qDebug() << "Did find baked version for: " << originalAssetHash << assetPath; // we found a baked version of the requested asset to serve, redirect to that redirectedAssetHash = bakedIt->toString(); wasRedirected = true; + } else { + qDebug() << "Did not find baked version for: " << originalAssetHash << assetPath; } } @@ -319,6 +453,15 @@ void AssetServer::handleGetMappingOperation(ReceivedMessage& message, SharedNode } else { replyPacket.write(QByteArray::fromHex(originalAssetHash.toUtf8())); replyPacket.writePrimitive(wasRedirected); + + + auto query = QUrlQuery(url.query()); + bool isSkybox = query.hasQueryItem("skybox"); + qDebug() << "Is skybox? " << isSkybox; + if (isSkybox) { + createMetaFile(originalAssetHash); + maybeBake(originalAssetHash, assetPath); + } } } else { replyPacket.writePrimitive(AssetServerError::AssetNotFound); @@ -437,7 +580,7 @@ void AssetServer::handleAssetGet(QSharedPointer message, Shared // Queue task auto task = new SendAssetTask(message, senderNode, _filesDirectory); - _taskPool.start(task); + _transferTaskPool.start(task); } void AssetServer::handleAssetUpload(QSharedPointer message, SharedNodePointer senderNode) { @@ -446,7 +589,7 @@ void AssetServer::handleAssetUpload(QSharedPointer message, Sha qDebug() << "Starting an UploadAssetTask for upload from" << uuidStringWithoutCurlyBraces(senderNode->getUUID()); auto task = new UploadAssetTask(message, senderNode, _filesDirectory); - _taskPool.start(task); + _transferTaskPool.start(task); } else { // this is a node the domain told us is not allowed to rez entities // for now this also means it isn't allowed to add assets @@ -632,6 +775,7 @@ bool AssetServer::setMapping(AssetPath path, AssetHash hash) { if (writeMappingsToFile()) { // persistence succeeded, we are good to go qDebug() << "Set mapping:" << path << "=>" << hash; + maybeBake(path, hash); return true; } else { // failed to persist this mapping to file - put back the old one in our in-memory representation @@ -836,18 +980,17 @@ bool AssetServer::renameMapping(AssetPath oldPath, AssetPath newPath) { static const QString HIDDEN_BAKED_CONTENT_FOLDER = "/.baked/"; -void AssetServer::handleCompletedBake(AssetHash originalAssetHash, QDir temporaryOutputDir) { - // enumerate the baking result files in the temporary directory - QDirIterator dirIterator(temporaryOutputDir.absolutePath(), QDir::Files, QDirIterator::Subdirectories); - +void AssetServer::handleCompletedBake(AssetPath originalAssetPath, AssetHash originalAssetHash, QVector bakedFilePaths) { bool errorCompletingBake { false }; - while (dirIterator.hasNext()) { - QString filePath = dirIterator.next(); + qDebug() << "Completing bake for " << originalAssetHash; + for (auto& filePath: bakedFilePaths) { // figure out the hash for the contents of this file QFile file(filePath); + qDebug() << "File path: " << filePath; + AssetHash bakedFileHash; if (file.open(QIODevice::ReadOnly)) { @@ -873,26 +1016,33 @@ void AssetServer::handleCompletedBake(AssetHash originalAssetHash, QDir temporar } // setup the mapping for this bake file - auto relativeFilePath = temporaryOutputDir.relativeFilePath(filePath); - static const QString BAKED_ASSET_SIMPLE_NAME = "asset.fbx"; + auto relativeFilePath = QUrl(filePath).fileName(); + qDebug() << "Relative file path is: " << relativeFilePath; + static const QString BAKED_ASSET_SIMPLE_FBX_NAME = "asset.fbx"; + static const QString BAKED_ASSET_SIMPLE_TEXTURE_NAME = "texture.ktx"; if (relativeFilePath.endsWith(".fbx", Qt::CaseInsensitive)) { // for an FBX file, we replace the filename with the simple name // (to handle the case where two mapped assets have the same hash but different names) - relativeFilePath = BAKED_ASSET_SIMPLE_NAME; + relativeFilePath = BAKED_ASSET_SIMPLE_FBX_NAME; + } else if (!originalAssetPath.endsWith(".fbx")) { + relativeFilePath = BAKED_ASSET_SIMPLE_TEXTURE_NAME; + } QString bakeMapping = HIDDEN_BAKED_CONTENT_FOLDER + originalAssetHash + "/" + relativeFilePath; // add a mapping (under the hidden baked folder) for this file resulting from the bake - if (setMapping(bakeMapping , bakedFileHash)) { + if (setMapping(bakeMapping, bakedFileHash)) { qDebug() << "Added" << bakeMapping << "for bake file" << bakedFileHash << "from bake of" << originalAssetHash; } else { + qDebug() << "Failed to set mapping"; // stop handling this bake, couldn't add a mapping for this bake file errorCompletingBake = true; break; } } else { + qDebug() << "Failed to open baked file: " << filePath; // stop handling this bake, we couldn't open one of the files for reading errorCompletingBake = true; break; diff --git a/assignment-client/src/assets/AssetServer.h b/assignment-client/src/assets/AssetServer.h index 191ad25b1f..2dfc5f6c35 100644 --- a/assignment-client/src/assets/AssetServer.h +++ b/assignment-client/src/assets/AssetServer.h @@ -14,12 +14,29 @@ #include #include +#include #include #include "AssetUtils.h" #include "ReceivedMessage.h" +class BakeAssetTask : public QObject, public QRunnable { + Q_OBJECT +public: + BakeAssetTask(const QString& assetHash, const QString& assetPath, const QString& filePath); + + void run() override; + +signals: + void bakeComplete(QString assetHash, QString assetPath, QVector outputFiles); + +private: + QString _assetHash; + QString _assetPath; + QString _filePath; +}; + class AssetServer : public ThreadedAssignment { Q_OBJECT public: @@ -63,8 +80,17 @@ private: /// Delete any unmapped files from the local asset directory void cleanupUnmappedFiles(); + QString getPathToAssetHash(const AssetHash& assetHash); + + void bakeAssets(); + void maybeBake(const AssetPath& path, const AssetHash& hash); + void createEmptyMetaFile(const AssetHash& hash); + bool hasMetaFile(const AssetHash& hash); + bool needsToBeBaked(const AssetPath& path, const AssetHash& assetHash); + void bakeAsset(const QString& assetHash, const QString& assetPath, const QString& filePath); + /// Move baked content for asset to baked directory and update baked status - void handleCompletedBake(AssetHash originalAssetHash, QDir temporaryOutputDir); + void handleCompletedBake(AssetPath assetPath, AssetHash originalAssetHash, QVector bakedFilePaths); /// Create meta file to describe baked content for original asset bool createMetaFile(AssetHash originalAssetHash); @@ -73,7 +99,12 @@ private: QDir _resourcesDirectory; QDir _filesDirectory; - QThreadPool _taskPool; + + /// Task pool for handling uploads and downloads of assets + QThreadPool _transferTaskPool; + + QHash> _pendingBakes; + QThreadPool _bakingTaskPool; }; #endif diff --git a/cmake/modules/FindFBX.cmake b/cmake/modules/FindFBX.cmake index 9a1d08a010..6eb6d11f9d 100644 --- a/cmake/modules/FindFBX.cmake +++ b/cmake/modules/FindFBX.cmake @@ -57,7 +57,7 @@ endif() function(_fbx_find_library _name _lib _suffix) if (MSVC_VERSION EQUAL 1910) - set(VS_PREFIX vs2017) + set(VS_PREFIX vs2015) elseif (MSVC_VERSION EQUAL 1900) set(VS_PREFIX vs2015) elseif (MSVC_VERSION EQUAL 1800) diff --git a/libraries/baking/CMakeLists.txt b/libraries/baking/CMakeLists.txt new file mode 100644 index 0000000000..44c52ae0d8 --- /dev/null +++ b/libraries/baking/CMakeLists.txt @@ -0,0 +1,17 @@ +set(TARGET_NAME baking) +setup_hifi_library(Concurrent) + +find_package(FBX) +if (FBX_FOUND) + if (CMAKE_THREAD_LIBS_INIT) + target_link_libraries(${TARGET_NAME} ${FBX_LIBRARIES} "${CMAKE_THREAD_LIBS_INIT}") + else () + target_link_libraries(${TARGET_NAME} ${FBX_LIBRARIES}) + endif () + target_include_directories(${TARGET_NAME} SYSTEM PRIVATE ${FBX_INCLUDE_DIR}) +endif () + +set_target_properties(${TARGET_NAME} PROPERTIES EXCLUDE_FROM_ALL TRUE EXCLUDE_FROM_DEFAULT_BUILD TRUE) + +link_hifi_libraries(shared model networking ktx image) +include_hifi_library_headers(gpu) diff --git a/tools/oven/src/Baker.cpp b/libraries/baking/src/Baker.cpp similarity index 100% rename from tools/oven/src/Baker.cpp rename to libraries/baking/src/Baker.cpp diff --git a/tools/oven/src/Baker.h b/libraries/baking/src/Baker.h similarity index 100% rename from tools/oven/src/Baker.h rename to libraries/baking/src/Baker.h diff --git a/libraries/baking/src/FBXBaker.cpp b/libraries/baking/src/FBXBaker.cpp new file mode 100644 index 0000000000..1a11071b5b --- /dev/null +++ b/libraries/baking/src/FBXBaker.cpp @@ -0,0 +1,568 @@ +// +// FBXBaker.cpp +// tools/oven/src +// +// Created by Stephen Birarda on 3/30/17. +// Copyright 2017 High Fidelity, Inc. +// +// Distributed under the Apache License, Version 2.0. +// See the accompanying file LICENSE or http://www.apache.org/licenses/LICENSE-2.0.html +// + +#include // need this include so we don't get an error looking for std::isnan + +#include + +#include +#include +#include +#include +#include +#include + +#include + +#include +#include + +#include + +#include "ModelBakingLoggingCategory.h" +#include "TextureBaker.h" + +#include "FBXBaker.h" + +std::once_flag onceFlag; +FBXSDKManagerUniquePointer FBXBaker::_sdkManager { nullptr }; + +FBXBaker::FBXBaker(const QUrl& fbxURL, TextureBakerThreadGetter textureThreadGetter, + const QString& bakedOutputDir, const QString& originalOutputDir) : + _fbxURL(fbxURL), + _bakedOutputDir(bakedOutputDir), + _originalOutputDir(originalOutputDir), + _textureThreadGetter(textureThreadGetter) +{ + std::call_once(onceFlag, [](){ + // create the static FBX SDK manager + _sdkManager = FBXSDKManagerUniquePointer(FbxManager::Create(), [](FbxManager* manager){ + manager->Destroy(); + }); + }); +} + +void FBXBaker::bake() { + auto tempDir = PathUtils::generateTemporaryDir(); + + if (tempDir.isEmpty()) { + handleError("Failed to create a temporary directory."); + return; + } + + _tempDir = tempDir; + + _originalFBXFilePath = _tempDir.filePath(_fbxURL.fileName()); + qDebug() << "Made temporary dir " << _tempDir; + qDebug() << "Origin file path: " << _originalFBXFilePath; + + // setup the output folder for the results of this bake + setupOutputFolder(); + + if (hasErrors()) { + return; + } + + connect(this, &FBXBaker::sourceCopyReadyToLoad, this, &FBXBaker::bakeSourceCopy); + + // make a local copy of the FBX file + loadSourceFBX(); +} + +void FBXBaker::bakeSourceCopy() { + // load the scene from the FBX file + importScene(); + + if (hasErrors()) { + return; + } + + // enumerate the textures found in the scene and start a bake for them + rewriteAndBakeSceneTextures(); + + if (hasErrors()) { + return; + } + + // export the FBX with re-written texture references + exportScene(); + + if (hasErrors()) { + return; + } + + // check if we're already done with textures (in case we had none to re-write) + checkIfTexturesFinished(); +} + +void FBXBaker::setupOutputFolder() { + // make sure there isn't already an output directory using the same name + int iteration = 0; + + if (QDir(_bakedOutputDir).exists()) { + qWarning() << "Output path" << _bakedOutputDir << "already exists. Continuing."; + //_bakedOutputDir = _baseOutputPath + "/" + _fbxName + "-" + QString::number(++iteration) + "/"; + } else { + qCDebug(model_baking) << "Creating FBX output folder" << _bakedOutputDir; + + // attempt to make the output folder + if (!QDir().mkpath(_bakedOutputDir)) { + handleError("Failed to create FBX output folder " + _bakedOutputDir); + return; + } + // attempt to make the output folder + if (!QDir().mkpath(_originalOutputDir)) { + handleError("Failed to create FBX output folder " + _bakedOutputDir); + return; + } + } +} + +void FBXBaker::loadSourceFBX() { + // check if the FBX is local or first needs to be downloaded + if (_fbxURL.isLocalFile()) { + // load up the local file + QFile localFBX { _fbxURL.toLocalFile() }; + + qDebug() << "Local file url: " << _fbxURL << _fbxURL.toString() << _fbxURL.toLocalFile() << ", copying to: " << _originalFBXFilePath; + + if (!localFBX.exists()) { + //QMessageBox::warning(this, "Could not find " + _fbxURL.toString(), ""); + handleError("Could not find " + _fbxURL.toString()); + return; + } + + // make a copy in the output folder + if (!_originalOutputDir.isEmpty()) { + qDebug() << "Copying to: " << _originalOutputDir << "/" << _fbxURL.fileName(); + localFBX.copy(_originalOutputDir + "/" + _fbxURL.fileName()); + } + + localFBX.copy(_originalFBXFilePath); + + // emit our signal to start the import of the FBX source copy + emit sourceCopyReadyToLoad(); + } else { + // remote file, kick off a download + auto& networkAccessManager = NetworkAccessManager::getInstance(); + + QNetworkRequest networkRequest; + + // setup the request to follow re-directs and always hit the network + networkRequest.setAttribute(QNetworkRequest::FollowRedirectsAttribute, true); + networkRequest.setAttribute(QNetworkRequest::CacheLoadControlAttribute, QNetworkRequest::AlwaysNetwork); + networkRequest.setHeader(QNetworkRequest::UserAgentHeader, HIGH_FIDELITY_USER_AGENT); + + + networkRequest.setUrl(_fbxURL); + + qCDebug(model_baking) << "Downloading" << _fbxURL; + auto networkReply = networkAccessManager.get(networkRequest); + + connect(networkReply, &QNetworkReply::finished, this, &FBXBaker::handleFBXNetworkReply); + } +} + +void FBXBaker::handleFBXNetworkReply() { + auto requestReply = qobject_cast(sender()); + + if (requestReply->error() == QNetworkReply::NoError) { + qCDebug(model_baking) << "Downloaded" << _fbxURL; + + // grab the contents of the reply and make a copy in the output folder + QFile copyOfOriginal(_originalFBXFilePath); + + qDebug(model_baking) << "Writing copy of original FBX to" << _originalFBXFilePath << copyOfOriginal.fileName(); + + if (!copyOfOriginal.open(QIODevice::WriteOnly)) { + // add an error to the error list for this FBX stating that a duplicate of the original FBX could not be made + handleError("Could not create copy of " + _fbxURL.toString() + " (Failed to open " + _originalFBXFilePath + ")"); + return; + } + if (copyOfOriginal.write(requestReply->readAll()) == -1) { + handleError("Could not create copy of " + _fbxURL.toString() + " (Failed to write)"); + return; + } + + // close that file now that we are done writing to it + copyOfOriginal.close(); + + if (!_originalOutputDir.isEmpty()) { + copyOfOriginal.copy(_originalOutputDir + "/" + _fbxURL.fileName()); + } + + // emit our signal to start the import of the FBX source copy + emit sourceCopyReadyToLoad(); + } else { + // add an error to our list stating that the FBX could not be downloaded + handleError("Failed to download " + _fbxURL.toString()); + } +} + +void FBXBaker::importScene() { + // create an FBX SDK importer + FbxImporter* importer = FbxImporter::Create(_sdkManager.get(), ""); + + qDebug() << "file path: " << _originalFBXFilePath.toLocal8Bit().data() << QDir(_originalFBXFilePath).exists(); + // import the copy of the original FBX file + bool importStatus = importer->Initialize(_originalFBXFilePath.toLocal8Bit().data()); + + if (!importStatus) { + // failed to initialize importer, print an error and return + handleError("Failed to import " + _fbxURL.toString() + " - " + importer->GetStatus().GetErrorString()); + return; + } else { + qCDebug(model_baking) << "Imported" << _fbxURL << "to FbxScene"; + } + + // setup a new scene to hold the imported file + _scene = FbxScene::Create(_sdkManager.get(), "bakeScene"); + + // import the file to the created scene + importer->Import(_scene); + + // destroy the importer that is no longer needed + importer->Destroy(); +} + +QString texturePathRelativeToFBX(QUrl fbxURL, QUrl textureURL) { + auto fbxPath = fbxURL.toString(QUrl::RemoveFilename | QUrl::RemoveQuery | QUrl::RemoveFragment); + auto texturePath = textureURL.toString(QUrl::RemoveFilename | QUrl::RemoveQuery | QUrl::RemoveFragment); + + if (texturePath.startsWith(fbxPath)) { + // texture path is a child of the FBX path, return the texture path without the fbx path + return texturePath.mid(fbxPath.length()); + } else { + // the texture path was not a child of the FBX path, return the empty string + return ""; + } +} + +QString FBXBaker::createBakedTextureFileName(const QFileInfo& textureFileInfo) { + // first make sure we have a unique base name for this texture + // in case another texture referenced by this model has the same base name + auto nameMatches = _textureNameMatchCount[textureFileInfo.baseName()]; + + QString bakedTextureFileName { textureFileInfo.completeBaseName() }; + + if (nameMatches > 0) { + // there are already nameMatches texture with this name + // append - and that number to our baked texture file name so that it is unique + bakedTextureFileName += "-" + QString::number(nameMatches); + } + + bakedTextureFileName += BAKED_TEXTURE_EXT; + + // increment the number of name matches + ++nameMatches; + + return bakedTextureFileName; +} + +QUrl FBXBaker::getTextureURL(const QFileInfo& textureFileInfo, FbxFileTexture* fileTexture) { + QUrl urlToTexture; + + if (textureFileInfo.exists() && textureFileInfo.isFile()) { + // set the texture URL to the local texture that we have confirmed exists + urlToTexture = QUrl::fromLocalFile(textureFileInfo.absoluteFilePath()); + } else { + // external texture that we'll need to download or find + + // first check if it the RelativePath to the texture in the FBX was relative + QString relativeFileName = fileTexture->GetRelativeFileName(); + auto apparentRelativePath = QFileInfo(relativeFileName.replace("\\", "/")); + + // this is a relative file path which will require different handling + // depending on the location of the original FBX + if (_fbxURL.isLocalFile() && apparentRelativePath.exists() && apparentRelativePath.isFile()) { + // the absolute path we ran into for the texture in the FBX exists on this machine + // so use that file + urlToTexture = QUrl::fromLocalFile(apparentRelativePath.absoluteFilePath()); + } else { + // we didn't find the texture on this machine at the absolute path + // so assume that it is right beside the FBX to match the behaviour of interface + urlToTexture = _fbxURL.resolved(apparentRelativePath.fileName()); + } + } + + return urlToTexture; +} + +image::TextureUsage::Type textureTypeForMaterialProperty(FbxProperty& property, FbxSurfaceMaterial* material) { + using namespace image::TextureUsage; + + // this is a property we know has a texture, we need to match it to a High Fidelity known texture type + // since that information is passed to the baking process + + // grab the hierarchical name for this property and lowercase it for case-insensitive compare + auto propertyName = QString(property.GetHierarchicalName()).toLower(); + + // figure out the type of the property based on what known value string it matches + if ((propertyName.contains("diffuse") && !propertyName.contains("tex_global_diffuse")) + || propertyName.contains("tex_color_map")) { + return ALBEDO_TEXTURE; + } else if (propertyName.contains("transparentcolor") || propertyName.contains("transparencyfactor")) { + return ALBEDO_TEXTURE; + } else if (propertyName.contains("bump")) { + return BUMP_TEXTURE; + } else if (propertyName.contains("normal")) { + return NORMAL_TEXTURE; + } else if ((propertyName.contains("specular") && !propertyName.contains("tex_global_specular")) + || propertyName.contains("reflection")) { + return SPECULAR_TEXTURE; + } else if (propertyName.contains("tex_metallic_map")) { + return METALLIC_TEXTURE; + } else if (propertyName.contains("shininess")) { + return GLOSS_TEXTURE; + } else if (propertyName.contains("tex_roughness_map")) { + return ROUGHNESS_TEXTURE; + } else if (propertyName.contains("emissive")) { + return EMISSIVE_TEXTURE; + } else if (propertyName.contains("ambientcolor")) { + return LIGHTMAP_TEXTURE; + } else if (propertyName.contains("ambientfactor")) { + // we need to check what the ambient factor is, since that tells Interface to process this texture + // either as an occlusion texture or a light map + auto lambertMaterial = FbxCast(material); + + if (lambertMaterial->AmbientFactor == 0) { + return LIGHTMAP_TEXTURE; + } else if (lambertMaterial->AmbientFactor > 0) { + return OCCLUSION_TEXTURE; + } else { + return UNUSED_TEXTURE; + } + + } else if (propertyName.contains("tex_ao_map")) { + return OCCLUSION_TEXTURE; + } + + return UNUSED_TEXTURE; +} + +void FBXBaker::rewriteAndBakeSceneTextures() { + + // enumerate the surface materials to find the textures used in the scene + int numMaterials = _scene->GetMaterialCount(); + for (int i = 0; i < numMaterials; i++) { + FbxSurfaceMaterial* material = _scene->GetMaterial(i); + + if (material) { + // enumerate the properties of this material to see what texture channels it might have + FbxProperty property = material->GetFirstProperty(); + + while (property.IsValid()) { + // first check if this property has connected textures, if not we don't need to bother with it here + if (property.GetSrcObjectCount() > 0) { + + // figure out the type of texture from the material property + auto textureType = textureTypeForMaterialProperty(property, material); + + if (textureType != image::TextureUsage::UNUSED_TEXTURE) { + int numTextures = property.GetSrcObjectCount(); + + for (int j = 0; j < numTextures; j++) { + FbxFileTexture* fileTexture = property.GetSrcObject(j); + + // use QFileInfo to easily split up the existing texture filename into its components + QString fbxTextureFileName { fileTexture->GetFileName() }; + QFileInfo textureFileInfo { fbxTextureFileName.replace("\\", "/") }; + + // make sure this texture points to something and isn't one we've already re-mapped + if (!textureFileInfo.filePath().isEmpty() + && textureFileInfo.suffix() != BAKED_TEXTURE_EXT.mid(1)) { + + // construct the new baked texture file name and file path + // ensuring that the baked texture will have a unique name + // even if there was another texture with the same name at a different path + auto bakedTextureFileName = createBakedTextureFileName(textureFileInfo); + QString bakedTextureFilePath { + _bakedOutputDir + "/" + bakedTextureFileName + }; + _outputFiles.push_back(bakedTextureFilePath); + + qCDebug(model_baking).noquote() << "Re-mapping" << fileTexture->GetFileName() + << "to" << bakedTextureFilePath; + + // figure out the URL to this texture, embedded or external + auto urlToTexture = getTextureURL(textureFileInfo, fileTexture); + + // write the new filename into the FBX scene + fileTexture->SetFileName(bakedTextureFilePath.toUtf8().data()); + + // write the relative filename to be the baked texture file name since it will + // be right beside the FBX + fileTexture->SetRelativeFileName(bakedTextureFileName.toLocal8Bit().constData()); + + if (!_bakingTextures.contains(urlToTexture)) { + // bake this texture asynchronously + bakeTexture(urlToTexture, textureType, _bakedOutputDir); + } + } + } + } + } + + property = material->GetNextProperty(property); + } + } + } +} + +void FBXBaker::bakeTexture(const QUrl& textureURL, image::TextureUsage::Type textureType, const QDir& outputDir) { + // start a bake for this texture and add it to our list to keep track of + QSharedPointer bakingTexture { + new TextureBaker(textureURL, textureType, outputDir), + &TextureBaker::deleteLater + }; + + // make sure we hear when the baking texture is done + connect(bakingTexture.data(), &Baker::finished, this, &FBXBaker::handleBakedTexture); + + // keep a shared pointer to the baking texture + _bakingTextures.insert(textureURL, bakingTexture); + + // start baking the texture on one of our available worker threads + bakingTexture->moveToThread(_textureThreadGetter()); + QMetaObject::invokeMethod(bakingTexture.data(), "bake"); +} + +void FBXBaker::handleBakedTexture() { + TextureBaker* bakedTexture = qobject_cast(sender()); + + // make sure we haven't already run into errors, and that this is a valid texture + if (bakedTexture) { + if (!hasErrors()) { + if (!bakedTexture->hasErrors()) { + if (!_originalOutputDir.isEmpty()) { + // we've been asked to make copies of the originals, so we need to make copies of this if it is a linked texture + + // use the path to the texture being baked to determine if this was an embedded or a linked texture + + // it is embeddded if the texure being baked was inside the original output folder + // since that is where the FBX SDK places the .fbm folder it generates when importing the FBX + + auto originalOutputFolder = QUrl::fromLocalFile(_originalOutputDir); + + if (!originalOutputFolder.isParentOf(bakedTexture->getTextureURL())) { + // for linked textures we want to save a copy of original texture beside the original FBX + + qCDebug(model_baking) << "Saving original texture for" << bakedTexture->getTextureURL(); + + // check if we have a relative path to use for the texture + auto relativeTexturePath = texturePathRelativeToFBX(_fbxURL, bakedTexture->getTextureURL()); + + QFile originalTextureFile { + _originalOutputDir + "/" + relativeTexturePath + bakedTexture->getTextureURL().fileName() + }; + + if (relativeTexturePath.length() > 0) { + // make the folders needed by the relative path + } + + if (originalTextureFile.open(QIODevice::WriteOnly) && originalTextureFile.write(bakedTexture->getOriginalTexture()) != -1) { + qCDebug(model_baking) << "Saved original texture file" << originalTextureFile.fileName() + << "for" << _fbxURL; + } else { + handleError("Could not save original external texture " + originalTextureFile.fileName() + + " for " + _fbxURL.toString()); + return; + } + } + } + + + // now that this texture has been baked and handled, we can remove that TextureBaker from our hash + _bakingTextures.remove(bakedTexture->getTextureURL()); + + checkIfTexturesFinished(); + } else { + // there was an error baking this texture - add it to our list of errors + _errorList.append(bakedTexture->getErrors()); + + // we don't emit finished yet so that the other textures can finish baking first + _pendingErrorEmission = true; + + // now that this texture has been baked, even though it failed, we can remove that TextureBaker from our list + _bakingTextures.remove(bakedTexture->getTextureURL()); + + checkIfTexturesFinished(); + } + } else { + // we have errors to attend to, so we don't do extra processing for this texture + // but we do need to remove that TextureBaker from our list + // and then check if we're done with all textures + _bakingTextures.remove(bakedTexture->getTextureURL()); + + checkIfTexturesFinished(); + } + } +} + +void FBXBaker::exportScene() { + // setup the exporter + FbxExporter* exporter = FbxExporter::Create(_sdkManager.get(), ""); + + // save the relative path to this FBX inside our passed output folder + + auto fileName = _fbxURL.fileName(); + auto baseName = fileName.left(fileName.lastIndexOf('.')); + auto bakedFilename = baseName + BAKED_FBX_EXTENSION; + + _bakedFBXFilePath = _bakedOutputDir + "/" + bakedFilename; + + bool exportStatus = exporter->Initialize(_bakedFBXFilePath.toLocal8Bit().data()); + + if (!exportStatus) { + // failed to initialize exporter, print an error and return + handleError("Failed to export FBX file at " + _fbxURL.toString() + " to " + _bakedFBXFilePath + + "- error: " + exporter->GetStatus().GetErrorString()); + } + + _outputFiles.push_back(_bakedFBXFilePath); + + // export the scene + exporter->Export(_scene); + + qCDebug(model_baking) << "Exported" << _fbxURL << "with re-written paths to" << _bakedFBXFilePath; +} + + +void FBXBaker::removeEmbeddedMediaFolder() { + // now that the bake is complete, remove the embedded media folder produced by the FBX SDK when it imports an FBX + //auto embeddedMediaFolderName = _fbxURL.fileName().replace(".fbx", ".fbm"); + //QDir(_bakedOutputDir + ORIGINAL_OUTPUT_SUBFOLDER + embeddedMediaFolderName).removeRecursively(); +} + +void FBXBaker::checkIfTexturesFinished() { + // check if we're done everything we need to do for this FBX + // and emit our finished signal if we're done + + if (_bakingTextures.isEmpty()) { + // remove the embedded media folder that the FBX SDK produces when reading the original + removeEmbeddedMediaFolder(); + + if (hasErrors()) { + // if we're checking for completion but we have errors + // that means one or more of our texture baking operations failed + + if (_pendingErrorEmission) { + emit finished(); + } + + return; + } else { + qCDebug(model_baking) << "Finished baking, emitting finsihed" << _fbxURL; + + emit finished(); + } + } +} diff --git a/libraries/baking/src/FBXBaker.h b/libraries/baking/src/FBXBaker.h new file mode 100644 index 0000000000..0e52f2e4c3 --- /dev/null +++ b/libraries/baking/src/FBXBaker.h @@ -0,0 +1,105 @@ +// +// FBXBaker.h +// tools/oven/src +// +// Created by Stephen Birarda on 3/30/17. +// Copyright 2017 High Fidelity, Inc. +// +// Distributed under the Apache License, Version 2.0. +// See the accompanying file LICENSE or http://www.apache.org/licenses/LICENSE-2.0.html +// + +#ifndef hifi_FBXBaker_h +#define hifi_FBXBaker_h + +#include +#include +#include +#include + +#include "Baker.h" +#include "TextureBaker.h" + +#include "ModelBakingLoggingCategory.h" + +#include + +namespace fbxsdk { + class FbxManager; + class FbxProperty; + class FbxScene; + class FbxFileTexture; +} + +static const QString BAKED_FBX_EXTENSION = ".baked.fbx"; +using FBXSDKManagerUniquePointer = std::unique_ptr>; + +using TextureBakerThreadGetter = std::function; + +class FBXBaker : public Baker { + Q_OBJECT +public: + FBXBaker(const QUrl& fbxURL, TextureBakerThreadGetter textureThreadGetter, + const QString& bakedOutputDir, const QString& originalOutputDir = ""); + + QUrl getFBXUrl() const { return _fbxURL; } + QString getBakedFBXFilePath() const { return _bakedFBXFilePath; } + std::vector getOutputFiles() const { return _outputFiles; } + +public slots: + // all calls to FBXBaker::bake for FBXBaker instances must be from the same thread + // because the Autodesk SDK will cause a crash if it is called from multiple threads + virtual void bake() override; + +signals: + void sourceCopyReadyToLoad(); + +private slots: + void bakeSourceCopy(); + void handleFBXNetworkReply(); + void handleBakedTexture(); + +private: + void setupOutputFolder(); + + void loadSourceFBX(); + + void importScene(); + void rewriteAndBakeSceneTextures(); + void exportScene(); + void removeEmbeddedMediaFolder(); + + void checkIfTexturesFinished(); + + QString createBakedTextureFileName(const QFileInfo& textureFileInfo); + QUrl getTextureURL(const QFileInfo& textureFileInfo, fbxsdk::FbxFileTexture* fileTexture); + + void bakeTexture(const QUrl& textureURL, image::TextureUsage::Type textureType, const QDir& outputDir); + + QUrl _fbxURL; + + QString _bakedFBXFilePath; + + QString _bakedOutputDir; + + // If set, the original FBX and textures will also be copied here + QString _originalOutputDir; + + QDir _tempDir; + QString _originalFBXFilePath; + + // List of baked output files, includes the FBX and textures + std::vector _outputFiles; + + static FBXSDKManagerUniquePointer _sdkManager; + fbxsdk::FbxScene* _scene { nullptr }; + + QMultiHash> _bakingTextures; + QHash _textureNameMatchCount; + + TextureBakerThreadGetter _textureThreadGetter; + + bool _pendingErrorEmission { false }; +}; + +#endif // hifi_FBXBaker_h diff --git a/tools/oven/src/ModelBakingLoggingCategory.cpp b/libraries/baking/src/ModelBakingLoggingCategory.cpp similarity index 100% rename from tools/oven/src/ModelBakingLoggingCategory.cpp rename to libraries/baking/src/ModelBakingLoggingCategory.cpp diff --git a/tools/oven/src/ModelBakingLoggingCategory.h b/libraries/baking/src/ModelBakingLoggingCategory.h similarity index 100% rename from tools/oven/src/ModelBakingLoggingCategory.h rename to libraries/baking/src/ModelBakingLoggingCategory.h diff --git a/libraries/baking/src/TextureBaker.cpp b/libraries/baking/src/TextureBaker.cpp new file mode 100644 index 0000000000..70df511d2c --- /dev/null +++ b/libraries/baking/src/TextureBaker.cpp @@ -0,0 +1,131 @@ +// +// TextureBaker.cpp +// tools/oven/src +// +// Created by Stephen Birarda on 4/5/17. +// Copyright 2017 High Fidelity, Inc. +// +// Distributed under the Apache License, Version 2.0. +// See the accompanying file LICENSE or http://www.apache.org/licenses/LICENSE-2.0.html +// + +#include +#include +#include +#include + +#include +#include +#include +#include + +#include "ModelBakingLoggingCategory.h" + +#include "TextureBaker.h" + +const QString BAKED_TEXTURE_EXT = ".ktx"; + +TextureBaker::TextureBaker(const QUrl& textureURL, image::TextureUsage::Type textureType, const QDir& outputDirectory) : + _textureURL(textureURL), + _textureType(textureType), + _outputDirectory(outputDirectory) +{ + // figure out the baked texture filename + auto originalFilename = textureURL.fileName(); + _bakedTextureFileName = originalFilename.left(originalFilename.lastIndexOf('.')) + BAKED_TEXTURE_EXT; +} + +void TextureBaker::bake() { + // once our texture is loaded, kick off a the processing + connect(this, &TextureBaker::originalTextureLoaded, this, &TextureBaker::processTexture); + + // first load the texture (either locally or remotely) + loadTexture(); +} + +void TextureBaker::loadTexture() { + // check if the texture is local or first needs to be downloaded + if (_textureURL.isLocalFile()) { + // load up the local file + QFile localTexture { _textureURL.toLocalFile() }; + + if (!localTexture.open(QIODevice::ReadOnly)) { + handleError("Unable to open texture " + _textureURL.toString()); + return; + } + + _originalTexture = localTexture.readAll(); + + emit originalTextureLoaded(); + } else { + // remote file, kick off a download + auto& networkAccessManager = NetworkAccessManager::getInstance(); + + QNetworkRequest networkRequest; + + // setup the request to follow re-directs and always hit the network + networkRequest.setAttribute(QNetworkRequest::FollowRedirectsAttribute, true); + networkRequest.setAttribute(QNetworkRequest::CacheLoadControlAttribute, QNetworkRequest::AlwaysNetwork); + networkRequest.setHeader(QNetworkRequest::UserAgentHeader, HIGH_FIDELITY_USER_AGENT); + + networkRequest.setUrl(_textureURL); + + qCDebug(model_baking) << "Downloading" << _textureURL; + + // kickoff the download, wait for slot to tell us it is done + auto networkReply = networkAccessManager.get(networkRequest); + connect(networkReply, &QNetworkReply::finished, this, &TextureBaker::handleTextureNetworkReply); + } +} + +void TextureBaker::handleTextureNetworkReply() { + auto requestReply = qobject_cast(sender()); + + if (requestReply->error() == QNetworkReply::NoError) { + qCDebug(model_baking) << "Downloaded texture" << _textureURL; + + // store the original texture so it can be passed along for the bake + _originalTexture = requestReply->readAll(); + + emit originalTextureLoaded(); + } else { + // add an error to our list stating that this texture could not be downloaded + handleError("Error downloading " + _textureURL.toString() + " - " + requestReply->errorString()); + } +} + +void TextureBaker::processTexture() { + auto processedTexture = image::processImage(_originalTexture, _textureURL.toString().toStdString(), + ABSOLUTE_MAX_TEXTURE_NUM_PIXELS, _textureType); + + if (!processedTexture) { + handleError("Could not process texture " + _textureURL.toString()); + return; + } + + // the baked textures need to have the source hash added for cache checks in Interface + // so we add that to the processed texture before handling it off to be serialized + auto hashData = QCryptographicHash::hash(_originalTexture, QCryptographicHash::Md5); + std::string hash = hashData.toHex().toStdString(); + processedTexture->setSourceHash(hash); + + auto memKTX = gpu::Texture::serialize(*processedTexture); + + if (!memKTX) { + handleError("Could not serialize " + _textureURL.toString() + " to KTX"); + return; + } + + const char* data = reinterpret_cast(memKTX->_storage->data()); + const size_t length = memKTX->_storage->size(); + + // attempt to write the baked texture to the destination file path + QFile bakedTextureFile { _outputDirectory.absoluteFilePath(_bakedTextureFileName) }; + + if (!bakedTextureFile.open(QIODevice::WriteOnly) || bakedTextureFile.write(data, length) == -1) { + handleError("Could not write baked texture for " + _textureURL.toString()); + } + + qCDebug(model_baking) << "Baked texture" << _textureURL; + emit finished(); +} diff --git a/libraries/baking/src/TextureBaker.h b/libraries/baking/src/TextureBaker.h new file mode 100644 index 0000000000..ee1e968f20 --- /dev/null +++ b/libraries/baking/src/TextureBaker.h @@ -0,0 +1,59 @@ +// +// TextureBaker.h +// tools/oven/src +// +// Created by Stephen Birarda on 4/5/17. +// Copyright 2017 High Fidelity, Inc. +// +// Distributed under the Apache License, Version 2.0. +// See the accompanying file LICENSE or http://www.apache.org/licenses/LICENSE-2.0.html +// + +#ifndef hifi_TextureBaker_h +#define hifi_TextureBaker_h + +#include +#include +#include + +#include + +#include "Baker.h" + +extern const QString BAKED_TEXTURE_EXT; + +class TextureBaker : public Baker { + Q_OBJECT + +public: + TextureBaker(const QUrl& textureURL, image::TextureUsage::Type textureType, const QDir& outputDirectory); + + const QByteArray& getOriginalTexture() const { return _originalTexture; } + + QUrl getTextureURL() const { return _textureURL; } + + QString getDestinationFilePath() const { return _outputDirectory.absoluteFilePath(_bakedTextureFileName); } + QString getBakedTextureFileName() const { return _bakedTextureFileName; } + +public slots: + virtual void bake() override; + +signals: + void originalTextureLoaded(); + +private slots: + void processTexture(); + +private: + void loadTexture(); + void handleTextureNetworkReply(); + + QUrl _textureURL; + QByteArray _originalTexture; + image::TextureUsage::Type _textureType; + + QDir _outputDirectory; + QString _bakedTextureFileName; +}; + +#endif // hifi_TextureBaker_h diff --git a/libraries/fbx/CMakeLists.txt b/libraries/fbx/CMakeLists.txt index d9c073f213..3355ccd06e 100644 --- a/libraries/fbx/CMakeLists.txt +++ b/libraries/fbx/CMakeLists.txt @@ -1,4 +1,7 @@ set(TARGET_NAME fbx) setup_hifi_library() -link_hifi_libraries(shared model networking) -include_hifi_library_headers(gpu) + + + +link_hifi_libraries(shared model networking image) +include_hifi_library_headers(gpu image) diff --git a/libraries/fbx/src/FBXReader.cpp b/libraries/fbx/src/FBXReader.cpp index 6d4f586c52..b1d2cf17d4 100644 --- a/libraries/fbx/src/FBXReader.cpp +++ b/libraries/fbx/src/FBXReader.cpp @@ -846,12 +846,14 @@ FBXGeometry* FBXReader::extractFBXGeometry(const QVariantHash& mapping, const QS QByteArray filename = subobject.properties.at(0).toByteArray(); QByteArray filepath = filename.replace('\\', '/'); filename = fileOnUrl(filepath, url); + qDebug() << "Filename" << filepath << filename; _textureFilepaths.insert(getID(object.properties), filepath); _textureFilenames.insert(getID(object.properties), filename); } else if (subobject.name == "TextureName" && subobject.properties.length() >= TEXTURE_NAME_MIN_SIZE) { // trim the name from the timestamp QString name = QString(subobject.properties.at(0).toByteArray()); name = name.left(name.indexOf('[')); + qDebug() << "Filename" << name; _textureNames.insert(getID(object.properties), name); } else if (subobject.name == "Texture_Alpha_Source" && subobject.properties.length() >= TEXTURE_ALPHA_SOURCE_MIN_SIZE) { tex.assign(tex.alphaSource, subobject.properties.at(0).value()); @@ -1840,5 +1842,7 @@ FBXGeometry* readFBX(QIODevice* device, const QVariantHash& mapping, const QStri reader._loadLightmaps = loadLightmaps; reader._lightmapLevel = lightmapLevel; + qDebug() << "Reading FBX: " << url; + return reader.extractFBXGeometry(mapping, url); } diff --git a/libraries/model-networking/src/model-networking/ModelCache.cpp b/libraries/model-networking/src/model-networking/ModelCache.cpp index 704455c981..468d22ce9e 100644 --- a/libraries/model-networking/src/model-networking/ModelCache.cpp +++ b/libraries/model-networking/src/model-networking/ModelCache.cpp @@ -77,6 +77,8 @@ void GeometryMappingResource::downloadFinished(const QByteArray& data) { texdir += '/'; } _textureBaseUrl = resolveTextureBaseUrl(url, _url.resolved(texdir)); + } else { + _textureBaseUrl = _effectiveBaseURL; } auto animGraphVariant = mapping.value("animGraphUrl"); @@ -239,7 +241,10 @@ private: }; void GeometryDefinitionResource::downloadFinished(const QByteArray& data) { - QThreadPool::globalInstance()->start(new GeometryReader(_self, _url, _mapping, data, _combineParts)); + qDebug() << "Processing geometry: " << _effectiveBaseURL; + _url = _effectiveBaseURL; + _textureBaseUrl = _effectiveBaseURL; + QThreadPool::globalInstance()->start(new GeometryReader(_self, _effectiveBaseURL, _mapping, data, _combineParts)); } void GeometryDefinitionResource::setGeometryDefinition(FBXGeometry::Pointer fbxGeometry) { @@ -250,6 +255,7 @@ void GeometryDefinitionResource::setGeometryDefinition(FBXGeometry::Pointer fbxG QHash materialIDAtlas; for (const FBXMaterial& material : _fbxGeometry->materials) { materialIDAtlas[material.materialID] = _materials.size(); + qDebug() << "setGeometryDefinition() " << _textureBaseUrl; _materials.push_back(std::make_shared(material, _textureBaseUrl)); } @@ -342,6 +348,7 @@ Geometry::Geometry(const Geometry& geometry) { _materials.reserve(geometry._materials.size()); for (const auto& material : geometry._materials) { + qDebug() << "Geometry() no base url..."; _materials.push_back(std::make_shared(*material)); } @@ -427,6 +434,7 @@ void GeometryResource::deleter() { void GeometryResource::setTextures() { if (_fbxGeometry) { for (const FBXMaterial& material : _fbxGeometry->materials) { + qDebug() << "setTextures() " << _textureBaseUrl; _materials.push_back(std::make_shared(material, _textureBaseUrl)); } } @@ -528,6 +536,7 @@ model::TextureMapPointer NetworkMaterial::fetchTextureMap(const QUrl& url, image NetworkMaterial::NetworkMaterial(const FBXMaterial& material, const QUrl& textureBaseUrl) : model::Material(*material._material) { + qDebug() << "Created network material with base url: " << textureBaseUrl; _textures = Textures(MapChannel::NUM_MAP_CHANNELS); if (!material.albedoTexture.filename.isEmpty()) { auto map = fetchTextureMap(textureBaseUrl, material.albedoTexture, image::TextureUsage::ALBEDO_TEXTURE, MapChannel::ALBEDO_MAP); diff --git a/libraries/model-networking/src/model-networking/ModelCache.h b/libraries/model-networking/src/model-networking/ModelCache.h index 6a1cc4c466..a122e03eb9 100644 --- a/libraries/model-networking/src/model-networking/ModelCache.h +++ b/libraries/model-networking/src/model-networking/ModelCache.h @@ -131,6 +131,7 @@ private: Geometry::Pointer& _geometryRef; }; + /// Stores cached model geometries. class ModelCache : public ResourceCache, public Dependency { Q_OBJECT diff --git a/libraries/model-networking/src/model-networking/TextureCache.cpp b/libraries/model-networking/src/model-networking/TextureCache.cpp index 85bde4c2f1..5b43a4090f 100644 --- a/libraries/model-networking/src/model-networking/TextureCache.cpp +++ b/libraries/model-networking/src/model-networking/TextureCache.cpp @@ -21,6 +21,7 @@ #include #include #include +#include #if DEBUG_DUMP_TEXTURE_LOADS #include @@ -189,8 +190,15 @@ NetworkTexturePointer TextureCache::getTexture(const QUrl& url, image::TextureUs if (url.scheme() == RESOURCE_SCHEME) { return getResourceTexture(url); } + auto modifiedUrl = url; + if (type == image::TextureUsage::CUBE_TEXTURE) { + QUrlQuery query { url.query() }; + query.addQueryItem("skybox", ""); + qDebug() << "Updating cubemap texture query from" << url.query() << "to" << query.toString(); + modifiedUrl.setQuery(query.toString()); + } TextureExtra extra = { type, content, maxNumPixels }; - return ResourceCache::getResource(url, QUrl(), &extra).staticCast(); + return ResourceCache::getResource(modifiedUrl, QUrl(), &extra).staticCast(); } gpu::TexturePointer TextureCache::getTextureByHash(const std::string& hash) { diff --git a/libraries/networking/src/AssetResourceRequest.cpp b/libraries/networking/src/AssetResourceRequest.cpp index e228582ca9..01fe971125 100644 --- a/libraries/networking/src/AssetResourceRequest.cpp +++ b/libraries/networking/src/AssetResourceRequest.cpp @@ -63,7 +63,7 @@ void AssetResourceRequest::doSend() { // This is an ATP path, we'll need to figure out what the mapping is. // This may incur a roundtrip to the asset-server, or it may return immediately from the cache in AssetClient. - auto path = _url.path(); + auto path = _url.path() + (_url.hasQuery() ? "?" + _url.query() : ""); requestMappingForPath(path); } } @@ -93,10 +93,8 @@ void AssetResourceRequest::requestMappingForPath(const AssetPath& path) { // if we got a redirected path we need to store that with the resource request as relative path URL if (request->wasRedirected()) { + qDebug() << "Request was redirected"; _relativePathURL = ATP_SCHEME + request->getRedirectedPath(); - - // truncate the filename for the re-directed asset so we actually have a path - _relativePathURL = _relativePathURL.adjusted(QUrl::RemoveFilename); } break; diff --git a/libraries/networking/src/MappingRequest.cpp b/libraries/networking/src/MappingRequest.cpp index bd7c05bdf7..5fb9c32a0b 100644 --- a/libraries/networking/src/MappingRequest.cpp +++ b/libraries/networking/src/MappingRequest.cpp @@ -96,6 +96,9 @@ void GetMappingRequest::doStart() { // if it did grab that re-directed path if (_wasRedirected) { _redirectedPath = message->readString(); + qDebug() << "Got redirected from " << _path << " to " << _redirectedPath; + } else { + qDebug() << "Not redirected: " << _path; } } diff --git a/libraries/networking/src/ResourceCache.cpp b/libraries/networking/src/ResourceCache.cpp index fbdfa4b87a..b7ab1269f9 100644 --- a/libraries/networking/src/ResourceCache.cpp +++ b/libraries/networking/src/ResourceCache.cpp @@ -731,6 +731,13 @@ void Resource::handleReplyFinished() { if (result == ResourceRequest::Success) { auto extraInfo = _url == _activeUrl ? "" : QString(", %1").arg(_activeUrl.toDisplayString()); qCDebug(networking).noquote() << QString("Request finished for %1%2").arg(_url.toDisplayString(), extraInfo); + + auto relativePathURL = _request->getRelativePathUrl(); + qDebug() << "Relative path is: " << relativePathURL; + if (!relativePathURL.isEmpty()) { + qDebug() << "setting effective path"; + _effectiveBaseURL = relativePathURL; + } auto data = _request->getData(); emit loaded(data); diff --git a/libraries/networking/src/ResourceCache.h b/libraries/networking/src/ResourceCache.h index f94e1e26d2..40df6418a5 100644 --- a/libraries/networking/src/ResourceCache.h +++ b/libraries/networking/src/ResourceCache.h @@ -452,6 +452,7 @@ protected: bool handleFailedRequest(ResourceRequest::Result result); QUrl _url; + QUrl _effectiveBaseURL{ _url }; QUrl _activeUrl; ByteRange _requestByteRange; diff --git a/libraries/networking/src/ResourceRequest.h b/libraries/networking/src/ResourceRequest.h index bae0882e97..0a4dc12293 100644 --- a/libraries/networking/src/ResourceRequest.h +++ b/libraries/networking/src/ResourceRequest.h @@ -66,6 +66,7 @@ public: Result getResult() const { return _result; } QString getResultString() const; QUrl getUrl() const { return _url; } + QUrl getRelativePathUrl() const { return _relativePathURL; } bool loadedFromCache() const { return _loadedFromCache; } bool getRangeRequestSuccessful() const { return _rangeRequestSuccessful; } bool getTotalSizeOfResource() const { return _totalSizeOfResource; } diff --git a/libraries/networking/src/udt/PacketHeaders.h b/libraries/networking/src/udt/PacketHeaders.h index ea831bd415..6b57c8a3cb 100644 --- a/libraries/networking/src/udt/PacketHeaders.h +++ b/libraries/networking/src/udt/PacketHeaders.h @@ -139,11 +139,16 @@ public: const static QSet getNonVerifiedPackets() { const static QSet NON_VERIFIED_PACKETS = QSet() - << PacketTypeEnum::Value::NodeJsonStats << PacketTypeEnum::Value::EntityQuery - << PacketTypeEnum::Value::OctreeDataNack << PacketTypeEnum::Value::EntityEditNack - << PacketTypeEnum::Value::DomainListRequest << PacketTypeEnum::Value::StopNode - << PacketTypeEnum::Value::DomainDisconnectRequest << PacketTypeEnum::Value::UsernameFromIDRequest - << PacketTypeEnum::Value::NodeKickRequest << PacketTypeEnum::Value::NodeMuteRequest; + << PacketTypeEnum::Value::NodeJsonStats + << PacketTypeEnum::Value::EntityQuery + << PacketTypeEnum::Value::OctreeDataNack + << PacketTypeEnum::Value::EntityEditNack + << PacketTypeEnum::Value::DomainListRequest + << PacketTypeEnum::Value::StopNode + << PacketTypeEnum::Value::DomainDisconnectRequest + << PacketTypeEnum::Value::UsernameFromIDRequest + << PacketTypeEnum::Value::NodeKickRequest + << PacketTypeEnum::Value::NodeMuteRequest; return NON_VERIFIED_PACKETS; } diff --git a/tools/oven/CMakeLists.txt b/tools/oven/CMakeLists.txt index f4fca3304c..0d692b5465 100644 --- a/tools/oven/CMakeLists.txt +++ b/tools/oven/CMakeLists.txt @@ -2,7 +2,7 @@ set(TARGET_NAME oven) setup_hifi_project(Widgets Gui Concurrent) -link_hifi_libraries(networking shared image gpu ktx) +link_hifi_libraries(networking shared image gpu ktx fbx baking) setup_memory_debugger() From cefd16ad9585b1c260b2e027cfad7ea36686f721 Mon Sep 17 00:00:00 2001 From: Atlante45 Date: Wed, 9 Aug 2017 17:12:47 -0700 Subject: [PATCH 011/129] Fix asset server debug messages logging category --- assignment-client/src/assets/AssetServer.cpp | 98 +++++++++---------- .../src/assets/AssetServerLogging.cpp | 14 +++ .../src/assets/AssetServerLogging.h | 19 ++++ 3 files changed, 82 insertions(+), 49 deletions(-) create mode 100644 assignment-client/src/assets/AssetServerLogging.cpp create mode 100644 assignment-client/src/assets/AssetServerLogging.h diff --git a/assignment-client/src/assets/AssetServer.cpp b/assignment-client/src/assets/AssetServer.cpp index 6e3195f570..e28ad9ead8 100644 --- a/assignment-client/src/assets/AssetServer.cpp +++ b/assignment-client/src/assets/AssetServer.cpp @@ -13,6 +13,7 @@ #include "AssetServer.h" #include +#include #include #include @@ -27,17 +28,16 @@ #include #include +#include +#include +#include #include #include -#include "NetworkLogging.h" -#include "NodeType.h" +#include "AssetServerLogging.h" #include "SendAssetTask.h" #include "UploadAssetTask.h" -#include -#include -#include static const uint8_t MIN_CORES_FOR_MULTICORE = 4; static const uint8_t CPU_AFFINITY_COUNT_HIGH = 2; @@ -196,7 +196,7 @@ void updateConsumedCores() { if (isInterfaceRunning) { coreCount = coreCount > MIN_CORES_FOR_MULTICORE ? CPU_AFFINITY_COUNT_HIGH : CPU_AFFINITY_COUNT_LOW; } - qDebug() << "Setting max consumed cores to " << coreCount; + qCDebug(asset_server) << "Setting max consumed cores to " << coreCount; setMaxCores(coreCount); } @@ -236,7 +236,7 @@ AssetServer::AssetServer(ReceivedMessage& message) : void AssetServer::run() { - qDebug() << "Waiting for connection to domain to request settings from domain-server."; + qCDebug(asset_server) << "Waiting for connection to domain to request settings from domain-server."; // wait until we have the domain-server settings, otherwise we bail DomainHandler& domainHandler = DependencyManager::get()->getDomainHandler(); @@ -257,7 +257,7 @@ void AssetServer::completeSetup() { static const QString ASSET_SERVER_SETTINGS_KEY = "asset_server"; if (!settingsObject.contains(ASSET_SERVER_SETTINGS_KEY)) { - qCritical() << "Received settings from the domain-server with no asset-server section. Stopping assignment."; + qCCritical(asset_server) << "Received settings from the domain-server with no asset-server section. Stopping assignment."; setFinished(true); return; } @@ -272,7 +272,7 @@ void AssetServer::completeSetup() { const int BITS_PER_MEGABITS = 1000 * 1000; int maxBandwidth = maxBandwidthFloat * BITS_PER_MEGABITS; nodeList->setConnectionMaxBandwidth(maxBandwidth); - qInfo() << "Set maximum bandwith per connection to" << maxBandwidthFloat << "Mb/s." + qCInfo(asset_server) << "Set maximum bandwith per connection to" << maxBandwidthFloat << "Mb/s." " (" << maxBandwidth << "bits/s)"; } @@ -281,7 +281,7 @@ void AssetServer::completeSetup() { auto assetsJSONValue = assetServerObject[ASSETS_PATH_OPTION]; if (!assetsJSONValue.isString()) { - qCritical() << "Received an assets path from the domain-server that could not be parsed. Stopping assignment."; + qCCritical(asset_server) << "Received an assets path from the domain-server that could not be parsed. Stopping assignment."; setFinished(true); return; } @@ -298,19 +298,19 @@ void AssetServer::completeSetup() { _resourcesDirectory = QDir(absoluteFilePath); - qDebug() << "Creating resources directory"; + qCDebug(asset_server) << "Creating resources directory"; _resourcesDirectory.mkpath("."); _filesDirectory = _resourcesDirectory; if (!_resourcesDirectory.mkpath(ASSET_FILES_SUBDIR) || !_filesDirectory.cd(ASSET_FILES_SUBDIR)) { - qCritical() << "Unable to create file directory for asset-server files. Stopping assignment."; + qCCritical(asset_server) << "Unable to create file directory for asset-server files. Stopping assignment."; setFinished(true); return; } // load whatever mappings we currently have from the local file if (loadMappingsFromFile()) { - qInfo() << "Serving files from: " << _filesDirectory.path(); + qCInfo(asset_server) << "Serving files from: " << _filesDirectory.path(); // Check the asset directory to output some information about what we have auto files = _filesDirectory.entryList(QDir::Files); @@ -318,7 +318,7 @@ void AssetServer::completeSetup() { QRegExp hashFileRegex { ASSET_HASH_REGEX_STRING }; auto hashedFiles = files.filter(hashFileRegex); - qInfo() << "There are" << hashedFiles.size() << "asset files in the asset directory."; + qCInfo(asset_server) << "There are" << hashedFiles.size() << "asset files in the asset directory."; if (_fileMappings.count() > 0) { cleanupUnmappedFiles(); @@ -328,7 +328,7 @@ void AssetServer::completeSetup() { bakeAssets(); } else { - qCritical() << "Asset Server assignment will not continue because mapping file could not be loaded."; + qCCritical(asset_server) << "Asset Server assignment will not continue because mapping file could not be loaded."; setFinished(true); } } @@ -341,7 +341,7 @@ void AssetServer::cleanupUnmappedFiles() { // grab the currently mapped hashes auto mappedHashes = _fileMappings.values(); - qInfo() << "Performing unmapped asset cleanup."; + qCInfo(asset_server) << "Performing unmapped asset cleanup."; for (const auto& fileInfo : files) { if (hashFileRegex.exactMatch(fileInfo.fileName())) { @@ -350,9 +350,9 @@ void AssetServer::cleanupUnmappedFiles() { QFile removeableFile { fileInfo.absoluteFilePath() }; if (removeableFile.remove()) { - qDebug() << "\tDeleted" << fileInfo.fileName() << "from asset files directory since it is unmapped."; + qCDebug(asset_server) << "\tDeleted" << fileInfo.fileName() << "from asset files directory since it is unmapped."; } else { - qDebug() << "\tAttempt to delete unmapped file" << fileInfo.fileName() << "failed"; + qCDebug(asset_server) << "\tAttempt to delete unmapped file" << fileInfo.fileName() << "failed"; } } } @@ -538,7 +538,7 @@ void AssetServer::handleAssetGetInfo(QSharedPointer message, Sh MessageID messageID; if (message->getSize() < qint64(SHA256_HASH_LENGTH + sizeof(messageID))) { - qDebug() << "ERROR bad file request"; + qCDebug(asset_server) << "ERROR bad file request"; return; } @@ -557,11 +557,11 @@ void AssetServer::handleAssetGetInfo(QSharedPointer message, Sh QFileInfo fileInfo { _filesDirectory.filePath(fileName) }; if (fileInfo.exists() && fileInfo.isReadable()) { - qDebug() << "Opening file: " << fileInfo.filePath(); + qCDebug(asset_server) << "Opening file: " << fileInfo.filePath(); replyPacket->writePrimitive(AssetServerError::NoError); replyPacket->writePrimitive(fileInfo.size()); } else { - qDebug() << "Asset not found: " << QString(hexHash); + qCDebug(asset_server) << "Asset not found: " << QString(hexHash); replyPacket->writePrimitive(AssetServerError::AssetNotFound); } @@ -574,7 +574,7 @@ void AssetServer::handleAssetGet(QSharedPointer message, Shared auto minSize = qint64(sizeof(MessageID) + SHA256_HASH_LENGTH + sizeof(DataOffset) + sizeof(DataOffset)); if (message->getSize() < minSize) { - qDebug() << "ERROR bad file request"; + qCDebug(asset_server) << "ERROR bad file request"; return; } @@ -586,7 +586,7 @@ void AssetServer::handleAssetGet(QSharedPointer message, Shared void AssetServer::handleAssetUpload(QSharedPointer message, SharedNodePointer senderNode) { if (senderNode->getCanWriteToAssetServer()) { - qDebug() << "Starting an UploadAssetTask for upload from" << uuidStringWithoutCurlyBraces(senderNode->getUUID()); + qCDebug(asset_server) << "Starting an UploadAssetTask for upload from" << uuidStringWithoutCurlyBraces(senderNode->getUUID()); auto task = new UploadAssetTask(message, senderNode, _filesDirectory); _transferTaskPool.start(task); @@ -701,12 +701,12 @@ bool AssetServer::loadMappingsFromFile() { bool shouldDrop = false; if (!isValidFilePath(it.key())) { - qWarning() << "Will not keep mapping for" << it.key() << "since it is not a valid path."; + qCWarning(asset_server) << "Will not keep mapping for" << it.key() << "since it is not a valid path."; shouldDrop = true; } if (!isValidHash(it.value().toString())) { - qWarning() << "Will not keep mapping for" << it.key() << "since it does not have a valid hash."; + qCWarning(asset_server) << "Will not keep mapping for" << it.key() << "since it does not have a valid hash."; shouldDrop = true; } @@ -717,15 +717,15 @@ bool AssetServer::loadMappingsFromFile() { } } - qInfo() << "Loaded" << _fileMappings.count() << "mappings from map file at" << mapFilePath; + qCInfo(asset_server) << "Loaded" << _fileMappings.count() << "mappings from map file at" << mapFilePath; return true; } } - qCritical() << "Failed to read mapping file at" << mapFilePath; + qCCritical(asset_server) << "Failed to read mapping file at" << mapFilePath; return false; } else { - qInfo() << "No existing mappings loaded from file since no file was found at" << mapFilePath; + qCInfo(asset_server) << "No existing mappings loaded from file since no file was found at" << mapFilePath; } return true; @@ -740,13 +740,13 @@ bool AssetServer::writeMappingsToFile() { QJsonDocument jsonDocument { jsonObject }; if (mapFile.write(jsonDocument.toJson()) != -1) { - qDebug() << "Wrote JSON mappings to file at" << mapFilePath; + qCDebug(asset_server) << "Wrote JSON mappings to file at" << mapFilePath; return true; } else { - qWarning() << "Failed to write JSON mappings to file at" << mapFilePath; + qCWarning(asset_server) << "Failed to write JSON mappings to file at" << mapFilePath; } } else { - qWarning() << "Failed to open map file at" << mapFilePath; + qCWarning(asset_server) << "Failed to open map file at" << mapFilePath; } return false; @@ -756,12 +756,12 @@ bool AssetServer::setMapping(AssetPath path, AssetHash hash) { path = path.trimmed(); if (!isValidFilePath(path)) { - qWarning() << "Cannot set a mapping for invalid path:" << path << "=>" << hash; + qCWarning(asset_server) << "Cannot set a mapping for invalid path:" << path << "=>" << hash; return false; } if (!isValidHash(hash)) { - qWarning() << "Cannot set a mapping for invalid hash" << path << "=>" << hash; + qCWarning(asset_server) << "Cannot set a mapping for invalid hash" << path << "=>" << hash; return false; } @@ -774,7 +774,7 @@ bool AssetServer::setMapping(AssetPath path, AssetHash hash) { // attempt to write to file if (writeMappingsToFile()) { // persistence succeeded, we are good to go - qDebug() << "Set mapping:" << path << "=>" << hash; + qCDebug(asset_server) << "Set mapping:" << path << "=>" << hash; maybeBake(path, hash); return true; } else { @@ -785,7 +785,7 @@ bool AssetServer::setMapping(AssetPath path, AssetHash hash) { _fileMappings[path] = oldMapping; } - qWarning() << "Failed to persist mapping:" << path << "=>" << hash; + qCWarning(asset_server) << "Failed to persist mapping:" << path << "=>" << hash; return false; } @@ -825,9 +825,9 @@ bool AssetServer::deleteMappings(AssetPathList& paths) { auto sizeNow = _fileMappings.size(); if (sizeBefore != sizeNow) { - qDebug() << "Deleted" << sizeBefore - sizeNow << "mappings in folder: " << path; + qCDebug(asset_server) << "Deleted" << sizeBefore - sizeNow << "mappings in folder: " << path; } else { - qDebug() << "Did not find any mappings to delete in folder:" << path; + qCDebug(asset_server) << "Did not find any mappings to delete in folder:" << path; } } else { @@ -836,9 +836,9 @@ bool AssetServer::deleteMappings(AssetPathList& paths) { // add this hash to the list we need to check for asset removal from server hashesToCheckForDeletion << oldMapping.toString(); - qDebug() << "Deleted a mapping:" << path << "=>" << oldMapping.toString(); + qCDebug(asset_server) << "Deleted a mapping:" << path << "=>" << oldMapping.toString(); } else { - qDebug() << "Unable to delete a mapping that was not found:" << path; + qCDebug(asset_server) << "Unable to delete a mapping that was not found:" << path; } } } @@ -864,15 +864,15 @@ bool AssetServer::deleteMappings(AssetPathList& paths) { QFile removeableFile { _filesDirectory.absoluteFilePath(hash) }; if (removeableFile.remove()) { - qDebug() << "\tDeleted" << hash << "from asset files directory since it is now unmapped."; + qCDebug(asset_server) << "\tDeleted" << hash << "from asset files directory since it is now unmapped."; } else { - qDebug() << "\tAttempt to delete unmapped file" << hash << "failed"; + qCDebug(asset_server) << "\tAttempt to delete unmapped file" << hash << "failed"; } } return true; } else { - qWarning() << "Failed to persist deleted mappings, rolling back"; + qCWarning(asset_server) << "Failed to persist deleted mappings, rolling back"; // we didn't delete the previous mapping, put it back in our in-memory representation _fileMappings = oldMappings; @@ -886,7 +886,7 @@ bool AssetServer::renameMapping(AssetPath oldPath, AssetPath newPath) { newPath = newPath.trimmed(); if (!isValidFilePath(oldPath) || !isValidFilePath(newPath)) { - qWarning() << "Cannot perform rename with invalid paths - both should have leading forward and no ending slashes:" + qCWarning(asset_server) << "Cannot perform rename with invalid paths - both should have leading forward and no ending slashes:" << oldPath << "=>" << newPath; return false; @@ -896,7 +896,7 @@ bool AssetServer::renameMapping(AssetPath oldPath, AssetPath newPath) { if (pathIsFolder(oldPath)) { if (!pathIsFolder(newPath)) { // we were asked to rename a path to a folder to a path that isn't a folder, this is a fail - qWarning() << "Cannot rename mapping from folder path" << oldPath << "to file path" << newPath; + qCWarning(asset_server) << "Cannot rename mapping from folder path" << oldPath << "to file path" << newPath; return false; } @@ -922,21 +922,21 @@ bool AssetServer::renameMapping(AssetPath oldPath, AssetPath newPath) { if (writeMappingsToFile()) { // persisted the changed mappings, return success - qDebug() << "Renamed folder mapping:" << oldPath << "=>" << newPath; + qCDebug(asset_server) << "Renamed folder mapping:" << oldPath << "=>" << newPath; return true; } else { // couldn't persist the renamed paths, rollback and return failure _fileMappings = oldMappings; - qWarning() << "Failed to persist renamed folder mapping:" << oldPath << "=>" << newPath; + qCWarning(asset_server) << "Failed to persist renamed folder mapping:" << oldPath << "=>" << newPath; return false; } } else { if (pathIsFolder(newPath)) { // we were asked to rename a path to a file to a path that is a folder, this is a fail - qWarning() << "Cannot rename mapping from file path" << oldPath << "to folder path" << newPath; + qCWarning(asset_server) << "Cannot rename mapping from file path" << oldPath << "to folder path" << newPath; return false; } @@ -952,7 +952,7 @@ bool AssetServer::renameMapping(AssetPath oldPath, AssetPath newPath) { if (writeMappingsToFile()) { // persisted the renamed mapping, return success - qDebug() << "Renamed mapping:" << oldPath << "=>" << newPath; + qCDebug(asset_server) << "Renamed mapping:" << oldPath << "=>" << newPath; return true; } else { @@ -967,7 +967,7 @@ bool AssetServer::renameMapping(AssetPath oldPath, AssetPath newPath) { _fileMappings.remove(newPath); } - qDebug() << "Failed to persist renamed mapping:" << oldPath << "=>" << newPath; + qCDebug(asset_server) << "Failed to persist renamed mapping:" << oldPath << "=>" << newPath; return false; } diff --git a/assignment-client/src/assets/AssetServerLogging.cpp b/assignment-client/src/assets/AssetServerLogging.cpp new file mode 100644 index 0000000000..39a02107ea --- /dev/null +++ b/assignment-client/src/assets/AssetServerLogging.cpp @@ -0,0 +1,14 @@ +// +// AssetServerLogging.cpp +// assignment-client/src/assets +// +// Created by Clement Brisset on 8/9/17. +// Copyright 2017 High Fidelity, Inc. +// +// Distributed under the Apache License, Version 2.0. +// See the accompanying file LICENSE or http://www.apache.org/licenses/LICENSE-2.0.html +// + +#include "AssetServerLogging.h" + +Q_LOGGING_CATEGORY(asset_server, "hifi.asset-server") diff --git a/assignment-client/src/assets/AssetServerLogging.h b/assignment-client/src/assets/AssetServerLogging.h new file mode 100644 index 0000000000..986e01ecc5 --- /dev/null +++ b/assignment-client/src/assets/AssetServerLogging.h @@ -0,0 +1,19 @@ +// +// AssetServerLogging.h +// assignment-client/src/assets +// +// Created by Clement Brisset on 8/9/17. +// Copyright 2017 High Fidelity, Inc. +// +// Distributed under the Apache License, Version 2.0. +// See the accompanying file LICENSE or http://www.apache.org/licenses/LICENSE-2.0.html +// + +#ifndef hifi_AssetServerLogging_h +#define hifi_AssetServerLogging_h + +#include + +Q_DECLARE_LOGGING_CATEGORY(asset_server) + +#endif // hifi_AssetServerLogging_h From fac6015bb0e391850d3d68b28d40b2daf3e6921d Mon Sep 17 00:00:00 2001 From: Atlante45 Date: Wed, 9 Aug 2017 18:25:58 -0700 Subject: [PATCH 012/129] Asset baking status first pass --- assignment-client/src/assets/AssetServer.cpp | 11 ++++++ assignment-client/src/assets/AssetServer.h | 3 ++ assignment-client/src/assets/AutoBaker.cpp | 38 +++++++++++++++++++ assignment-client/src/assets/AutoBaker.h | 39 ++++++++++++++++++++ 4 files changed, 91 insertions(+) create mode 100644 assignment-client/src/assets/AutoBaker.cpp create mode 100644 assignment-client/src/assets/AutoBaker.h diff --git a/assignment-client/src/assets/AssetServer.cpp b/assignment-client/src/assets/AssetServer.cpp index e28ad9ead8..1f74a401e3 100644 --- a/assignment-client/src/assets/AssetServer.cpp +++ b/assignment-client/src/assets/AssetServer.cpp @@ -331,6 +331,17 @@ void AssetServer::completeSetup() { qCCritical(asset_server) << "Asset Server assignment will not continue because mapping file could not be loaded."; setFinished(true); } + + QRegExp hashFileRegex { "^[a-f0-9]{" + QString::number(SHA256_HASH_HEX_LENGTH) + "}" }; + auto files = _filesDirectory.entryInfoList(QDir::Files); + auto mappedHashes = _fileMappings.values(); + for (const auto& fileInfo : files) { + AssetHash hash = fileInfo.fileName(); + bool isAsset = hashFileRegex.exactMatch(hash); + if (isAsset && _baker.assetNeedsBaking(hash)) { + _baker.addPendingBake(hash); + } + } } void AssetServer::cleanupUnmappedFiles() { diff --git a/assignment-client/src/assets/AssetServer.h b/assignment-client/src/assets/AssetServer.h index 2dfc5f6c35..2ce223760f 100644 --- a/assignment-client/src/assets/AssetServer.h +++ b/assignment-client/src/assets/AssetServer.h @@ -19,6 +19,7 @@ #include #include "AssetUtils.h" +#include "AutoBaker.h" #include "ReceivedMessage.h" class BakeAssetTask : public QObject, public QRunnable { @@ -105,6 +106,8 @@ private: QHash> _pendingBakes; QThreadPool _bakingTaskPool; + + AutoBaker _baker; }; #endif diff --git a/assignment-client/src/assets/AutoBaker.cpp b/assignment-client/src/assets/AutoBaker.cpp new file mode 100644 index 0000000000..c56e00d025 --- /dev/null +++ b/assignment-client/src/assets/AutoBaker.cpp @@ -0,0 +1,38 @@ +// +// AutoBaker.cpp +// assignment-client/src/assets +// +// Created by Clement Brisset on 8/9/17 +// Copyright 2015 High Fidelity, Inc. +// +// Distributed under the Apache License, Version 2.0. +// See the accompanying file LICENSE or http://www.apache.org/licenses/LICENSE-2.0.html +// + +#include "AutoBaker.h" + +void AutoBaker::addPendingBake(AssetHash hash) { + _pendingBakes.push_back(hash); +} + +bool AutoBaker::assetNeedsBaking(AssetHash hash) { + return true; +} + +AutoBaker::Status AutoBaker::getAssetStatus(AssetHash hash) { + auto pendingIt = std::find(_pendingBakes.cbegin(), _pendingBakes.cend(), hash); + if (pendingIt != _pendingBakes.cend()) { + return Pending; + } + + auto bakingIt = std::find(_currentlyBaking.cbegin(), _currentlyBaking.cend(), hash); + if (bakingIt != _currentlyBaking.cend()) { + return Baking; + } + + if (assetNeedsBaking(hash)) { + return NotBaked; + } else { + return Baked; + } +} diff --git a/assignment-client/src/assets/AutoBaker.h b/assignment-client/src/assets/AutoBaker.h new file mode 100644 index 0000000000..7b92118aee --- /dev/null +++ b/assignment-client/src/assets/AutoBaker.h @@ -0,0 +1,39 @@ +// +// AutoBaker.h +// assignment-client/src/assets +// +// Created by Clement Brisset on 8/9/17 +// Copyright 2015 High Fidelity, Inc. +// +// Distributed under the Apache License, Version 2.0. +// See the accompanying file LICENSE or http://www.apache.org/licenses/LICENSE-2.0.html +// + +#ifndef hifi_AutoBaker_h +#define hifi_AutoBaker_h + +#include + +#include "AssetUtils.h" + +class AutoBaker { +public: + enum Status { + NotBaked, + Pending, + Baking, + Baked + }; + + void addPendingBake(AssetHash hash); + + bool assetNeedsBaking(AssetHash hash); + + Status getAssetStatus(AssetHash hash); + +private: + std::vector _pendingBakes; + std::vector _currentlyBaking; +}; + +#endif /* hifi_AutoBaker_h */ From fd3156b57c2a6df23cbe3ce2b55dd7e0724b84d8 Mon Sep 17 00:00:00 2001 From: Atlante45 Date: Thu, 10 Aug 2017 18:00:16 -0700 Subject: [PATCH 013/129] Move BakingStatus --- assignment-client/src/assets/AssetServer.cpp | 1 + assignment-client/src/assets/AutoBaker.h | 9 +------ libraries/networking/src/AssetUtils.cpp | 13 +++++++++ libraries/networking/src/AssetUtils.h | 9 +++++++ libraries/networking/src/MappingRequest.cpp | 2 ++ libraries/shared/src/shared/Algorithms.h | 28 ++++++++++++++++++++ 6 files changed, 54 insertions(+), 8 deletions(-) create mode 100644 libraries/shared/src/shared/Algorithms.h diff --git a/assignment-client/src/assets/AssetServer.cpp b/assignment-client/src/assets/AssetServer.cpp index 1f74a401e3..ea0010cb26 100644 --- a/assignment-client/src/assets/AssetServer.cpp +++ b/assignment-client/src/assets/AssetServer.cpp @@ -489,6 +489,7 @@ void AssetServer::handleGetAllMappingOperation(ReceivedMessage& message, SharedN for (auto it = _fileMappings.cbegin(); it != _fileMappings.cend(); ++ it) { replyPacket.writeString(it.key()); replyPacket.write(QByteArray::fromHex(it.value().toString().toUtf8())); + replyPacket.writePrimitive(_baker.getAssetStatus(it.value().toString())); } } diff --git a/assignment-client/src/assets/AutoBaker.h b/assignment-client/src/assets/AutoBaker.h index 7b92118aee..805130bf52 100644 --- a/assignment-client/src/assets/AutoBaker.h +++ b/assignment-client/src/assets/AutoBaker.h @@ -18,18 +18,11 @@ class AutoBaker { public: - enum Status { - NotBaked, - Pending, - Baking, - Baked - }; - void addPendingBake(AssetHash hash); bool assetNeedsBaking(AssetHash hash); - Status getAssetStatus(AssetHash hash); + BakingStatus getAssetStatus(AssetHash hash); private: std::vector _pendingBakes; diff --git a/libraries/networking/src/AssetUtils.cpp b/libraries/networking/src/AssetUtils.cpp index 7818c8e5ce..7c8ffd6746 100644 --- a/libraries/networking/src/AssetUtils.cpp +++ b/libraries/networking/src/AssetUtils.cpp @@ -84,3 +84,16 @@ bool isValidHash(const AssetHash& hash) { QRegExp hashRegex { ASSET_HASH_REGEX_STRING }; return hashRegex.exactMatch(hash); } + +QString bakingStatusToString(BakingStatus status) { + switch (status) { + case NotBaked: + return "Not Baked"; + case Pending: + return "Pending"; + case Baking: + return "Baking"; + case Baked: + return "Baked"; + } +} diff --git a/libraries/networking/src/AssetUtils.h b/libraries/networking/src/AssetUtils.h index 4137193274..a94363126c 100644 --- a/libraries/networking/src/AssetUtils.h +++ b/libraries/networking/src/AssetUtils.h @@ -52,6 +52,13 @@ enum AssetMappingOperationType : uint8_t { Rename }; +enum BakingStatus { + NotBaked, + Pending, + Baking, + Baked +}; + QUrl getATPUrl(const QString& hash); QByteArray hashData(const QByteArray& data); @@ -63,4 +70,6 @@ bool isValidFilePath(const AssetPath& path); bool isValidPath(const AssetPath& path); bool isValidHash(const QString& hashString); +QString bakingStatusToString(BakingStatus status); + #endif // hifi_AssetUtils_h diff --git a/libraries/networking/src/MappingRequest.cpp b/libraries/networking/src/MappingRequest.cpp index 5fb9c32a0b..437c7c1b41 100644 --- a/libraries/networking/src/MappingRequest.cpp +++ b/libraries/networking/src/MappingRequest.cpp @@ -136,6 +136,8 @@ void GetAllMappingsRequest::doStart() { for (auto i = 0; i < numberOfMappings; ++i) { auto path = message->readString(); auto hash = message->read(SHA256_HASH_LENGTH).toHex(); + BakingStatus status; + message->readPrimitive(&status); _mappings[path] = hash; } } diff --git a/libraries/shared/src/shared/Algorithms.h b/libraries/shared/src/shared/Algorithms.h new file mode 100644 index 0000000000..4d486efc43 --- /dev/null +++ b/libraries/shared/src/shared/Algorithms.h @@ -0,0 +1,28 @@ +// +// Algorithms.h +// libraries/shared/src/shared +// +// Created by Clement Brisset on 8/9/17 +// Copyright 2017 High Fidelity, Inc. +// +// Distributed under the Apache License, Version 2.0. +// See the accompanying file LICENSE or http://www.apache.org/licenses/LICENSE-2.0.html +// + +#ifndef Algorithms_h +#define Algorithms_h + +#include + +namespace alg { + + + + + + + + +} + +#endif /* Algorithms_hpp */ From 859ec57ded4f8925e5201ca1ce59df99c86a44ea Mon Sep 17 00:00:00 2001 From: Atlante45 Date: Fri, 11 Aug 2017 15:27:13 -0700 Subject: [PATCH 014/129] Move Baking Status --- assignment-client/src/assets/AutoBaker.cpp | 13 +++++++----- libraries/shared/src/shared/Algorithms.h | 24 +++++++++++++++------- 2 files changed, 25 insertions(+), 12 deletions(-) diff --git a/assignment-client/src/assets/AutoBaker.cpp b/assignment-client/src/assets/AutoBaker.cpp index c56e00d025..fae895e635 100644 --- a/assignment-client/src/assets/AutoBaker.cpp +++ b/assignment-client/src/assets/AutoBaker.cpp @@ -10,23 +10,26 @@ // #include "AutoBaker.h" +#include + +using namespace alg; void AutoBaker::addPendingBake(AssetHash hash) { _pendingBakes.push_back(hash); + + // Maybe start baking it right away } bool AutoBaker::assetNeedsBaking(AssetHash hash) { return true; } -AutoBaker::Status AutoBaker::getAssetStatus(AssetHash hash) { - auto pendingIt = std::find(_pendingBakes.cbegin(), _pendingBakes.cend(), hash); - if (pendingIt != _pendingBakes.cend()) { +BakingStatus AutoBaker::getAssetStatus(AssetHash hash) { + if (find(_pendingBakes, hash) != std::end(_pendingBakes)) { return Pending; } - auto bakingIt = std::find(_currentlyBaking.cbegin(), _currentlyBaking.cend(), hash); - if (bakingIt != _currentlyBaking.cend()) { + if (find(_currentlyBaking, hash) != std::end(_currentlyBaking)) { return Baking; } diff --git a/libraries/shared/src/shared/Algorithms.h b/libraries/shared/src/shared/Algorithms.h index 4d486efc43..b85a3df6c5 100644 --- a/libraries/shared/src/shared/Algorithms.h +++ b/libraries/shared/src/shared/Algorithms.h @@ -9,20 +9,30 @@ // See the accompanying file LICENSE or http://www.apache.org/licenses/LICENSE-2.0.html // -#ifndef Algorithms_h -#define Algorithms_h +#ifndef hifi_Algorithms_h +#define hifi_Algorithms_h #include +#include +#include namespace alg { +template +auto find(const Container& container, const ValueType& value) -> decltype(std::begin(container)) { + return std::find(std::begin(container), std::end(container), value); +} +template +auto find_if(const Container& container, Predicate&& predicate) -> decltype(std::begin(container)) { + return std::find_if(std::begin(container), std::end(container), std::forward(predicate)); +} - - - - +template +auto find_if_not(const Container& container, Predicate&& predicate) -> decltype(std::begin(container)) { + return std::find_if_not(std::begin(container), std::end(container), std::forward(predicate)); +} } -#endif /* Algorithms_hpp */ +#endif // hifi_Algorithms_hpp From 28c9aa031ca8a4cc3fdf2e128324635760703c38 Mon Sep 17 00:00:00 2001 From: Atlante45 Date: Mon, 21 Aug 2017 15:59:19 -0700 Subject: [PATCH 015/129] Setup script model status --- .../AssetMappingsScriptingInterface.cpp | 47 ++++++++++++------- .../AssetMappingsScriptingInterface.h | 4 ++ libraries/networking/src/AssetUtils.h | 8 +++- libraries/networking/src/MappingRequest.cpp | 2 +- libraries/networking/src/MappingRequest.h | 2 +- tools/atp-client/src/ATPClientApp.cpp | 2 +- 6 files changed, 45 insertions(+), 20 deletions(-) diff --git a/interface/src/scripting/AssetMappingsScriptingInterface.cpp b/interface/src/scripting/AssetMappingsScriptingInterface.cpp index 5a81fe8749..b3509075bd 100644 --- a/interface/src/scripting/AssetMappingsScriptingInterface.cpp +++ b/interface/src/scripting/AssetMappingsScriptingInterface.cpp @@ -21,19 +21,6 @@ #include #include -void AssetMappingModel::clear() { - // make sure we are on the same thread before we touch the hash - if (thread() != QThread::currentThread()) { - QMetaObject::invokeMethod(this, "clear"); - return; - } - - qDebug() << "Clearing loaded asset mappings for Asset Browser"; - - _pathToItemMap.clear(); - QStandardItemModel::clear(); -} - AssetMappingsScriptingInterface::AssetMappingsScriptingInterface() { _proxyModel.setSourceModel(&_assetMappingModel); _proxyModel.setSortRole(Qt::DisplayRole); @@ -154,7 +141,7 @@ void AssetMappingsScriptingInterface::getAllMappings(QJSValue callback) { auto map = callback.engine()->newObject(); for (auto& kv : mappings ) { - map.setProperty(kv.first, kv.second); + map.setProperty(kv.first, kv.second.hash); } if (callback.isCallable()) { @@ -184,6 +171,10 @@ void AssetMappingsScriptingInterface::renameMapping(QString oldPath, QString new request->start(); } +AssetMappingModel::AssetMappingModel() { + setupHeaders(); +} + bool AssetMappingModel::isKnownFolder(QString path) const { if (!path.endsWith("/")) { return false; @@ -228,17 +219,19 @@ void AssetMappingModel::refresh() { if (it == _pathToItemMap.end()) { auto item = new QStandardItem(parts[i]); bool isFolder = i < length - 1; + auto statusString = isFolder ? "--" : bakingStatusToString(mapping.second.status); item->setData(isFolder ? fullPath + "/" : fullPath, Qt::UserRole); item->setData(isFolder, Qt::UserRole + 1); item->setData(parts[i], Qt::UserRole + 2); item->setData("atp:" + fullPath, Qt::UserRole + 3); item->setData(fullPath, Qt::UserRole + 4); + item->setData(statusString, Qt::UserRole + 5); + if (lastItem) { - lastItem->setChild(lastItem->rowCount(), 0, item); + lastItem->appendRow(item); } else { appendRow(item); } - lastItem = item; _pathToItemMap[fullPath] = lastItem; } else { @@ -300,3 +293,25 @@ void AssetMappingModel::refresh() { request->start(); } + +void AssetMappingModel::clear() { + // make sure we are on the same thread before we touch the hash + if (thread() != QThread::currentThread()) { + QMetaObject::invokeMethod(this, "clear"); + return; + } + + qDebug() << "Clearing loaded asset mappings for Asset Browser"; + + _pathToItemMap.clear(); + QStandardItemModel::clear(); + setupHeaders(); // restore headers +} + +void AssetMappingModel::setupHeaders() { + setHorizontalHeaderLabels(QStringList() << "Name" << "Use Baked?"); + QHash roleNames; + roleNames[Qt::DisplayRole] = "name"; + roleNames[Qt::UserRole + 5] = "baked"; + setItemRoleNames(roleNames); +} \ No newline at end of file diff --git a/interface/src/scripting/AssetMappingsScriptingInterface.h b/interface/src/scripting/AssetMappingsScriptingInterface.h index b7fcea2491..948bc9c9e7 100644 --- a/interface/src/scripting/AssetMappingsScriptingInterface.h +++ b/interface/src/scripting/AssetMappingsScriptingInterface.h @@ -26,6 +26,8 @@ class AssetMappingModel : public QStandardItemModel { Q_OBJECT public: + AssetMappingModel(); + Q_INVOKABLE void refresh(); bool isKnownMapping(QString path) const { return _pathToItemMap.contains(path); } @@ -38,6 +40,8 @@ signals: void errorGettingMappings(QString errorString); private: + void setupHeaders(); + QHash _pathToItemMap; }; diff --git a/libraries/networking/src/AssetUtils.h b/libraries/networking/src/AssetUtils.h index a94363126c..4b78d8aaaf 100644 --- a/libraries/networking/src/AssetUtils.h +++ b/libraries/networking/src/AssetUtils.h @@ -23,7 +23,6 @@ using DataOffset = int64_t; using AssetPath = QString; using AssetHash = QString; -using AssetMapping = std::map; using AssetPathList = QStringList; const size_t SHA256_HASH_LENGTH = 32; @@ -59,6 +58,13 @@ enum BakingStatus { Baked }; +struct MappingInfo { + AssetHash hash; + BakingStatus status; +}; + +using AssetMapping = std::map; + QUrl getATPUrl(const QString& hash); QByteArray hashData(const QByteArray& data); diff --git a/libraries/networking/src/MappingRequest.cpp b/libraries/networking/src/MappingRequest.cpp index 437c7c1b41..5ad1a83d2e 100644 --- a/libraries/networking/src/MappingRequest.cpp +++ b/libraries/networking/src/MappingRequest.cpp @@ -138,7 +138,7 @@ void GetAllMappingsRequest::doStart() { auto hash = message->read(SHA256_HASH_LENGTH).toHex(); BakingStatus status; message->readPrimitive(&status); - _mappings[path] = hash; + _mappings[path] = { hash, status }; } } emit finished(this); diff --git a/libraries/networking/src/MappingRequest.h b/libraries/networking/src/MappingRequest.h index e1cd8a39e8..3875915721 100644 --- a/libraries/networking/src/MappingRequest.h +++ b/libraries/networking/src/MappingRequest.h @@ -130,7 +130,7 @@ signals: private: virtual void doStart() override; - std::map _mappings; + AssetMapping _mappings; }; diff --git a/tools/atp-client/src/ATPClientApp.cpp b/tools/atp-client/src/ATPClientApp.cpp index 7d091aec74..c5edf27b67 100644 --- a/tools/atp-client/src/ATPClientApp.cpp +++ b/tools/atp-client/src/ATPClientApp.cpp @@ -332,7 +332,7 @@ void ATPClientApp::listAssets() { } else if (result == GetAllMappingsRequest::NoError) { auto mappings = request->getMappings(); for (auto& kv : mappings ) { - qDebug() << kv.first << kv.second; + qDebug() << kv.first << kv.second.hash; } } else { qDebug() << "error -- " << request->getError() << " -- " << request->getErrorString(); From 9e6502fe9222bbc7e9070eef3e3a8b723289fca3 Mon Sep 17 00:00:00 2001 From: Atlante45 Date: Tue, 22 Aug 2017 13:31:28 -0700 Subject: [PATCH 016/129] More UI work --- interface/resources/qml/AssetServer.qml | 110 ++++++++++++------ interface/resources/qml/controls-uit/Tree.qml | 109 ++++++++++++++--- .../qml/hifi/dialogs/RunningScripts.qml | 4 + .../AssetMappingsScriptingInterface.cpp | 8 +- .../AssetMappingsScriptingInterface.h | 3 +- 5 files changed, 178 insertions(+), 56 deletions(-) diff --git a/interface/resources/qml/AssetServer.qml b/interface/resources/qml/AssetServer.qml index ee37dbd8db..0b307bbc3b 100644 --- a/interface/resources/qml/AssetServer.qml +++ b/interface/resources/qml/AssetServer.qml @@ -48,6 +48,7 @@ ScrollingWindow { Component.onCompleted: { ApplicationInterface.uploadRequest.connect(uploadClicked); assetMappingsModel.errorGettingMappings.connect(handleGetMappingsError); + reload(); } @@ -464,20 +465,10 @@ ScrollingWindow { HifiControls.VerticalSpacer {} Row { - id: buttonRow anchors.left: parent.left anchors.right: parent.right spacing: hifi.dimensions.contentSpacing.x - HifiControls.GlyphButton { - glyph: hifi.glyphs.reload - color: hifi.buttons.black - colorScheme: root.colorScheme - width: hifi.dimensions.controlLineHeight - - onClicked: root.reload() - } - HifiControls.Button { text: "Add To World" color: hifi.buttons.black @@ -510,8 +501,64 @@ ScrollingWindow { onClicked: root.deleteFile() enabled: treeView.selection.hasSelection } - } + + HifiControls.GlyphButton { + glyph: hifi.glyphs.reload + color: hifi.buttons.black + colorScheme: root.colorScheme + width: hifi.dimensions.controlLineHeight + + onClicked: root.reload() + } + } + } + + HifiControls.Tree { + id: treeView + anchors.top: assetDirectory.bottom + anchors.bottom: infoRow.top + anchors.margins: hifi.dimensions.contentMargin.x + 2 // Extra for border + anchors.left: parent.left + anchors.right: parent.right + + treeModel: assetProxyModel + selectionMode: SelectionMode.ExtendedSelection + headerVisible: true + sortIndicatorVisible: true + + canEdit: true + colorScheme: root.colorScheme + + modifyEl: renameEl + + TableViewColumn { + id: nameColumn + title: "Name:" + role: "name" + width: treeView.width - bakedColumn.width; + } + TableViewColumn { + id: bakedColumn + title: "Use Baked?" + role: "baked" + width: 120 + } + + MouseArea { + propagateComposedEvents: true + anchors.fill: parent + acceptedButtons: Qt.RightButton + onClicked: { + if (!HMD.active) { // Popup only displays properly on desktop + var index = treeView.indexAt(mouse.x, mouse.y); + treeView.selection.setCurrentIndex(index, 0x0002); + contextMenu.currentIndex = index; + contextMenu.popup(); + } + } + } + Menu { id: contextMenu title: "Edit" @@ -541,36 +588,25 @@ ScrollingWindow { } } - HifiControls.Tree { - id: treeView - anchors.top: assetDirectory.bottom + Row { + id: infoRow + anchors.left: treeView.left + anchors.right: treeView.right anchors.bottom: uploadSection.top - anchors.margins: hifi.dimensions.contentMargin.x + 2 // Extra for border - anchors.left: parent.left - anchors.right: parent.right - - treeModel: assetProxyModel - canEdit: true - colorScheme: root.colorScheme - selectionMode: SelectionMode.ExtendedSelection - - modifyEl: renameEl - - MouseArea { - propagateComposedEvents: true - anchors.fill: parent - acceptedButtons: Qt.RightButton - onClicked: { - if (!HMD.active) { // Popup only displays properly on desktop - var index = treeView.indexAt(mouse.x, mouse.y); - treeView.selection.setCurrentIndex(index, 0x0002); - contextMenu.currentIndex = index; - contextMenu.popup(); - } - } + anchors.bottomMargin: hifi.dimensions.contentSpacing.y + spacing: hifi.dimensions.contentSpacing.x + + HifiControls.Label { + text: treeView.selection.selectedIndexes.length + " ITEMS SELECTED" + colorScheme: root.colorScheme } + HifiControls.CheckBox { + text: "Use baked (optimized) versions" + colorScheme: root.colorScheme + } } + HifiControls.ContentSection { id: uploadSection name: "Upload A File" diff --git a/interface/resources/qml/controls-uit/Tree.qml b/interface/resources/qml/controls-uit/Tree.qml index 8bce092947..711c6b4e5f 100644 --- a/interface/resources/qml/controls-uit/Tree.qml +++ b/interface/resources/qml/controls-uit/Tree.qml @@ -20,6 +20,7 @@ TreeView { property var treeModel: ListModel { } property var canEdit: false + property bool centerHeaderText: false property int colorScheme: hifi.colorSchemes.light readonly property bool isLightColorScheme: colorScheme == hifi.colorSchemes.light @@ -30,14 +31,9 @@ TreeView { model: treeModel } - TableViewColumn { - role: "display"; - } - anchors { left: parent.left; right: parent.right } - + headerVisible: false - headerDelegate: Item { } // Fix OSX QML bug that displays scrollbar starting too low. // Use rectangle to draw border with rounded corners. frameVisible: false @@ -60,6 +56,64 @@ TreeView { // Needed in order for rows to keep displaying rows after end of table entries. backgroundColor: parent.isLightColorScheme ? hifi.colors.tableRowLightEven : hifi.colors.tableRowDarkEven alternateBackgroundColor: parent.isLightColorScheme ? hifi.colors.tableRowLightOdd : hifi.colors.tableRowDarkOdd + + headerDelegate: Rectangle { + height: hifi.dimensions.tableHeaderHeight + color: isLightColorScheme ? hifi.colors.tableBackgroundLight : hifi.colors.tableBackgroundDark + + RalewayRegular { + id: titleText + text: styleData.value + size: hifi.fontSizes.tableHeading + font.capitalization: Font.AllUppercase + color: hifi.colors.baseGrayHighlight + horizontalAlignment: (centerHeaderText ? Text.AlignHCenter : Text.AlignLeft) + elide: Text.ElideRight + anchors { + left: parent.left + leftMargin: hifi.dimensions.tablePadding + right: sortIndicatorVisible && sortIndicatorColumn === styleData.column ? titleSort.left : parent.right + rightMargin: hifi.dimensions.tablePadding + verticalCenter: parent.verticalCenter + } + } + + HiFiGlyphs { + id: titleSort + text: sortIndicatorOrder == Qt.AscendingOrder ? hifi.glyphs.caratUp : hifi.glyphs.caratDn + color: isLightColorScheme ? hifi.colors.darkGray : hifi.colors.baseGrayHighlight + opacity: 0.6; + size: hifi.fontSizes.tableHeadingIcon + anchors { + right: parent.right + verticalCenter: titleText.verticalCenter + } + visible: sortIndicatorVisible && sortIndicatorColumn === styleData.column + } + + Rectangle { + width: 1 + anchors { + left: parent.left + top: parent.top + topMargin: 1 + bottom: parent.bottom + bottomMargin: 2 + } + color: isLightColorScheme ? hifi.colors.lightGrayText : hifi.colors.baseGrayHighlight + visible: styleData.column > 0 + } + + Rectangle { + height: 1 + anchors { + left: parent.left + right: parent.right + bottom: parent.bottom + } + color: isLightColorScheme ? hifi.colors.lightGrayText : hifi.colors.baseGrayHighlight + } + } branchDelegate: HiFiGlyphs { text: styleData.isExpanded ? hifi.glyphs.caratDn : hifi.glyphs.caratR @@ -75,28 +129,53 @@ TreeView { handle: Item { id: scrollbarHandle - implicitWidth: 6 + implicitWidth: hifi.dimensions.scrollbarHandleWidth Rectangle { anchors { fill: parent - leftMargin: 2 // Move it right - rightMargin: -2 // "" - topMargin: 3 // Shrink vertically + topMargin: treeView.headerVisible ? hifi.dimensions.tableHeaderHeight + 3 : 3 bottomMargin: 3 // "" + leftMargin: 1 // Move it right + rightMargin: -1 // "" } - radius: 3 - color: hifi.colors.tableScrollHandleDark + radius: hifi.dimensions.scrollbarHandleWidth / 2 + color: treeView.isLightColorScheme ? hifi.colors.tableScrollHandleLight : hifi.colors.tableScrollHandleDark } } scrollBarBackground: Item { - implicitWidth: 9 + implicitWidth: hifi.dimensions.scrollbarBackgroundWidth Rectangle { anchors { fill: parent + topMargin: treeView.headerVisible ? hifi.dimensions.tableHeaderHeight - 1 : -1 margins: -1 // Expand } - color: hifi.colors.tableBackgroundDark + color: treeView.isLightColorScheme ? hifi.colors.tableScrollBackgroundLight : hifi.colors.tableScrollBackgroundDark + + // Extend header color above scrollbar background + Rectangle { + anchors { + top: parent.top + topMargin: -hifi.dimensions.tableHeaderHeight + left: parent.left + right: parent.right + } + height: hifi.dimensions.tableHeaderHeight + color: treeView.isLightColorScheme ? hifi.colors.tableBackgroundLight : hifi.colors.tableBackgroundDark + visible: treeView.headerVisible + } + Rectangle { + // Extend header bottom border + anchors { + top: parent.top + left: parent.left + right: parent.right + } + height: 1 + color: treeView.isLightColorScheme ? hifi.colors.lightGrayText : hifi.colors.baseGrayHighlight + visible: treeView.headerVisible + } } } @@ -148,6 +227,8 @@ TreeView { color: colorScheme == hifi.colorSchemes.light ? (styleData.selected ? hifi.colors.black : hifi.colors.baseGrayHighlight) : (styleData.selected ? hifi.colors.black : hifi.colors.lightGrayText) + + elide: Text.ElideRight } } Component { diff --git a/interface/resources/qml/hifi/dialogs/RunningScripts.qml b/interface/resources/qml/hifi/dialogs/RunningScripts.qml index 952cc03733..a649c2c4d6 100644 --- a/interface/resources/qml/hifi/dialogs/RunningScripts.qml +++ b/interface/resources/qml/hifi/dialogs/RunningScripts.qml @@ -371,6 +371,10 @@ ScrollingWindow { colorScheme: hifi.colorSchemes.dark anchors.left: parent.left anchors.right: parent.right + + TableViewColumn { + role: "display"; + } } HifiControls.VerticalSpacer { diff --git a/interface/src/scripting/AssetMappingsScriptingInterface.cpp b/interface/src/scripting/AssetMappingsScriptingInterface.cpp index b3509075bd..f2929501ab 100644 --- a/interface/src/scripting/AssetMappingsScriptingInterface.cpp +++ b/interface/src/scripting/AssetMappingsScriptingInterface.cpp @@ -172,7 +172,7 @@ void AssetMappingsScriptingInterface::renameMapping(QString oldPath, QString new } AssetMappingModel::AssetMappingModel() { - setupHeaders(); + setupRoles(); } bool AssetMappingModel::isKnownFolder(QString path) const { @@ -288,6 +288,8 @@ void AssetMappingModel::refresh() { emit errorGettingMappings(request->getErrorString()); } + emit updated(); + request->deleteLater(); }); @@ -305,11 +307,9 @@ void AssetMappingModel::clear() { _pathToItemMap.clear(); QStandardItemModel::clear(); - setupHeaders(); // restore headers } -void AssetMappingModel::setupHeaders() { - setHorizontalHeaderLabels(QStringList() << "Name" << "Use Baked?"); +void AssetMappingModel::setupRoles() { QHash roleNames; roleNames[Qt::DisplayRole] = "name"; roleNames[Qt::UserRole + 5] = "baked"; diff --git a/interface/src/scripting/AssetMappingsScriptingInterface.h b/interface/src/scripting/AssetMappingsScriptingInterface.h index 948bc9c9e7..0615c5e84d 100644 --- a/interface/src/scripting/AssetMappingsScriptingInterface.h +++ b/interface/src/scripting/AssetMappingsScriptingInterface.h @@ -38,9 +38,10 @@ public slots: signals: void errorGettingMappings(QString errorString); + void updated(); private: - void setupHeaders(); + void setupRoles(); QHash _pathToItemMap; }; From 348be788f77885d157300a7a95dca0f21b19ce3f Mon Sep 17 00:00:00 2001 From: Atlante45 Date: Thu, 24 Aug 2017 11:14:09 -0700 Subject: [PATCH 017/129] Add BakingEnabled protocol --- assignment-client/src/assets/AssetServer.cpp | 54 +++++++++++++------ assignment-client/src/assets/AssetServer.h | 3 ++ assignment-client/src/assets/AutoBaker.cpp | 10 ++-- assignment-client/src/assets/AutoBaker.h | 2 +- interface/resources/qml/AssetServer.qml | 11 ++-- interface/resources/qml/controls-uit/Tree.qml | 20 ++++++- libraries/networking/src/AssetClient.cpp | 40 ++++++++++++++ libraries/networking/src/AssetClient.h | 4 ++ libraries/networking/src/AssetUtils.cpp | 4 ++ libraries/networking/src/AssetUtils.h | 7 ++- libraries/networking/src/MappingRequest.cpp | 43 +++++++++++++++ libraries/networking/src/MappingRequest.h | 15 ++++++ .../networking/src/udt/PacketHeaders.cpp | 4 +- libraries/networking/src/udt/PacketHeaders.h | 4 ++ .../src/AssetScriptingInterface.cpp | 14 +++++ .../src/AssetScriptingInterface.h | 2 + 16 files changed, 205 insertions(+), 32 deletions(-) diff --git a/assignment-client/src/assets/AssetServer.cpp b/assignment-client/src/assets/AssetServer.cpp index ea0010cb26..c5de1005f5 100644 --- a/assignment-client/src/assets/AssetServer.cpp +++ b/assignment-client/src/assets/AssetServer.cpp @@ -334,11 +334,10 @@ void AssetServer::completeSetup() { QRegExp hashFileRegex { "^[a-f0-9]{" + QString::number(SHA256_HASH_HEX_LENGTH) + "}" }; auto files = _filesDirectory.entryInfoList(QDir::Files); - auto mappedHashes = _fileMappings.values(); - for (const auto& fileInfo : files) { - AssetHash hash = fileInfo.fileName(); - bool isAsset = hashFileRegex.exactMatch(hash); - if (isAsset && _baker.assetNeedsBaking(hash)) { + for (auto& it = _fileMappings.cbegin(); it != _fileMappings.cend(); ++it) { + AssetPath path = it.key(); + AssetHash hash = it.value().toString(); + if (_baker.assetNeedsBaking(path, hash)) { _baker.addPendingBake(hash); } } @@ -381,26 +380,24 @@ void AssetServer::handleAssetMappingOperation(QSharedPointer me replyPacket->writePrimitive(messageID); switch (operationType) { - case AssetMappingOperationType::Get: { + case AssetMappingOperationType::Get: handleGetMappingOperation(*message, senderNode, *replyPacket); break; - } - case AssetMappingOperationType::GetAll: { + case AssetMappingOperationType::GetAll: handleGetAllMappingOperation(*message, senderNode, *replyPacket); break; - } - case AssetMappingOperationType::Set: { + case AssetMappingOperationType::Set: handleSetMappingOperation(*message, senderNode, *replyPacket); break; - } - case AssetMappingOperationType::Delete: { + case AssetMappingOperationType::Delete: handleDeleteMappingsOperation(*message, senderNode, *replyPacket); break; - } - case AssetMappingOperationType::Rename: { + case AssetMappingOperationType::Rename: handleRenameMappingOperation(*message, senderNode, *replyPacket); break; - } + case AssetMappingOperationType::SetBakingEnabled: + handleSetBakingEnabledOperation(*message, senderNode, *replyPacket); + break; } auto nodeList = DependencyManager::get(); @@ -545,6 +542,30 @@ void AssetServer::handleRenameMappingOperation(ReceivedMessage& message, SharedN } } +void AssetServer::handleSetBakingEnabledOperation(ReceivedMessage& message, SharedNodePointer senderNode, NLPacketList& replyPacket) { + if (senderNode->getCanWriteToAssetServer()) { + bool enabled { true }; + message.readPrimitive(&enabled); + + int numberOfMappings{ 0 }; + message.readPrimitive(&numberOfMappings); + + QStringList mappings; + + for (int i = 0; i < numberOfMappings; ++i) { + mappings << message.readString(); + } + + if (setBakingEnabled(mappings, enabled)) { + replyPacket.writePrimitive(AssetServerError::NoError); + } else { + replyPacket.writePrimitive(AssetServerError::MappingOperationFailed); + } + } else { + replyPacket.writePrimitive(AssetServerError::PermissionDenied); + } +} + void AssetServer::handleAssetGetInfo(QSharedPointer message, SharedNodePointer senderNode) { QByteArray assetHash; MessageID messageID; @@ -1100,4 +1121,7 @@ bool AssetServer::createMetaFile(AssetHash originalAssetHash) { } else { return false; } + +bool AssetServer::setBakingEnabled(AssetPathList& paths, bool enabled) { + return "test"; } diff --git a/assignment-client/src/assets/AssetServer.h b/assignment-client/src/assets/AssetServer.h index 2ce223760f..4d2b8da877 100644 --- a/assignment-client/src/assets/AssetServer.h +++ b/assignment-client/src/assets/AssetServer.h @@ -64,6 +64,7 @@ private: void handleSetMappingOperation(ReceivedMessage& message, SharedNodePointer senderNode, NLPacketList& replyPacket); void handleDeleteMappingsOperation(ReceivedMessage& message, SharedNodePointer senderNode, NLPacketList& replyPacket); void handleRenameMappingOperation(ReceivedMessage& message, SharedNodePointer senderNode, NLPacketList& replyPacket); + void handleSetBakingEnabledOperation(ReceivedMessage& message, SharedNodePointer senderNode, NLPacketList& replyPacket); // Mapping file operations must be called from main assignment thread only bool loadMappingsFromFile(); @@ -78,6 +79,8 @@ private: /// Rename mapping from `oldPath` to `newPath`. Returns true if successful bool renameMapping(AssetPath oldPath, AssetPath newPath); + bool setBakingEnabled(AssetPathList& paths, bool enabled); + /// Delete any unmapped files from the local asset directory void cleanupUnmappedFiles(); diff --git a/assignment-client/src/assets/AutoBaker.cpp b/assignment-client/src/assets/AutoBaker.cpp index fae895e635..38c564425b 100644 --- a/assignment-client/src/assets/AutoBaker.cpp +++ b/assignment-client/src/assets/AutoBaker.cpp @@ -20,8 +20,8 @@ void AutoBaker::addPendingBake(AssetHash hash) { // Maybe start baking it right away } -bool AutoBaker::assetNeedsBaking(AssetHash hash) { - return true; +bool AutoBaker::assetNeedsBaking(AssetPath path, AssetHash hash) { + return path.endsWith(".fbx"); } BakingStatus AutoBaker::getAssetStatus(AssetHash hash) { @@ -33,9 +33,5 @@ BakingStatus AutoBaker::getAssetStatus(AssetHash hash) { return Baking; } - if (assetNeedsBaking(hash)) { - return NotBaked; - } else { - return Baked; - } + return NotBaked; } diff --git a/assignment-client/src/assets/AutoBaker.h b/assignment-client/src/assets/AutoBaker.h index 805130bf52..258fb15931 100644 --- a/assignment-client/src/assets/AutoBaker.h +++ b/assignment-client/src/assets/AutoBaker.h @@ -20,7 +20,7 @@ class AutoBaker { public: void addPendingBake(AssetHash hash); - bool assetNeedsBaking(AssetHash hash); + bool assetNeedsBaking(AssetPath path, AssetHash hash); BakingStatus getAssetStatus(AssetHash hash); diff --git a/interface/resources/qml/AssetServer.qml b/interface/resources/qml/AssetServer.qml index 0b307bbc3b..a34de72270 100644 --- a/interface/resources/qml/AssetServer.qml +++ b/interface/resources/qml/AssetServer.qml @@ -542,7 +542,7 @@ ScrollingWindow { id: bakedColumn title: "Use Baked?" role: "baked" - width: 120 + width: 140 } MouseArea { @@ -596,14 +596,17 @@ ScrollingWindow { anchors.bottomMargin: hifi.dimensions.contentSpacing.y spacing: hifi.dimensions.contentSpacing.x - HifiControls.Label { - text: treeView.selection.selectedIndexes.length + " ITEMS SELECTED" - colorScheme: root.colorScheme + RalewayRegular { + size: hifi.fontSizes.sectionName + font.capitalization: Font.AllUppercase + text: selectedItems + " items selected" + color: hifi.colors.lightGrayText } HifiControls.CheckBox { text: "Use baked (optimized) versions" colorScheme: root.colorScheme + enabled: selectedItems > 0 } } diff --git a/interface/resources/qml/controls-uit/Tree.qml b/interface/resources/qml/controls-uit/Tree.qml index 711c6b4e5f..b7d9558cb6 100644 --- a/interface/resources/qml/controls-uit/Tree.qml +++ b/interface/resources/qml/controls-uit/Tree.qml @@ -212,12 +212,16 @@ TreeView { if (treeView.canEdit && styleData.selected) { return textFieldComponent; } else { - return labelComponent; + if (styleData.value.startsWith("HifiGlyphs#")) { + return glyphComponent; + } else { + return labelComponent; + } } } sourceComponent: getComponent() - + Component { id: labelComponent FiraSansSemiBold { @@ -231,6 +235,18 @@ TreeView { elide: Text.ElideRight } } + Component { + id: glyphComponent + HiFiGlyphs { + text: styleData.value.replace("HifiGlyphs#", "") + size: hifi.fontSizes.tableText + color: colorScheme == hifi.colorSchemes.light + ? (styleData.selected ? hifi.colors.black : hifi.colors.baseGrayHighlight) + : (styleData.selected ? hifi.colors.black : hifi.colors.lightGrayText) + + elide: Text.ElideRight + } + } Component { id: textFieldComponent diff --git a/libraries/networking/src/AssetClient.cpp b/libraries/networking/src/AssetClient.cpp index cb0b620a54..0d4e7c8388 100644 --- a/libraries/networking/src/AssetClient.cpp +++ b/libraries/networking/src/AssetClient.cpp @@ -185,6 +185,14 @@ RenameMappingRequest* AssetClient::createRenameMappingRequest(const AssetPath& o return request; } +SetBakingEnabledRequest* AssetClient::createSetBakingEnabledRequest(const AssetPathList& path, bool enabled) { + auto bakingEnabledRequest = new SetBakingEnabledRequest(path, enabled); + + bakingEnabledRequest->moveToThread(thread()); + + return bakingEnabledRequest; +} + AssetRequest* AssetClient::createRequest(const AssetHash& hash, const ByteRange& byteRange) { auto request = new AssetRequest(hash, byteRange); @@ -585,6 +593,38 @@ MessageID AssetClient::renameAssetMapping(const AssetPath& oldPath, const AssetP return INVALID_MESSAGE_ID; } +MessageID AssetClient::setBakingEnabled(const AssetPathList& paths, bool enabled, MappingOperationCallback callback) { + auto nodeList = DependencyManager::get(); + SharedNodePointer assetServer = nodeList->soloNodeOfType(NodeType::AssetServer); + + if (assetServer) { + auto packetList = NLPacketList::create(PacketType::AssetMappingOperation, QByteArray(), true, true); + + auto messageID = ++_currentID; + packetList->writePrimitive(messageID); + + packetList->writePrimitive(AssetMappingOperationType::SetBakingEnabled); + + packetList->writePrimitive(enabled); + + packetList->writePrimitive(int(paths.size())); + + for (auto& path : paths) { + packetList->writeString(path); + } + + if (nodeList->sendPacketList(std::move(packetList), *assetServer) != -1) { + _pendingMappingRequests[assetServer][messageID] = callback; + + return messageID; + + } + } + + callback(false, AssetServerError::NoError, QSharedPointer()); + return INVALID_MESSAGE_ID; +} + bool AssetClient::cancelMappingRequest(MessageID id) { Q_ASSERT(QThread::currentThread() == thread()); diff --git a/libraries/networking/src/AssetClient.h b/libraries/networking/src/AssetClient.h index 3f6602b76b..8035aa886e 100644 --- a/libraries/networking/src/AssetClient.h +++ b/libraries/networking/src/AssetClient.h @@ -32,6 +32,7 @@ class SetMappingRequest; class GetAllMappingsRequest; class DeleteMappingsRequest; class RenameMappingRequest; +class SetBakingEnabledRequest; class AssetRequest; class AssetUpload; @@ -56,6 +57,7 @@ public: Q_INVOKABLE DeleteMappingsRequest* createDeleteMappingsRequest(const AssetPathList& paths); Q_INVOKABLE SetMappingRequest* createSetMappingRequest(const AssetPath& path, const AssetHash& hash); Q_INVOKABLE RenameMappingRequest* createRenameMappingRequest(const AssetPath& oldPath, const AssetPath& newPath); + Q_INVOKABLE SetBakingEnabledRequest* createSetBakingEnabledRequest(const AssetPathList& path, bool enabled); Q_INVOKABLE AssetRequest* createRequest(const AssetHash& hash, const ByteRange& byteRange = ByteRange()); Q_INVOKABLE AssetUpload* createUpload(const QString& filename); Q_INVOKABLE AssetUpload* createUpload(const QByteArray& data); @@ -81,6 +83,7 @@ private: MessageID setAssetMapping(const QString& path, const AssetHash& hash, MappingOperationCallback callback); MessageID deleteAssetMappings(const AssetPathList& paths, MappingOperationCallback callback); MessageID renameAssetMapping(const AssetPath& oldPath, const AssetPath& newPath, MappingOperationCallback callback); + MessageID setBakingEnabled(const AssetPathList& paths, bool enabled, MappingOperationCallback callback); MessageID getAssetInfo(const QString& hash, GetInfoCallback callback); MessageID getAsset(const QString& hash, DataOffset start, DataOffset end, @@ -119,6 +122,7 @@ private: friend class SetMappingRequest; friend class DeleteMappingsRequest; friend class RenameMappingRequest; + friend class SetBakingEnabledRequest; }; #endif diff --git a/libraries/networking/src/AssetUtils.cpp b/libraries/networking/src/AssetUtils.cpp index 7c8ffd6746..af085d6463 100644 --- a/libraries/networking/src/AssetUtils.cpp +++ b/libraries/networking/src/AssetUtils.cpp @@ -87,6 +87,8 @@ bool isValidHash(const AssetHash& hash) { QString bakingStatusToString(BakingStatus status) { switch (status) { + case Unrelevant: + return "--"; case NotBaked: return "Not Baked"; case Pending: @@ -95,5 +97,7 @@ QString bakingStatusToString(BakingStatus status) { return "Baking"; case Baked: return "Baked"; + case Error: + return "Error"; } } diff --git a/libraries/networking/src/AssetUtils.h b/libraries/networking/src/AssetUtils.h index 4b78d8aaaf..f8f0171a5d 100644 --- a/libraries/networking/src/AssetUtils.h +++ b/libraries/networking/src/AssetUtils.h @@ -48,14 +48,17 @@ enum AssetMappingOperationType : uint8_t { GetAll, Set, Delete, - Rename + Rename, + SetBakingEnabled }; enum BakingStatus { + Unrelevant, NotBaked, Pending, Baking, - Baked + Baked, + Error }; struct MappingInfo { diff --git a/libraries/networking/src/MappingRequest.cpp b/libraries/networking/src/MappingRequest.cpp index 5ad1a83d2e..258b7c6359 100644 --- a/libraries/networking/src/MappingRequest.cpp +++ b/libraries/networking/src/MappingRequest.cpp @@ -273,3 +273,46 @@ void RenameMappingRequest::doStart() { emit finished(this); }); } + +SetBakingEnabledRequest::SetBakingEnabledRequest(const AssetPathList& paths, bool enabled) : _paths(paths), _enabled(enabled) { + for (auto& path : _paths) { + path = path.trimmed(); + } +}; + +void SetBakingEnabledRequest::doStart() { + + // short circuit the request if any of the paths are invalid + for (auto& path : _paths) { + if (!isValidPath(path)) { + _error = MappingRequest::InvalidPath; + emit finished(this); + return; + } + } + + auto assetClient = DependencyManager::get(); + + _mappingRequestID = assetClient->setBakingEnabled(_paths, _enabled, + [this, assetClient](bool responseReceived, AssetServerError error, QSharedPointer message) { + + _mappingRequestID = INVALID_MESSAGE_ID; + if (!responseReceived) { + _error = NetworkError; + } else { + switch (error) { + case AssetServerError::NoError: + _error = NoError; + break; + case AssetServerError::PermissionDenied: + _error = PermissionDenied; + break; + default: + _error = UnknownError; + break; + } + } + + emit finished(this); + }); +}; \ No newline at end of file diff --git a/libraries/networking/src/MappingRequest.h b/libraries/networking/src/MappingRequest.h index 3875915721..fc43375469 100644 --- a/libraries/networking/src/MappingRequest.h +++ b/libraries/networking/src/MappingRequest.h @@ -133,5 +133,20 @@ private: AssetMapping _mappings; }; +class SetBakingEnabledRequest : public MappingRequest { + Q_OBJECT +public: + SetBakingEnabledRequest(const AssetPathList& path, bool enabled); + +signals: + void finished(SetBakingEnabledRequest* thisRequest); + +private: + virtual void doStart() override; + + AssetPathList _paths; + bool _enabled; +}; + #endif // hifi_MappingRequest_h diff --git a/libraries/networking/src/udt/PacketHeaders.cpp b/libraries/networking/src/udt/PacketHeaders.cpp index f453d096b6..74d3c46eb3 100644 --- a/libraries/networking/src/udt/PacketHeaders.cpp +++ b/libraries/networking/src/udt/PacketHeaders.cpp @@ -67,7 +67,9 @@ PacketVersion versionForPacketType(PacketType packetType) { case PacketType::MicrophoneAudioWithEcho: case PacketType::AudioStreamStats: return static_cast(AudioVersion::HighDynamicRangeVolume); - + case PacketType::AssetMappingOperation: + case PacketType::AssetMappingOperationReply: + return static_cast(AssetMappingOperationVersion::SetBakingEnabledOperation); default: return 17; } diff --git a/libraries/networking/src/udt/PacketHeaders.h b/libraries/networking/src/udt/PacketHeaders.h index 6b57c8a3cb..d71470383b 100644 --- a/libraries/networking/src/udt/PacketHeaders.h +++ b/libraries/networking/src/udt/PacketHeaders.h @@ -336,4 +336,8 @@ enum class MessageDataVersion : PacketVersion { TextOrBinaryData = 18 }; +enum class AssetMappingOperationVersion : PacketVersion { + SetBakingEnabledOperation = 18 +}; + #endif // hifi_PacketHeaders_h diff --git a/libraries/script-engine/src/AssetScriptingInterface.cpp b/libraries/script-engine/src/AssetScriptingInterface.cpp index 51b3d7ffbe..25e8c0dcf3 100644 --- a/libraries/script-engine/src/AssetScriptingInterface.cpp +++ b/libraries/script-engine/src/AssetScriptingInterface.cpp @@ -88,6 +88,20 @@ void AssetScriptingInterface::downloadData(QString urlString, QScriptValue callb assetRequest->start(); } +void AssetScriptingInterface::setBakingEnabled(QString path, bool enabled, QScriptValue callback) { + auto setBakingEnabledRequest = DependencyManager::get()->createSetBakingEnabledRequest({ path }, enabled); + + QObject::connect(setBakingEnabledRequest, &SetBakingEnabledRequest::finished, this, [this, callback](SetBakingEnabledRequest* request) mutable { + if (callback.isFunction()) { + QString error = request->getErrorString(); + QScriptValueList args{ error }; + callback.call(_engine->currentContext()->thisObject(), args); + } + request->deleteLater(); + }); + setBakingEnabledRequest->start(); +} + #if (PR_BUILD || DEV_BUILD) void AssetScriptingInterface::sendFakedHandshake() { auto nodeList = DependencyManager::get(); diff --git a/libraries/script-engine/src/AssetScriptingInterface.h b/libraries/script-engine/src/AssetScriptingInterface.h index 0238329b73..2812be65f9 100644 --- a/libraries/script-engine/src/AssetScriptingInterface.h +++ b/libraries/script-engine/src/AssetScriptingInterface.h @@ -75,6 +75,8 @@ public: * @param {string} error */ Q_INVOKABLE void setMapping(QString path, QString hash, QScriptValue callback); + + Q_INVOKABLE void setBakingEnabled(QString path, bool enabled, QScriptValue callback); #if (PR_BUILD || DEV_BUILD) Q_INVOKABLE void sendFakedHandshake(); From 7d103d24f4ea038ed4387fb226e2a058731305c6 Mon Sep 17 00:00:00 2001 From: Atlante45 Date: Fri, 25 Aug 2017 15:21:08 -0700 Subject: [PATCH 018/129] Integrate my and Huffman's work --- assignment-client/src/assets/AssetServer.cpp | 115 +++++++++++++++---- assignment-client/src/assets/AssetServer.h | 21 ++-- 2 files changed, 102 insertions(+), 34 deletions(-) diff --git a/assignment-client/src/assets/AssetServer.cpp b/assignment-client/src/assets/AssetServer.cpp index c5de1005f5..67e192cd65 100644 --- a/assignment-client/src/assets/AssetServer.cpp +++ b/assignment-client/src/assets/AssetServer.cpp @@ -53,11 +53,13 @@ static const QList BAKEABLE_TEXTURE_EXTENSIONS = QImageReader::suppo static const QString BAKED_MODEL_SIMPLE_NAME = "asset.fbx"; static const QString BAKED_TEXTURE_SIMPLE_NAME = "texture.ktx"; -BakeAssetTask::BakeAssetTask(const QString& assetHash, const QString& assetPath, const QString& filePath) +BakeAssetTask::BakeAssetTask(const AssetHash& assetHash, const AssetPath& assetPath, const QString& filePath) : _assetHash(assetHash), _assetPath(assetPath), _filePath(filePath) { } void BakeAssetTask::run() { + _isBaking.store(true); + qRegisterMetaType >("QVector"); TextureBakerThreadGetter fn = []() -> QThread* { return QThread::currentThread(); }; @@ -86,7 +88,7 @@ void BakeAssetTask::run() { } } -void AssetServer::bakeAsset(const QString& assetHash, const QString& assetPath, const QString& filePath) { +void AssetServer::bakeAsset(const AssetHash& assetHash, const AssetPath& assetPath, const QString& filePath) { qDebug() << "Starting bake for: " << assetPath << assetHash; auto it = _pendingBakes.find(assetHash); if (it == _pendingBakes.end()) { @@ -94,9 +96,7 @@ void AssetServer::bakeAsset(const QString& assetHash, const QString& assetPath, task->setAutoDelete(false); _pendingBakes[assetHash] = task; - connect(task.get(), &BakeAssetTask::bakeComplete, this, [this, assetPath](QString assetHash, QString assetPath, QVector outputFiles) { - handleCompletedBake(assetPath, assetHash, outputFiles); - }); + connect(task.get(), &BakeAssetTask::bakeComplete, this, &AssetServer::handleCompletedBake); _bakingTaskPool.start(task.get()); } else { @@ -108,6 +108,46 @@ QString AssetServer::getPathToAssetHash(const AssetHash& assetHash) { return _filesDirectory.absoluteFilePath(assetHash); } +BakingStatus AssetServer::getAssetStatus(const AssetPath& path, const AssetHash& hash) { + auto it = _pendingBakes.find(hash); + if (it == _pendingBakes.end()) { + return (*it)->isBaking() ? Baking : Pending; + } + + if (path.startsWith("/.baked/")) { + return Baked; + } + + auto dotIndex = path.lastIndexOf("."); + if (dotIndex == -1) { + return Unrelevant; + } + + auto extension = path.mid(dotIndex + 1); + + QString bakedFilename; + + if (BAKEABLE_MODEL_EXTENSIONS.contains(extension)) { + bakedFilename = BAKED_MODEL_SIMPLE_NAME; + } else if (BAKEABLE_TEXTURE_EXTENSIONS.contains(extension.toLocal8Bit()) && hasMetaFile(hash)) { + bakedFilename = BAKED_TEXTURE_SIMPLE_NAME; + } else { + return Unrelevant; + } + + auto bakedPath = "/.baked/" + hash + "/" + bakedFilename; + auto jt = _fileMappings.find(bakedPath); + if (jt != _fileMappings.end()) { + if (jt->toString() == hash) { + return NotBaked; + } else { + return Baked; + } + } + + return Pending; +} + void AssetServer::bakeAssets() { auto it = _fileMappings.cbegin(); for (; it != _fileMappings.cend(); ++it) { @@ -129,9 +169,9 @@ void AssetServer::createEmptyMetaFile(const AssetHash& hash) { QFile metaFile { metaFilePath }; if (!metaFile.exists()) { - qDebug() << "Creating metfaile for " << hash; + qDebug() << "Creating metafile for " << hash; if (metaFile.open(QFile::WriteOnly)) { - qDebug() << "Created metfaile for " << hash; + qDebug() << "Created metafile for " << hash; metaFile.write("{}"); } } @@ -331,16 +371,6 @@ void AssetServer::completeSetup() { qCCritical(asset_server) << "Asset Server assignment will not continue because mapping file could not be loaded."; setFinished(true); } - - QRegExp hashFileRegex { "^[a-f0-9]{" + QString::number(SHA256_HASH_HEX_LENGTH) + "}" }; - auto files = _filesDirectory.entryInfoList(QDir::Files); - for (auto& it = _fileMappings.cbegin(); it != _fileMappings.cend(); ++it) { - AssetPath path = it.key(); - AssetHash hash = it.value().toString(); - if (_baker.assetNeedsBaking(path, hash)) { - _baker.addPendingBake(hash); - } - } } void AssetServer::cleanupUnmappedFiles() { @@ -486,7 +516,7 @@ void AssetServer::handleGetAllMappingOperation(ReceivedMessage& message, SharedN for (auto it = _fileMappings.cbegin(); it != _fileMappings.cend(); ++ it) { replyPacket.writeString(it.key()); replyPacket.write(QByteArray::fromHex(it.value().toString().toUtf8())); - replyPacket.writePrimitive(_baker.getAssetStatus(it.value().toString())); + replyPacket.writePrimitive(getAssetStatus(it.value().toString())); } } @@ -1012,6 +1042,12 @@ bool AssetServer::renameMapping(AssetPath oldPath, AssetPath newPath) { } static const QString HIDDEN_BAKED_CONTENT_FOLDER = "/.baked/"; +static const QString BAKED_ASSET_SIMPLE_FBX_NAME = "asset.fbx"; +static const QString BAKED_ASSET_SIMPLE_TEXTURE_NAME = "texture.ktx"; + +QString getBakeMapping(const AssetHash& hash, const QString& relativeFilePath) { + return HIDDEN_BAKED_CONTENT_FOLDER + hash + "/" + relativeFilePath; +} void AssetServer::handleCompletedBake(AssetPath originalAssetPath, AssetHash originalAssetHash, QVector bakedFilePaths) { bool errorCompletingBake { false }; @@ -1051,19 +1087,17 @@ void AssetServer::handleCompletedBake(AssetPath originalAssetPath, AssetHash ori // setup the mapping for this bake file auto relativeFilePath = QUrl(filePath).fileName(); qDebug() << "Relative file path is: " << relativeFilePath; - static const QString BAKED_ASSET_SIMPLE_FBX_NAME = "asset.fbx"; - static const QString BAKED_ASSET_SIMPLE_TEXTURE_NAME = "texture.ktx"; if (relativeFilePath.endsWith(".fbx", Qt::CaseInsensitive)) { // for an FBX file, we replace the filename with the simple name // (to handle the case where two mapped assets have the same hash but different names) relativeFilePath = BAKED_ASSET_SIMPLE_FBX_NAME; - } else if (!originalAssetPath.endsWith(".fbx")) { + } else if (!originalAssetPath.endsWith(".fbx", Qt::CaseInsensitive)) { relativeFilePath = BAKED_ASSET_SIMPLE_TEXTURE_NAME; } - QString bakeMapping = HIDDEN_BAKED_CONTENT_FOLDER + originalAssetHash + "/" + relativeFilePath; + QString bakeMapping = getBakeMapping(originalAssetHash, relativeFilePath); // add a mapping (under the hidden baked folder) for this file resulting from the bake if (setMapping(bakeMapping, bakedFileHash)) { @@ -1121,7 +1155,38 @@ bool AssetServer::createMetaFile(AssetHash originalAssetHash) { } else { return false; } - -bool AssetServer::setBakingEnabled(AssetPathList& paths, bool enabled) { - return "test"; +} + +void AssetServer::setBakingEnabled(const AssetPathList& paths, bool enabled) { + for (const auto& path : paths) { + auto it = _fileMappings.find(path); + if (it != _fileMappings.end()) { + auto hash = it->toString(); + + auto dotIndex = path.lastIndexOf("."); + if (dotIndex == -1) { + continue; + } + + auto extension = path.mid(dotIndex + 1); + + QString bakedFilename; + + if (BAKEABLE_MODEL_EXTENSIONS.contains(extension)) { + bakedFilename = BAKED_MODEL_SIMPLE_NAME; + } else if (BAKEABLE_TEXTURE_EXTENSIONS.contains(extension.toLocal8Bit()) && hasMetaFile(hash)) { + bakedFilename = BAKED_TEXTURE_SIMPLE_NAME; + } else { + continue; + } + + auto bakedMapping = getBakeMapping(hash, bakedFilename); + + if (enabled) { + // TODO + } else { + // TODO + } + } + } } diff --git a/assignment-client/src/assets/AssetServer.h b/assignment-client/src/assets/AssetServer.h index 4d2b8da877..435e8e8b24 100644 --- a/assignment-client/src/assets/AssetServer.h +++ b/assignment-client/src/assets/AssetServer.h @@ -25,16 +25,19 @@ class BakeAssetTask : public QObject, public QRunnable { Q_OBJECT public: - BakeAssetTask(const QString& assetHash, const QString& assetPath, const QString& filePath); + BakeAssetTask(const AssetHash& assetHash, const AssetPath& assetPath, const QString& filePath); + + bool isBaking() { return _isBaking.load(); } void run() override; signals: - void bakeComplete(QString assetHash, QString assetPath, QVector outputFiles); + void bakeComplete(AssetHash assetHash, AssetPath assetPath, QVector outputFiles); private: - QString _assetHash; - QString _assetPath; + std::atomic _isBaking { false }; + AssetHash _assetHash; + AssetPath _assetPath; QString _filePath; }; @@ -79,19 +82,21 @@ private: /// Rename mapping from `oldPath` to `newPath`. Returns true if successful bool renameMapping(AssetPath oldPath, AssetPath newPath); - bool setBakingEnabled(AssetPathList& paths, bool enabled); + void setBakingEnabled(const AssetPathList& paths, bool enabled); /// Delete any unmapped files from the local asset directory void cleanupUnmappedFiles(); QString getPathToAssetHash(const AssetHash& assetHash); + BakingStatus getAssetStatus(const AssetPath& path, const AssetHash& hash); + void bakeAssets(); void maybeBake(const AssetPath& path, const AssetHash& hash); void createEmptyMetaFile(const AssetHash& hash); bool hasMetaFile(const AssetHash& hash); bool needsToBeBaked(const AssetPath& path, const AssetHash& assetHash); - void bakeAsset(const QString& assetHash, const QString& assetPath, const QString& filePath); + void bakeAsset(const AssetHash& assetHash, const AssetPath& assetPath, const QString& filePath); /// Move baked content for asset to baked directory and update baked status void handleCompletedBake(AssetPath assetPath, AssetHash originalAssetHash, QVector bakedFilePaths); @@ -107,10 +112,8 @@ private: /// Task pool for handling uploads and downloads of assets QThreadPool _transferTaskPool; - QHash> _pendingBakes; + QHash> _pendingBakes; QThreadPool _bakingTaskPool; - - AutoBaker _baker; }; #endif From eaa2b9fda841d9d460d7bea66a38d8430722172e Mon Sep 17 00:00:00 2001 From: Atlante45 Date: Fri, 25 Aug 2017 15:31:18 -0700 Subject: [PATCH 019/129] Fix packet version conflict --- assignment-client/src/assets/AssetServer.cpp | 5 +++-- assignment-client/src/assets/AssetServer.h | 2 +- libraries/networking/src/udt/PacketHeaders.cpp | 4 +--- libraries/networking/src/udt/PacketHeaders.h | 4 ---- 4 files changed, 5 insertions(+), 10 deletions(-) diff --git a/assignment-client/src/assets/AssetServer.cpp b/assignment-client/src/assets/AssetServer.cpp index 67e192cd65..3dc96caefc 100644 --- a/assignment-client/src/assets/AssetServer.cpp +++ b/assignment-client/src/assets/AssetServer.cpp @@ -516,7 +516,7 @@ void AssetServer::handleGetAllMappingOperation(ReceivedMessage& message, SharedN for (auto it = _fileMappings.cbegin(); it != _fileMappings.cend(); ++ it) { replyPacket.writeString(it.key()); replyPacket.write(QByteArray::fromHex(it.value().toString().toUtf8())); - replyPacket.writePrimitive(getAssetStatus(it.value().toString())); + replyPacket.writePrimitive(getAssetStatus(it.key(), it.value().toString())); } } @@ -1157,7 +1157,7 @@ bool AssetServer::createMetaFile(AssetHash originalAssetHash) { } } -void AssetServer::setBakingEnabled(const AssetPathList& paths, bool enabled) { +bool AssetServer::setBakingEnabled(const AssetPathList& paths, bool enabled) { for (const auto& path : paths) { auto it = _fileMappings.find(path); if (it != _fileMappings.end()) { @@ -1189,4 +1189,5 @@ void AssetServer::setBakingEnabled(const AssetPathList& paths, bool enabled) { } } } + return true; } diff --git a/assignment-client/src/assets/AssetServer.h b/assignment-client/src/assets/AssetServer.h index 435e8e8b24..fadd6109bf 100644 --- a/assignment-client/src/assets/AssetServer.h +++ b/assignment-client/src/assets/AssetServer.h @@ -82,7 +82,7 @@ private: /// Rename mapping from `oldPath` to `newPath`. Returns true if successful bool renameMapping(AssetPath oldPath, AssetPath newPath); - void setBakingEnabled(const AssetPathList& paths, bool enabled); + bool setBakingEnabled(const AssetPathList& paths, bool enabled); /// Delete any unmapped files from the local asset directory void cleanupUnmappedFiles(); diff --git a/libraries/networking/src/udt/PacketHeaders.cpp b/libraries/networking/src/udt/PacketHeaders.cpp index 74d3c46eb3..354a95850b 100644 --- a/libraries/networking/src/udt/PacketHeaders.cpp +++ b/libraries/networking/src/udt/PacketHeaders.cpp @@ -42,6 +42,7 @@ PacketVersion versionForPacketType(PacketType packetType) { return static_cast(MessageDataVersion::TextOrBinaryData); case PacketType::ICEServerHeartbeat: return 18; // ICE Server Heartbeat signing + case PacketType::AssetMappingOperation: case PacketType::AssetMappingOperationReply: return static_cast(AssetServerPacketVersion::RedirectedMappings); case PacketType::AssetGetInfo: @@ -67,9 +68,6 @@ PacketVersion versionForPacketType(PacketType packetType) { case PacketType::MicrophoneAudioWithEcho: case PacketType::AudioStreamStats: return static_cast(AudioVersion::HighDynamicRangeVolume); - case PacketType::AssetMappingOperation: - case PacketType::AssetMappingOperationReply: - return static_cast(AssetMappingOperationVersion::SetBakingEnabledOperation); default: return 17; } diff --git a/libraries/networking/src/udt/PacketHeaders.h b/libraries/networking/src/udt/PacketHeaders.h index 3a38879a79..c310065517 100644 --- a/libraries/networking/src/udt/PacketHeaders.h +++ b/libraries/networking/src/udt/PacketHeaders.h @@ -337,8 +337,4 @@ enum class MessageDataVersion : PacketVersion { TextOrBinaryData = 18 }; -enum class AssetMappingOperationVersion : PacketVersion { - SetBakingEnabledOperation = 18 -}; - #endif // hifi_PacketHeaders_h From a36ddf7b6d6e82323fb962e1db62d37f91e3c6c9 Mon Sep 17 00:00:00 2001 From: Atlante45 Date: Fri, 25 Aug 2017 16:13:29 -0700 Subject: [PATCH 020/129] Fix failed connection --- assignment-client/src/assets/AssetServer.cpp | 2 +- assignment-client/src/assets/AssetServer.h | 4 ++-- 2 files changed, 3 insertions(+), 3 deletions(-) diff --git a/assignment-client/src/assets/AssetServer.cpp b/assignment-client/src/assets/AssetServer.cpp index 3dc96caefc..7e1175d0a6 100644 --- a/assignment-client/src/assets/AssetServer.cpp +++ b/assignment-client/src/assets/AssetServer.cpp @@ -1049,7 +1049,7 @@ QString getBakeMapping(const AssetHash& hash, const QString& relativeFilePath) { return HIDDEN_BAKED_CONTENT_FOLDER + hash + "/" + relativeFilePath; } -void AssetServer::handleCompletedBake(AssetPath originalAssetPath, AssetHash originalAssetHash, QVector bakedFilePaths) { +void AssetServer::handleCompletedBake(QString originalAssetPath, QString originalAssetHash, QVector bakedFilePaths) { bool errorCompletingBake { false }; qDebug() << "Completing bake for " << originalAssetHash; diff --git a/assignment-client/src/assets/AssetServer.h b/assignment-client/src/assets/AssetServer.h index fadd6109bf..19ad23be08 100644 --- a/assignment-client/src/assets/AssetServer.h +++ b/assignment-client/src/assets/AssetServer.h @@ -32,7 +32,7 @@ public: void run() override; signals: - void bakeComplete(AssetHash assetHash, AssetPath assetPath, QVector outputFiles); + void bakeComplete(QString assetHash, QString assetPath, QVector outputFiles); private: std::atomic _isBaking { false }; @@ -99,7 +99,7 @@ private: void bakeAsset(const AssetHash& assetHash, const AssetPath& assetPath, const QString& filePath); /// Move baked content for asset to baked directory and update baked status - void handleCompletedBake(AssetPath assetPath, AssetHash originalAssetHash, QVector bakedFilePaths); + void handleCompletedBake(QString assetPath, QString originalAssetHash, QVector bakedFilePaths); /// Create meta file to describe baked content for original asset bool createMetaFile(AssetHash originalAssetHash); From 700b11e28be14c9177c77f8cb40598d0ec7749e8 Mon Sep 17 00:00:00 2001 From: Atlante45 Date: Fri, 25 Aug 2017 16:17:37 -0700 Subject: [PATCH 021/129] Fix bad conditional --- assignment-client/src/assets/AssetServer.cpp | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/assignment-client/src/assets/AssetServer.cpp b/assignment-client/src/assets/AssetServer.cpp index 7e1175d0a6..aa5224de0d 100644 --- a/assignment-client/src/assets/AssetServer.cpp +++ b/assignment-client/src/assets/AssetServer.cpp @@ -110,7 +110,7 @@ QString AssetServer::getPathToAssetHash(const AssetHash& assetHash) { BakingStatus AssetServer::getAssetStatus(const AssetPath& path, const AssetHash& hash) { auto it = _pendingBakes.find(hash); - if (it == _pendingBakes.end()) { + if (it != _pendingBakes.end()) { return (*it)->isBaking() ? Baking : Pending; } From 7d08a5788f547e869c1914b7d67dd9fc01568d48 Mon Sep 17 00:00:00 2001 From: Ryan Huffman Date: Fri, 25 Aug 2017 16:21:00 -0700 Subject: [PATCH 022/129] Move outputFiles from FBXBaker to Baker --- assignment-client/src/assets/AssetServer.cpp | 33 +++++++++----------- libraries/baking/src/Baker.h | 6 ++++ libraries/baking/src/FBXBaker.h | 4 --- libraries/baking/src/TextureBaker.cpp | 5 ++- 4 files changed, 24 insertions(+), 24 deletions(-) diff --git a/assignment-client/src/assets/AssetServer.cpp b/assignment-client/src/assets/AssetServer.cpp index 6e3195f570..ac1ef1c3ed 100644 --- a/assignment-client/src/assets/AssetServer.cpp +++ b/assignment-client/src/assets/AssetServer.cpp @@ -61,28 +61,24 @@ void BakeAssetTask::run() { qRegisterMetaType >("QVector"); TextureBakerThreadGetter fn = []() -> QThread* { return QThread::currentThread(); }; - if (_filePath.endsWith(".fbx")) { - FBXBaker baker(QUrl("file:///" + _filePath), fn, PathUtils::generateTemporaryDir()); + std::unique_ptr baker; - QEventLoop loop; - connect(&baker, &Baker::finished, &loop, &QEventLoop::quit); - QMetaObject::invokeMethod(&baker, "bake", Qt::QueuedConnection); - qDebug() << "Running the bake!"; - loop.exec(); - - qDebug() << "Finished baking: " << _assetHash << _assetPath << baker.getOutputFiles(); - emit bakeComplete(_assetHash, _assetPath, QVector::fromStdVector(baker.getOutputFiles())); + if (_assetPath.endsWith(".fbx")) { + baker = std::make_unique(QUrl("file:///" + _filePath), fn, PathUtils::generateTemporaryDir()); } else { - TextureBaker baker(QUrl("file:///" + _filePath), image::TextureUsage::CUBE_TEXTURE, PathUtils::generateTemporaryDir()); + baker = std::make_unique(QUrl("file:///" + _filePath), image::TextureUsage::CUBE_TEXTURE, PathUtils::generateTemporaryDir()); + } - QEventLoop loop; - connect(&baker, &Baker::finished, &loop, &QEventLoop::quit); - QMetaObject::invokeMethod(&baker, "bake", Qt::QueuedConnection); - qDebug() << "Running the bake!"; - loop.exec(); + QEventLoop loop; + connect(baker.get(), &Baker::finished, &loop, &QEventLoop::quit); + QMetaObject::invokeMethod(baker.get(), "bake", Qt::QueuedConnection); + loop.exec(); - qDebug() << "Finished baking: " << _assetHash << _assetPath << baker.getBakedTextureFileName(); - emit bakeComplete(_assetHash, _assetPath, { baker.getDestinationFilePath() }); + if (baker->hasErrors()) { + qDebug() << "Failed to bake: " << _assetHash << _assetPath; + } else { + qDebug() << "Finished baking: " << _assetHash << _assetPath << baker->getOutputFiles(); + emit bakeComplete(_assetHash, _assetPath, QVector::fromStdVector(baker->getOutputFiles())); } } @@ -454,7 +450,6 @@ void AssetServer::handleGetMappingOperation(ReceivedMessage& message, SharedNode replyPacket.write(QByteArray::fromHex(originalAssetHash.toUtf8())); replyPacket.writePrimitive(wasRedirected); - auto query = QUrlQuery(url.query()); bool isSkybox = query.hasQueryItem("skybox"); qDebug() << "Is skybox? " << isSkybox; diff --git a/libraries/baking/src/Baker.h b/libraries/baking/src/Baker.h index d7107428bf..fd76246497 100644 --- a/libraries/baking/src/Baker.h +++ b/libraries/baking/src/Baker.h @@ -24,6 +24,8 @@ public: bool hasWarnings() const { return !_warningList.isEmpty(); } QStringList getWarnings() const { return _warningList; } + std::vector getOutputFiles() const { return _outputFiles; } + public slots: virtual void bake() = 0; @@ -36,6 +38,10 @@ protected: void handleErrors(const QStringList& errors); + // List of baked output files. For instance, for an FBX this would + // include the .fbx and all of its texture files. + std::vector _outputFiles; + QStringList _errorList; QStringList _warningList; }; diff --git a/libraries/baking/src/FBXBaker.h b/libraries/baking/src/FBXBaker.h index 0e52f2e4c3..00e7987422 100644 --- a/libraries/baking/src/FBXBaker.h +++ b/libraries/baking/src/FBXBaker.h @@ -44,7 +44,6 @@ public: QUrl getFBXUrl() const { return _fbxURL; } QString getBakedFBXFilePath() const { return _bakedFBXFilePath; } - std::vector getOutputFiles() const { return _outputFiles; } public slots: // all calls to FBXBaker::bake for FBXBaker instances must be from the same thread @@ -88,9 +87,6 @@ private: QDir _tempDir; QString _originalFBXFilePath; - // List of baked output files, includes the FBX and textures - std::vector _outputFiles; - static FBXSDKManagerUniquePointer _sdkManager; fbxsdk::FbxScene* _scene { nullptr }; diff --git a/libraries/baking/src/TextureBaker.cpp b/libraries/baking/src/TextureBaker.cpp index 70df511d2c..44995fa026 100644 --- a/libraries/baking/src/TextureBaker.cpp +++ b/libraries/baking/src/TextureBaker.cpp @@ -120,10 +120,13 @@ void TextureBaker::processTexture() { const size_t length = memKTX->_storage->size(); // attempt to write the baked texture to the destination file path - QFile bakedTextureFile { _outputDirectory.absoluteFilePath(_bakedTextureFileName) }; + auto filePath = _outputDirectory.absoluteFilePath(_bakedTextureFileName); + QFile bakedTextureFile { filePath }; if (!bakedTextureFile.open(QIODevice::WriteOnly) || bakedTextureFile.write(data, length) == -1) { handleError("Could not write baked texture for " + _textureURL.toString()); + } else { + _outputFiles.push_back(filePath); } qCDebug(model_baking) << "Baked texture" << _textureURL; From 0755b77615ebdd3771e4a71ff977240f22d8c0b6 Mon Sep 17 00:00:00 2001 From: Atlante45 Date: Fri, 25 Aug 2017 16:31:45 -0700 Subject: [PATCH 023/129] Fix for mac build --- assignment-client/src/assets/AssetServer.cpp | 5 +++-- 1 file changed, 3 insertions(+), 2 deletions(-) diff --git a/assignment-client/src/assets/AssetServer.cpp b/assignment-client/src/assets/AssetServer.cpp index a3eab7d4d5..88a1eaeab1 100644 --- a/assignment-client/src/assets/AssetServer.cpp +++ b/assignment-client/src/assets/AssetServer.cpp @@ -79,8 +79,9 @@ void BakeAssetTask::run() { if (baker->hasErrors()) { qDebug() << "Failed to bake: " << _assetHash << _assetPath; } else { - qDebug() << "Finished baking: " << _assetHash << _assetPath << baker->getOutputFiles(); - emit bakeComplete(_assetHash, _assetPath, QVector::fromStdVector(baker->getOutputFiles())); + auto vectorOutputFiles = QVector::fromStdVector(baker->getOutputFiles()); + qDebug() << "Finished baking: " << _assetHash << _assetPath << vectorOutputFiles; + emit bakeComplete(_assetHash, _assetPath, vectorOutputFiles); } } From cc7dd2162bf1dca10901cdee8e300db6ddb97cbe Mon Sep 17 00:00:00 2001 From: Stephen Birarda Date: Fri, 25 Aug 2017 17:00:41 -0700 Subject: [PATCH 024/129] fixes for OS X build --- assignment-client/src/assets/AssetServer.cpp | 8 ++++++-- 1 file changed, 6 insertions(+), 2 deletions(-) diff --git a/assignment-client/src/assets/AssetServer.cpp b/assignment-client/src/assets/AssetServer.cpp index 88a1eaeab1..59e0809586 100644 --- a/assignment-client/src/assets/AssetServer.cpp +++ b/assignment-client/src/assets/AssetServer.cpp @@ -66,9 +66,13 @@ void BakeAssetTask::run() { std::unique_ptr baker; if (_assetPath.endsWith(".fbx")) { - baker = std::make_unique(QUrl("file:///" + _filePath), fn, PathUtils::generateTemporaryDir()); + baker = std::unique_ptr { + new FBXBaker(QUrl("file:///" + _filePath), fn, PathUtils::generateTemporaryDir()) + }; } else { - baker = std::make_unique(QUrl("file:///" + _filePath), image::TextureUsage::CUBE_TEXTURE, PathUtils::generateTemporaryDir()); + baker = std::unique_ptr { + new TextureBaker(QUrl("file:///" + _filePath), image::TextureUsage::CUBE_TEXTURE, PathUtils::generateTemporaryDir()) + }; } QEventLoop loop; From 79231cdee0905fca73c52f37330fb5419330072f Mon Sep 17 00:00:00 2001 From: Stephen Birarda Date: Fri, 25 Aug 2017 17:10:09 -0700 Subject: [PATCH 025/129] fix switched signature for completed bake handling --- assignment-client/src/assets/AssetServer.cpp | 2 +- assignment-client/src/assets/AssetServer.h | 2 +- 2 files changed, 2 insertions(+), 2 deletions(-) diff --git a/assignment-client/src/assets/AssetServer.cpp b/assignment-client/src/assets/AssetServer.cpp index 59e0809586..212b9d8d19 100644 --- a/assignment-client/src/assets/AssetServer.cpp +++ b/assignment-client/src/assets/AssetServer.cpp @@ -1049,7 +1049,7 @@ QString getBakeMapping(const AssetHash& hash, const QString& relativeFilePath) { return HIDDEN_BAKED_CONTENT_FOLDER + hash + "/" + relativeFilePath; } -void AssetServer::handleCompletedBake(QString originalAssetPath, QString originalAssetHash, QVector bakedFilePaths) { +void AssetServer::handleCompletedBake(QString originalAssetHash, QString originalAssetPath, QVector bakedFilePaths) { bool errorCompletingBake { false }; qDebug() << "Completing bake for " << originalAssetHash; diff --git a/assignment-client/src/assets/AssetServer.h b/assignment-client/src/assets/AssetServer.h index 19ad23be08..94576b329d 100644 --- a/assignment-client/src/assets/AssetServer.h +++ b/assignment-client/src/assets/AssetServer.h @@ -99,7 +99,7 @@ private: void bakeAsset(const AssetHash& assetHash, const AssetPath& assetPath, const QString& filePath); /// Move baked content for asset to baked directory and update baked status - void handleCompletedBake(QString assetPath, QString originalAssetHash, QVector bakedFilePaths); + void handleCompletedBake(QString originalAssetHash, QString assetPath, QVector bakedFilePaths); /// Create meta file to describe baked content for original asset bool createMetaFile(AssetHash originalAssetHash); From ba34a0ddef34269b072cc5421473233dccad238d Mon Sep 17 00:00:00 2001 From: Stephen Birarda Date: Mon, 28 Aug 2017 15:01:31 -0700 Subject: [PATCH 026/129] fail bake of partially baked FBX that references KTX --- libraries/baking/src/FBXBaker.cpp | 14 +++++++++----- 1 file changed, 9 insertions(+), 5 deletions(-) diff --git a/libraries/baking/src/FBXBaker.cpp b/libraries/baking/src/FBXBaker.cpp index 1a11071b5b..81b5de7546 100644 --- a/libraries/baking/src/FBXBaker.cpp +++ b/libraries/baking/src/FBXBaker.cpp @@ -105,11 +105,8 @@ void FBXBaker::bakeSourceCopy() { void FBXBaker::setupOutputFolder() { // make sure there isn't already an output directory using the same name - int iteration = 0; - if (QDir(_bakedOutputDir).exists()) { qWarning() << "Output path" << _bakedOutputDir << "already exists. Continuing."; - //_bakedOutputDir = _baseOutputPath + "/" + _fbxName + "-" + QString::number(++iteration) + "/"; } else { qCDebug(model_baking) << "Creating FBX output folder" << _bakedOutputDir; @@ -377,8 +374,15 @@ void FBXBaker::rewriteAndBakeSceneTextures() { QFileInfo textureFileInfo { fbxTextureFileName.replace("\\", "/") }; // make sure this texture points to something and isn't one we've already re-mapped - if (!textureFileInfo.filePath().isEmpty() - && textureFileInfo.suffix() != BAKED_TEXTURE_EXT.mid(1)) { + if (!textureFileInfo.filePath().isEmpty()) { + + if (textureFileInfo.suffix() == BAKED_TEXTURE_EXT.mid(1)) { + // re-baking an FBX that already references baked textures is a fail + // so we add an error and return from here + handleError("Cannot re-bake a partially baked FBX file that references baked KTX textures"); + + return; + } // construct the new baked texture file name and file path // ensuring that the baked texture will have a unique name From 6992bd6f48ab71145dd4c1b881682e50f8d30832 Mon Sep 17 00:00:00 2001 From: Stephen Birarda Date: Mon, 28 Aug 2017 15:38:31 -0700 Subject: [PATCH 027/129] add deletion of baked content when original removed --- assignment-client/src/assets/AssetServer.cpp | 31 +++++++++++++++----- assignment-client/src/assets/AssetServer.h | 3 ++ 2 files changed, 27 insertions(+), 7 deletions(-) diff --git a/assignment-client/src/assets/AssetServer.cpp b/assignment-client/src/assets/AssetServer.cpp index 212b9d8d19..753ca118cf 100644 --- a/assignment-client/src/assets/AssetServer.cpp +++ b/assignment-client/src/assets/AssetServer.cpp @@ -53,6 +53,8 @@ static const QList BAKEABLE_TEXTURE_EXTENSIONS = QImageReader::suppo static const QString BAKED_MODEL_SIMPLE_NAME = "asset.fbx"; static const QString BAKED_TEXTURE_SIMPLE_NAME = "texture.ktx"; +static const QString HIDDEN_BAKED_CONTENT_FOLDER = "/.baked/"; + BakeAssetTask::BakeAssetTask(const AssetHash& assetHash, const AssetPath& assetPath, const QString& filePath) : _assetHash(assetHash), _assetPath(assetPath), _filePath(filePath) { } @@ -115,7 +117,7 @@ BakingStatus AssetServer::getAssetStatus(const AssetPath& path, const AssetHash& return (*it)->isBaking() ? Baking : Pending; } - if (path.startsWith("/.baked/")) { + if (path.startsWith(HIDDEN_BAKED_CONTENT_FOLDER)) { return Baked; } @@ -136,7 +138,7 @@ BakingStatus AssetServer::getAssetStatus(const AssetPath& path, const AssetHash& return Unrelevant; } - auto bakedPath = "/.baked/" + hash + "/" + bakedFilename; + auto bakedPath = HIDDEN_BAKED_CONTENT_FOLDER + hash + "/" + bakedFilename; auto jt = _fileMappings.find(bakedPath); if (jt != _fileMappings.end()) { if (jt->toString() == hash) { @@ -179,14 +181,14 @@ void AssetServer::createEmptyMetaFile(const AssetHash& hash) { } bool AssetServer::hasMetaFile(const AssetHash& hash) { - QString metaFilePath = "/.baked/" + hash + "/meta.json"; + QString metaFilePath = HIDDEN_BAKED_CONTENT_FOLDER + hash + "/meta.json"; qDebug() << "in mappings?" << metaFilePath; return _fileMappings.contains(metaFilePath); } bool AssetServer::needsToBeBaked(const AssetPath& path, const AssetHash& assetHash) { - if (path.startsWith("/.baked/")) { + if (path.startsWith(HIDDEN_BAKED_CONTENT_FOLDER)) { return false; } @@ -207,7 +209,7 @@ bool AssetServer::needsToBeBaked(const AssetPath& path, const AssetHash& assetHa return false; } - auto bakedPath = "/.baked/" + assetHash + "/" + bakedFilename; + auto bakedPath = HIDDEN_BAKED_CONTENT_FOLDER + assetHash + "/" + bakedFilename; return !_fileMappings.contains(bakedPath); } @@ -392,6 +394,8 @@ void AssetServer::cleanupUnmappedFiles() { if (removeableFile.remove()) { qCDebug(asset_server) << "\tDeleted" << fileInfo.fileName() << "from asset files directory since it is unmapped."; + + removeBakedPathsForDeletedAsset(fileInfo.fileName()); } else { qCDebug(asset_server) << "\tAttempt to delete unmapped file" << fileInfo.fileName() << "failed"; } @@ -464,7 +468,7 @@ void AssetServer::handleGetMappingOperation(ReceivedMessage& message, SharedNode if (!bakedRootFile.isEmpty()) { // we ran into an asset for which we could have a baked version, let's check if it's ready - bakedAssetPath = "/.baked/" + originalAssetHash + "/" + bakedRootFile; + bakedAssetPath = HIDDEN_BAKED_CONTENT_FOLDER + originalAssetHash + "/" + bakedRootFile; auto bakedIt = _fileMappings.find(bakedAssetPath); if (bakedIt != _fileMappings.end()) { @@ -858,6 +862,18 @@ bool pathIsFolder(const AssetPath& path) { return path.endsWith('/'); } +void AssetServer::removeBakedPathsForDeletedAsset(AssetHash hash) { + // we deleted the file with this hash + + // check if we had baked content for that file that should also now be removed + // by calling deleteMappings for the hidden baked content folder for this hash + AssetPathList hiddenBakedFolder { HIDDEN_BAKED_CONTENT_FOLDER + hash + "/" }; + + qCDebug(asset_server) << "Deleting baked content below" << hiddenBakedFolder << "since" << hash << "was deleted"; + + deleteMappings(hiddenBakedFolder); +} + bool AssetServer::deleteMappings(AssetPathList& paths) { // take a copy of the current mappings in case persistence of these deletes fails auto oldMappings = _fileMappings; @@ -928,6 +944,8 @@ bool AssetServer::deleteMappings(AssetPathList& paths) { if (removeableFile.remove()) { qCDebug(asset_server) << "\tDeleted" << hash << "from asset files directory since it is now unmapped."; + + removeBakedPathsForDeletedAsset(hash); } else { qCDebug(asset_server) << "\tAttempt to delete unmapped file" << hash << "failed"; } @@ -1041,7 +1059,6 @@ bool AssetServer::renameMapping(AssetPath oldPath, AssetPath newPath) { } } -static const QString HIDDEN_BAKED_CONTENT_FOLDER = "/.baked/"; static const QString BAKED_ASSET_SIMPLE_FBX_NAME = "asset.fbx"; static const QString BAKED_ASSET_SIMPLE_TEXTURE_NAME = "texture.ktx"; diff --git a/assignment-client/src/assets/AssetServer.h b/assignment-client/src/assets/AssetServer.h index 94576b329d..cf193ba676 100644 --- a/assignment-client/src/assets/AssetServer.h +++ b/assignment-client/src/assets/AssetServer.h @@ -104,6 +104,9 @@ private: /// Create meta file to describe baked content for original asset bool createMetaFile(AssetHash originalAssetHash); + /// Remove baked paths when the original asset is deleteds + void removeBakedPathsForDeletedAsset(AssetHash originalAssetHash); + Mappings _fileMappings; QDir _resourcesDirectory; From e9258ec97a6f9718d424ffb075aea456e1ab101b Mon Sep 17 00:00:00 2001 From: Ryan Huffman Date: Mon, 28 Aug 2017 15:37:15 -0700 Subject: [PATCH 028/129] Fix loading standalone baked textures from Asset Server NetworkTexture was not properly handling redirected ATP files. For instance, if going from .jpg -> .ktx, the NetworkTexture class needs to be aware of this so it can stop the current request and make multiple requests for the individual mip levels. --- assignment-client/src/assets/AssetServer.cpp | 9 ++++--- libraries/baking/src/TextureBaker.cpp | 8 ++++++ libraries/baking/src/TextureBaker.h | 3 +++ .../src/model-networking/TextureCache.cpp | 21 +++++++++++++-- .../src/model-networking/TextureCache.h | 2 ++ .../networking/src/AssetResourceRequest.cpp | 27 ++++++++++++------- libraries/networking/src/ResourceCache.cpp | 3 ++- libraries/networking/src/ResourceCache.h | 3 ++- libraries/networking/src/ResourceRequest.h | 5 +++- 9 files changed, 63 insertions(+), 18 deletions(-) diff --git a/assignment-client/src/assets/AssetServer.cpp b/assignment-client/src/assets/AssetServer.cpp index 212b9d8d19..ce9e49326d 100644 --- a/assignment-client/src/assets/AssetServer.cpp +++ b/assignment-client/src/assets/AssetServer.cpp @@ -49,7 +49,7 @@ static const int INTERFACE_RUNNING_CHECK_FREQUENCY_MS = 1000; const QString ASSET_SERVER_LOGGING_TARGET_NAME = "asset-server"; static const QStringList BAKEABLE_MODEL_EXTENSIONS = { "fbx" }; -static const QList BAKEABLE_TEXTURE_EXTENSIONS = QImageReader::supportedImageFormats(); +static QStringList BAKEABLE_TEXTURE_EXTENSIONS; static const QString BAKED_MODEL_SIMPLE_NAME = "asset.fbx"; static const QString BAKED_TEXTURE_SIMPLE_NAME = "texture.ktx"; @@ -71,7 +71,8 @@ void BakeAssetTask::run() { }; } else { baker = std::unique_ptr { - new TextureBaker(QUrl("file:///" + _filePath), image::TextureUsage::CUBE_TEXTURE, PathUtils::generateTemporaryDir()) + new TextureBaker(QUrl("file:///" + _filePath), image::TextureUsage::CUBE_TEXTURE, + PathUtils::generateTemporaryDir()) }; } @@ -198,7 +199,7 @@ bool AssetServer::needsToBeBaked(const AssetPath& path, const AssetHash& assetHa auto extension = path.mid(dotIndex + 1); QString bakedFilename; - + if (BAKEABLE_MODEL_EXTENSIONS.contains(extension)) { bakedFilename = BAKED_MODEL_SIMPLE_NAME; } else if (BAKEABLE_TEXTURE_EXTENSIONS.contains(extension.toLocal8Bit()) && hasMetaFile(assetHash)) { @@ -247,6 +248,8 @@ AssetServer::AssetServer(ReceivedMessage& message) : _transferTaskPool(this), _bakingTaskPool(this) { + BAKEABLE_TEXTURE_EXTENSIONS = TextureBaker::getSupportedFormats(); + qDebug() << "Supported baking texture formats:" << BAKEABLE_MODEL_EXTENSIONS; // Most of the work will be I/O bound, reading from disk and constructing packet objects, // so the ideal is greater than the number of cores on the system. diff --git a/libraries/baking/src/TextureBaker.cpp b/libraries/baking/src/TextureBaker.cpp index 44995fa026..f80af66aa2 100644 --- a/libraries/baking/src/TextureBaker.cpp +++ b/libraries/baking/src/TextureBaker.cpp @@ -43,6 +43,14 @@ void TextureBaker::bake() { loadTexture(); } +const QStringList TextureBaker::getSupportedFormats() { + auto formats = QImageReader::supportedImageFormats(); + QStringList stringFormats; + std::transform(formats.begin(), formats.end(), std::back_inserter(stringFormats), + [](auto& format) -> QString { return format; }); + return stringFormats; +} + void TextureBaker::loadTexture() { // check if the texture is local or first needs to be downloaded if (_textureURL.isLocalFile()) { diff --git a/libraries/baking/src/TextureBaker.h b/libraries/baking/src/TextureBaker.h index ee1e968f20..76d0a69823 100644 --- a/libraries/baking/src/TextureBaker.h +++ b/libraries/baking/src/TextureBaker.h @@ -15,6 +15,7 @@ #include #include #include +#include #include @@ -28,6 +29,8 @@ class TextureBaker : public Baker { public: TextureBaker(const QUrl& textureURL, image::TextureUsage::Type textureType, const QDir& outputDirectory); + static const QStringList getSupportedFormats(); + const QByteArray& getOriginalTexture() const { return _originalTexture; } QUrl getTextureURL() const { return _textureURL; } diff --git a/libraries/model-networking/src/model-networking/TextureCache.cpp b/libraries/model-networking/src/model-networking/TextureCache.cpp index 5b43a4090f..cc29e841ce 100644 --- a/libraries/model-networking/src/model-networking/TextureCache.cpp +++ b/libraries/model-networking/src/model-networking/TextureCache.cpp @@ -299,6 +299,8 @@ NetworkTexture::NetworkTexture(const QUrl& url, image::TextureUsage::Type type, _textureSource = std::make_shared(); _lowestRequestedMipLevel = 0; + _shouldFailOnRedirect = !_sourceIsKTX; + if (type == image::TextureUsage::CUBE_TEXTURE) { setLoadPriority(this, SKYBOX_LOAD_PRIORITY); } else if (_sourceIsKTX) { @@ -428,6 +430,21 @@ void NetworkTexture::makeRequest() { } +bool NetworkTexture::handleFailedRequest(ResourceRequest::Result result) { + if (!_sourceIsKTX && result == ResourceRequest::Result::RedirectFail) { + auto newPath = _request->getRelativePathUrl(); + if (newPath.fileName().endsWith(".ktx")) { + qDebug() << "Redirecting to" << newPath << "from" << _url; + _sourceIsKTX = true; + _activeUrl = newPath; + _shouldFailOnRedirect = false; + makeRequest(); + return true; + } + } + return Resource::handleFailedRequest(result); +} + void NetworkTexture::startRequestForNextMipLevel() { auto self = _self.lock(); if (!self) { @@ -527,7 +544,7 @@ void NetworkTexture::ktxInitialDataRequestFinished() { _ktxHighMipData = _ktxMipRequest->getData(); handleFinishedInitialLoad(); } else { - if (handleFailedRequest(result)) { + if (Resource::handleFailedRequest(result)) { _ktxResourceState = PENDING_INITIAL_LOAD; } else { _ktxResourceState = FAILED_TO_LOAD; @@ -616,7 +633,7 @@ void NetworkTexture::ktxMipRequestFinished() { finishedLoading(false); } } else { - if (handleFailedRequest(result)) { + if (Resource::handleFailedRequest(result)) { _ktxResourceState = PENDING_MIP_REQUEST; } else { _ktxResourceState = FAILED_TO_LOAD; diff --git a/libraries/model-networking/src/model-networking/TextureCache.h b/libraries/model-networking/src/model-networking/TextureCache.h index f5a0ec5215..5bc5aa7d96 100644 --- a/libraries/model-networking/src/model-networking/TextureCache.h +++ b/libraries/model-networking/src/model-networking/TextureCache.h @@ -75,6 +75,8 @@ protected: virtual void downloadFinished(const QByteArray& data) override; + bool handleFailedRequest(ResourceRequest::Result result) override; + Q_INVOKABLE void loadContent(const QByteArray& content); Q_INVOKABLE void setImage(gpu::TexturePointer texture, int originalWidth, int originalHeight); diff --git a/libraries/networking/src/AssetResourceRequest.cpp b/libraries/networking/src/AssetResourceRequest.cpp index 01fe971125..c2e0bc1215 100644 --- a/libraries/networking/src/AssetResourceRequest.cpp +++ b/libraries/networking/src/AssetResourceRequest.cpp @@ -82,19 +82,23 @@ void AssetResourceRequest::requestMappingForPath(const AssetPath& path) { Q_ASSERT(_state == InProgress); Q_ASSERT(request == _assetMappingRequest); + bool failed = false; + switch (request->getError()) { case MappingRequest::NoError: // we have no error, we should have a resulting hash - use that to send of a request for that asset qCDebug(networking) << "Got mapping for:" << path << "=>" << request->getHash(); - requestHash(request->getHash()); - statTracker->incrementStat(STAT_ATP_MAPPING_REQUEST_SUCCESS); // if we got a redirected path we need to store that with the resource request as relative path URL - if (request->wasRedirected()) { - qDebug() << "Request was redirected"; + if (request->wasRedirected() && _failOnRedirect) { _relativePathURL = ATP_SCHEME + request->getRedirectedPath(); + _result = RedirectFail; + + failed = true; + } else { + requestHash(request->getHash()); } break; @@ -113,17 +117,20 @@ void AssetResourceRequest::requestMappingForPath(const AssetPath& path) { break; } - // since we've failed we know we are finished - _state = Finished; - emit finished(); - - statTracker->incrementStat(STAT_ATP_MAPPING_REQUEST_FAILED); - statTracker->incrementStat(STAT_ATP_REQUEST_FAILED); + failed = true; break; } } + if (failed) { + _state = Finished; + emit finished(); + + statTracker->incrementStat(STAT_ATP_MAPPING_REQUEST_FAILED); + statTracker->incrementStat(STAT_ATP_REQUEST_FAILED); + } + _assetMappingRequest->deleteLater(); _assetMappingRequest = nullptr; }); diff --git a/libraries/networking/src/ResourceCache.cpp b/libraries/networking/src/ResourceCache.cpp index b7ab1269f9..fbe7c05801 100644 --- a/libraries/networking/src/ResourceCache.cpp +++ b/libraries/networking/src/ResourceCache.cpp @@ -687,8 +687,9 @@ void Resource::makeRequest() { PROFILE_ASYNC_END(resource, "Resource:" + getType(), QString::number(_requestID)); return; } - + _request->setByteRange(_requestByteRange); + _request->setFailOnRedirect(_shouldFailOnRedirect); qCDebug(resourceLog).noquote() << "Starting request for:" << _url.toDisplayString(); emit loading(); diff --git a/libraries/networking/src/ResourceCache.h b/libraries/networking/src/ResourceCache.h index 40df6418a5..17531f45b0 100644 --- a/libraries/networking/src/ResourceCache.h +++ b/libraries/networking/src/ResourceCache.h @@ -449,12 +449,13 @@ protected: Q_INVOKABLE void allReferencesCleared(); /// Return true if the resource will be retried - bool handleFailedRequest(ResourceRequest::Result result); + virtual bool handleFailedRequest(ResourceRequest::Result result); QUrl _url; QUrl _effectiveBaseURL{ _url }; QUrl _activeUrl; ByteRange _requestByteRange; + bool _shouldFailOnRedirect { false }; // _loaded == true means we are in a loaded and usable state. It is possible that there may still be // active requests/loading while in this state. Example: Progressive KTX downloads, where higher resolution diff --git a/libraries/networking/src/ResourceRequest.h b/libraries/networking/src/ResourceRequest.h index 0a4dc12293..cf852c1e1b 100644 --- a/libraries/networking/src/ResourceRequest.h +++ b/libraries/networking/src/ResourceRequest.h @@ -57,7 +57,8 @@ public: AccessDenied, InvalidByteRange, InvalidURL, - NotFound + NotFound, + RedirectFail }; Q_ENUM(Result) @@ -70,6 +71,7 @@ public: bool loadedFromCache() const { return _loadedFromCache; } bool getRangeRequestSuccessful() const { return _rangeRequestSuccessful; } bool getTotalSizeOfResource() const { return _totalSizeOfResource; } + void setFailOnRedirect(bool failOnRedirect) { _failOnRedirect = failOnRedirect; } void setCacheEnabled(bool value) { _cacheEnabled = value; } void setByteRange(ByteRange byteRange) { _byteRange = byteRange; } @@ -89,6 +91,7 @@ protected: State _state { NotStarted }; Result _result; QByteArray _data; + bool _failOnRedirect { false }; bool _cacheEnabled { true }; bool _loadedFromCache { false }; ByteRange _byteRange; From 31345e4264a85bf7e297dd2e91b0f19145762557 Mon Sep 17 00:00:00 2001 From: Ryan Huffman Date: Wed, 30 Aug 2017 13:15:14 -0700 Subject: [PATCH 029/129] Fix ATP requests not keeping track of all mapping redirects --- libraries/networking/src/AssetResourceRequest.cpp | 6 ++++-- 1 file changed, 4 insertions(+), 2 deletions(-) diff --git a/libraries/networking/src/AssetResourceRequest.cpp b/libraries/networking/src/AssetResourceRequest.cpp index c2e0bc1215..c9ca6ebd43 100644 --- a/libraries/networking/src/AssetResourceRequest.cpp +++ b/libraries/networking/src/AssetResourceRequest.cpp @@ -92,10 +92,12 @@ void AssetResourceRequest::requestMappingForPath(const AssetPath& path) { statTracker->incrementStat(STAT_ATP_MAPPING_REQUEST_SUCCESS); // if we got a redirected path we need to store that with the resource request as relative path URL - if (request->wasRedirected() && _failOnRedirect) { + if (request->wasRedirected()) { _relativePathURL = ATP_SCHEME + request->getRedirectedPath(); - _result = RedirectFail; + } + if (request->wasRedirected() && _failOnRedirect) { + _result = RedirectFail; failed = true; } else { requestHash(request->getHash()); From f2997c0997d137d85b958208cc7377dd072c31ed Mon Sep 17 00:00:00 2001 From: Stephen Birarda Date: Tue, 29 Aug 2017 16:40:43 -0700 Subject: [PATCH 030/129] refuse to perform mapping operations in /.baked/ --- assignment-client/src/assets/AssetServer.cpp | 35 ++++++++++++++++---- 1 file changed, 28 insertions(+), 7 deletions(-) diff --git a/assignment-client/src/assets/AssetServer.cpp b/assignment-client/src/assets/AssetServer.cpp index 10aadc33cc..f7a8ecf453 100644 --- a/assignment-client/src/assets/AssetServer.cpp +++ b/assignment-client/src/assets/AssetServer.cpp @@ -533,11 +533,18 @@ void AssetServer::handleSetMappingOperation(ReceivedMessage& message, SharedNode auto assetHash = message.read(SHA256_HASH_LENGTH).toHex(); - if (setMapping(assetPath, assetHash)) { - replyPacket.writePrimitive(AssetServerError::NoError); + // don't process a set mapping operation that is inside the hidden baked folder + if (assetPath.startsWith(HIDDEN_BAKED_CONTENT_FOLDER)) { + qCDebug(asset_server) << "Refusing to process a set mapping operation inside" << HIDDEN_BAKED_CONTENT_FOLDER; + replyPacket.writePrimitive(AssetServerError::PermissionDenied); } else { - replyPacket.writePrimitive(AssetServerError::MappingOperationFailed); + if (setMapping(assetPath, assetHash)) { + replyPacket.writePrimitive(AssetServerError::NoError); + } else { + replyPacket.writePrimitive(AssetServerError::MappingOperationFailed); + } } + } else { replyPacket.writePrimitive(AssetServerError::PermissionDenied); } @@ -551,7 +558,14 @@ void AssetServer::handleDeleteMappingsOperation(ReceivedMessage& message, Shared QStringList mappingsToDelete; for (int i = 0; i < numberOfDeletedMappings; ++i) { - mappingsToDelete << message.readString(); + auto mapping = message.readString(); + + if (!mapping.startsWith(HIDDEN_BAKED_CONTENT_FOLDER)) { + mappingsToDelete << mapping; + } else { + qCDebug(asset_server) << "Refusing to delete mapping" << mapping + << "that is inside" << HIDDEN_BAKED_CONTENT_FOLDER; + } } if (deleteMappings(mappingsToDelete)) { @@ -569,11 +583,18 @@ void AssetServer::handleRenameMappingOperation(ReceivedMessage& message, SharedN QString oldPath = message.readString(); QString newPath = message.readString(); - if (renameMapping(oldPath, newPath)) { - replyPacket.writePrimitive(AssetServerError::NoError); + if (oldPath.startsWith(HIDDEN_BAKED_CONTENT_FOLDER) || newPath.startsWith(HIDDEN_BAKED_CONTENT_FOLDER)) { + qCDebug(asset_server) << "Cannot rename" << oldPath << "to" << newPath + << "since one of the paths is inside" << HIDDEN_BAKED_CONTENT_FOLDER; + replyPacket.writePrimitive(AssetServerError::PermissionDenied); } else { - replyPacket.writePrimitive(AssetServerError::MappingOperationFailed); + if (renameMapping(oldPath, newPath)) { + replyPacket.writePrimitive(AssetServerError::NoError); + } else { + replyPacket.writePrimitive(AssetServerError::MappingOperationFailed); + } } + } else { replyPacket.writePrimitive(AssetServerError::PermissionDenied); } From 3abca67f1a48426162e3c56e5e29743d4bea0522 Mon Sep 17 00:00:00 2001 From: Stephen Birarda Date: Tue, 29 Aug 2017 16:41:02 -0700 Subject: [PATCH 031/129] remove use of auto in TextureBaker for OS X compile --- libraries/baking/src/TextureBaker.cpp | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/libraries/baking/src/TextureBaker.cpp b/libraries/baking/src/TextureBaker.cpp index f80af66aa2..548e3921e4 100644 --- a/libraries/baking/src/TextureBaker.cpp +++ b/libraries/baking/src/TextureBaker.cpp @@ -47,7 +47,7 @@ const QStringList TextureBaker::getSupportedFormats() { auto formats = QImageReader::supportedImageFormats(); QStringList stringFormats; std::transform(formats.begin(), formats.end(), std::back_inserter(stringFormats), - [](auto& format) -> QString { return format; }); + [](QByteArray& format) -> QString { return format; }); return stringFormats; } From 70430c009d2ac68a7e6c2a448c57b547f4f1cb92 Mon Sep 17 00:00:00 2001 From: Ryan Huffman Date: Wed, 30 Aug 2017 13:15:14 -0700 Subject: [PATCH 032/129] Fix ATP requests not keeping track of all mapping redirects --- libraries/networking/src/AssetResourceRequest.cpp | 6 ++++-- 1 file changed, 4 insertions(+), 2 deletions(-) diff --git a/libraries/networking/src/AssetResourceRequest.cpp b/libraries/networking/src/AssetResourceRequest.cpp index c2e0bc1215..c9ca6ebd43 100644 --- a/libraries/networking/src/AssetResourceRequest.cpp +++ b/libraries/networking/src/AssetResourceRequest.cpp @@ -92,10 +92,12 @@ void AssetResourceRequest::requestMappingForPath(const AssetPath& path) { statTracker->incrementStat(STAT_ATP_MAPPING_REQUEST_SUCCESS); // if we got a redirected path we need to store that with the resource request as relative path URL - if (request->wasRedirected() && _failOnRedirect) { + if (request->wasRedirected()) { _relativePathURL = ATP_SCHEME + request->getRedirectedPath(); - _result = RedirectFail; + } + if (request->wasRedirected() && _failOnRedirect) { + _result = RedirectFail; failed = true; } else { requestHash(request->getHash()); From ed21ad11d27f60eacb8f94b34addca4031259ef5 Mon Sep 17 00:00:00 2001 From: Stephen Birarda Date: Wed, 30 Aug 2017 15:14:53 -0700 Subject: [PATCH 033/129] add baking to default build, add default to baking status --- libraries/baking/CMakeLists.txt | 2 -- libraries/networking/src/AssetUtils.cpp | 5 +++-- 2 files changed, 3 insertions(+), 4 deletions(-) diff --git a/libraries/baking/CMakeLists.txt b/libraries/baking/CMakeLists.txt index 44c52ae0d8..ee2d861539 100644 --- a/libraries/baking/CMakeLists.txt +++ b/libraries/baking/CMakeLists.txt @@ -11,7 +11,5 @@ if (FBX_FOUND) target_include_directories(${TARGET_NAME} SYSTEM PRIVATE ${FBX_INCLUDE_DIR}) endif () -set_target_properties(${TARGET_NAME} PROPERTIES EXCLUDE_FROM_ALL TRUE EXCLUDE_FROM_DEFAULT_BUILD TRUE) - link_hifi_libraries(shared model networking ktx image) include_hifi_library_headers(gpu) diff --git a/libraries/networking/src/AssetUtils.cpp b/libraries/networking/src/AssetUtils.cpp index af085d6463..df3aa7ad0b 100644 --- a/libraries/networking/src/AssetUtils.cpp +++ b/libraries/networking/src/AssetUtils.cpp @@ -87,8 +87,6 @@ bool isValidHash(const AssetHash& hash) { QString bakingStatusToString(BakingStatus status) { switch (status) { - case Unrelevant: - return "--"; case NotBaked: return "Not Baked"; case Pending: @@ -99,5 +97,8 @@ QString bakingStatusToString(BakingStatus status) { return "Baked"; case Error: return "Error"; + default: + case Unrelevant: + return "--"; } } From 81fdc27ca871afacbff9e60c5b9f2c4ac5f611b8 Mon Sep 17 00:00:00 2001 From: Stephen Birarda Date: Wed, 30 Aug 2017 15:34:20 -0700 Subject: [PATCH 034/129] use 2017.1 FBX SDK on all platforms, change linux location --- cmake/modules/FindFBX.cmake | 10 +++------- 1 file changed, 3 insertions(+), 7 deletions(-) diff --git a/cmake/modules/FindFBX.cmake b/cmake/modules/FindFBX.cmake index 6eb6d11f9d..3963f4d2e6 100644 --- a/cmake/modules/FindFBX.cmake +++ b/cmake/modules/FindFBX.cmake @@ -17,11 +17,7 @@ # which uses the MIT license (https://github.com/ufz-vislab/VtkFbxConverter/blob/master/LICENSE.txt) if (NOT FBX_VERSION) - if (WIN32) - set(FBX_VERSION 2017.1) - else() - set(FBX_VERSION 2017.0.1) - endif() + set(FBX_VERSION 2017.1) endif() string(REGEX REPLACE "^([0-9]+).*$" "\\1" FBX_VERSION_MAJOR "${FBX_VERSION}") @@ -29,7 +25,7 @@ string(REGEX REPLACE "^[0-9]+\\.([0-9]+).*$" "\\1" FBX_VERSION_MINOR "${FBX_VER string(REGEX REPLACE "^[0-9]+\\.[0-9]+\\.([0-9]+).*$" "\\1" FBX_VERSION_PATCH "${FBX_VERSION}") set(FBX_MAC_LOCATIONS "/Applications/Autodesk/FBX\ SDK/${FBX_VERSION}") -set(FBX_LINUX_LOCATIONS "/usr/local/lib/gcc4/x64/debug/") +set(FBX_LINUX_LOCATIONS "/usr/local/fbxsdk") if (WIN32) string(REGEX REPLACE "\\\\" "/" WIN_PROGRAM_FILES_X64_DIRECTORY $ENV{ProgramW6432}) @@ -73,7 +69,7 @@ function(_fbx_find_library _name _lib _suffix) find_library(${_name} NAMES ${_lib} HINTS ${FBX_SEARCH_LOCATIONS} - PATH_SUFFIXES lib/${fbx_compiler}/${_suffix} lib/${fbx_compiler}/ub/${_suffix} lib/${VS_PREFIX}/x64/${_suffix} + PATH_SUFFIXES lib/${fbx_compiler}/${_suffix} lib/${fbx_compiler}/x64/${_suffix} lib/${fbx_compiler}/ub/${_suffix} lib/${VS_PREFIX}/x64/${_suffix} ) mark_as_advanced(${_name}) From 4e1e483e27a9f87b83f5bdaf989f1cf773dcaf30 Mon Sep 17 00:00:00 2001 From: Stephen Birarda Date: Wed, 30 Aug 2017 16:50:20 -0700 Subject: [PATCH 035/129] link DL libs with FBX SDK for baking --- libraries/baking/CMakeLists.txt | 4 ++++ 1 file changed, 4 insertions(+) diff --git a/libraries/baking/CMakeLists.txt b/libraries/baking/CMakeLists.txt index ee2d861539..806220ec30 100644 --- a/libraries/baking/CMakeLists.txt +++ b/libraries/baking/CMakeLists.txt @@ -9,6 +9,10 @@ if (FBX_FOUND) target_link_libraries(${TARGET_NAME} ${FBX_LIBRARIES}) endif () target_include_directories(${TARGET_NAME} SYSTEM PRIVATE ${FBX_INCLUDE_DIR}) + + if (UNIX) + target_link_libraries(${TARGET_NAME} ${CMAKE_DL_LIBS}) + endif (UNIX) endif () link_hifi_libraries(shared model networking ktx image) From d68338f0b02f19dd747e954413ac8d4957cbd079 Mon Sep 17 00:00:00 2001 From: Ryan Huffman Date: Wed, 30 Aug 2017 16:53:31 -0700 Subject: [PATCH 036/129] Add error handling to asset server baking --- assignment-client/src/assets/AssetServer.cpp | 251 +++++++++++++------ assignment-client/src/assets/AssetServer.h | 24 +- 2 files changed, 192 insertions(+), 83 deletions(-) diff --git a/assignment-client/src/assets/AssetServer.cpp b/assignment-client/src/assets/AssetServer.cpp index ce9e49326d..b91c8795c2 100644 --- a/assignment-client/src/assets/AssetServer.cpp +++ b/assignment-client/src/assets/AssetServer.cpp @@ -83,6 +83,7 @@ void BakeAssetTask::run() { if (baker->hasErrors()) { qDebug() << "Failed to bake: " << _assetHash << _assetPath; + emit bakeFailed(_assetHash, _assetPath); } else { auto vectorOutputFiles = QVector::fromStdVector(baker->getOutputFiles()); qDebug() << "Finished baking: " << _assetHash << _assetPath << vectorOutputFiles; @@ -99,6 +100,7 @@ void AssetServer::bakeAsset(const AssetHash& assetHash, const AssetPath& assetPa _pendingBakes[assetHash] = task; connect(task.get(), &BakeAssetTask::bakeComplete, this, &AssetServer::handleCompletedBake); + connect(task.get(), &BakeAssetTask::bakeFailed, this, &AssetServer::handleFailedBake); _bakingTaskPool.start(task.get()); } else { @@ -140,7 +142,7 @@ BakingStatus AssetServer::getAssetStatus(const AssetPath& path, const AssetHash& auto bakedPath = "/.baked/" + hash + "/" + bakedFilename; auto jt = _fileMappings.find(bakedPath); if (jt != _fileMappings.end()) { - if (jt->toString() == hash) { + if (jt->first == hash) { return NotBaked; } else { return Baked; @@ -153,8 +155,8 @@ BakingStatus AssetServer::getAssetStatus(const AssetPath& path, const AssetHash& void AssetServer::bakeAssets() { auto it = _fileMappings.cbegin(); for (; it != _fileMappings.cend(); ++it) { - auto path = it.key(); - auto hash = it.value().toString(); + auto path = it->first; + auto hash = it->second; maybeBake(path, hash); } } @@ -181,9 +183,8 @@ void AssetServer::createEmptyMetaFile(const AssetHash& hash) { bool AssetServer::hasMetaFile(const AssetHash& hash) { QString metaFilePath = "/.baked/" + hash + "/meta.json"; - qDebug() << "in mappings?" << metaFilePath; - return _fileMappings.contains(metaFilePath); + return _fileMappings.find(metaFilePath) != _fileMappings.end(); } bool AssetServer::needsToBeBaked(const AssetPath& path, const AssetHash& assetHash) { @@ -200,16 +201,25 @@ bool AssetServer::needsToBeBaked(const AssetPath& path, const AssetHash& assetHa QString bakedFilename; + bool loaded; + AssetMeta meta; + std::tie(loaded, meta) = readMetaFile(assetHash); + + // TODO: Allow failed bakes that happened on old versions to be re-baked + if (loaded && meta.failedLastBake) { + return false; + } + if (BAKEABLE_MODEL_EXTENSIONS.contains(extension)) { bakedFilename = BAKED_MODEL_SIMPLE_NAME; - } else if (BAKEABLE_TEXTURE_EXTENSIONS.contains(extension.toLocal8Bit()) && hasMetaFile(assetHash)) { + } else if (loaded && BAKEABLE_TEXTURE_EXTENSIONS.contains(extension.toLocal8Bit())) { bakedFilename = BAKED_TEXTURE_SIMPLE_NAME; } else { return false; } auto bakedPath = "/.baked/" + assetHash + "/" + bakedFilename; - return !_fileMappings.contains(bakedPath); + return _fileMappings.find(bakedPath) == _fileMappings.end(); } bool interfaceRunning() { @@ -364,7 +374,7 @@ void AssetServer::completeSetup() { qCInfo(asset_server) << "There are" << hashedFiles.size() << "asset files in the asset directory."; - if (_fileMappings.count() > 0) { + if (_fileMappings.size() > 0) { cleanupUnmappedFiles(); } @@ -382,21 +392,26 @@ void AssetServer::cleanupUnmappedFiles() { auto files = _filesDirectory.entryInfoList(QDir::Files); - // grab the currently mapped hashes - auto mappedHashes = _fileMappings.values(); - qCInfo(asset_server) << "Performing unmapped asset cleanup."; for (const auto& fileInfo : files) { - if (hashFileRegex.exactMatch(fileInfo.fileName())) { - if (!mappedHashes.contains(fileInfo.fileName())) { + auto filename = fileInfo.fileName(); + if (hashFileRegex.exactMatch(filename)) { + bool matched { false }; + for (auto& pair : _fileMappings) { + if (pair.second == filename) { + matched = true; + break; + } + } + if (!matched) { // remove the unmapped file QFile removeableFile { fileInfo.absoluteFilePath() }; if (removeableFile.remove()) { - qCDebug(asset_server) << "\tDeleted" << fileInfo.fileName() << "from asset files directory since it is unmapped."; + qCDebug(asset_server) << "\tDeleted" << filename << "from asset files directory since it is unmapped."; } else { - qCDebug(asset_server) << "\tAttempt to delete unmapped file" << fileInfo.fileName() << "failed"; + qCDebug(asset_server) << "\tAttempt to delete unmapped file" << filename << "failed"; } } } @@ -458,9 +473,8 @@ void AssetServer::handleGetMappingOperation(ReceivedMessage& message, SharedNode } else if (BAKEABLE_TEXTURE_EXTENSIONS.contains(assetPathExtension.toLocal8Bit())) { bakedRootFile = BAKED_TEXTURE_SIMPLE_NAME; } - qDebug() << bakedRootFile << assetPathExtension; - auto originalAssetHash = it->toString(); + auto originalAssetHash = it->second; QString redirectedAssetHash; QString bakedAssetPath; quint8 wasRedirected = false; @@ -473,7 +487,7 @@ void AssetServer::handleGetMappingOperation(ReceivedMessage& message, SharedNode if (bakedIt != _fileMappings.end()) { qDebug() << "Did find baked version for: " << originalAssetHash << assetPath; // we found a baked version of the requested asset to serve, redirect to that - redirectedAssetHash = bakedIt->toString(); + redirectedAssetHash = bakedIt->second; wasRedirected = true; } else { qDebug() << "Did not find baked version for: " << originalAssetHash << assetPath; @@ -498,9 +512,8 @@ void AssetServer::handleGetMappingOperation(ReceivedMessage& message, SharedNode auto query = QUrlQuery(url.query()); bool isSkybox = query.hasQueryItem("skybox"); - qDebug() << "Is skybox? " << isSkybox; if (isSkybox) { - createMetaFile(originalAssetHash); + writeMetaFile(originalAssetHash); maybeBake(originalAssetHash, assetPath); } } @@ -512,14 +525,14 @@ void AssetServer::handleGetMappingOperation(ReceivedMessage& message, SharedNode void AssetServer::handleGetAllMappingOperation(ReceivedMessage& message, SharedNodePointer senderNode, NLPacketList& replyPacket) { replyPacket.writePrimitive(AssetServerError::NoError); - auto count = _fileMappings.size(); + uint32_t count = _fileMappings.size(); replyPacket.writePrimitive(count); - for (auto it = _fileMappings.cbegin(); it != _fileMappings.cend(); ++ it) { - replyPacket.writeString(it.key()); - replyPacket.write(QByteArray::fromHex(it.value().toString().toUtf8())); - replyPacket.writePrimitive(getAssetStatus(it.key(), it.value().toString())); + for (auto it = _fileMappings.cbegin(); it != _fileMappings.cend(); ++it) { + replyPacket.writeString(it->first); + replyPacket.write(QByteArray::fromHex(it->second.toUtf8())); + replyPacket.writePrimitive(getAssetStatus(it->first, it->second)); } } @@ -759,31 +772,38 @@ bool AssetServer::loadMappingsFromFile() { auto jsonDocument = QJsonDocument::fromJson(mapFile.readAll(), &error); if (error.error == QJsonParseError::NoError) { - _fileMappings = jsonDocument.object().toVariantHash(); - - // remove any mappings that don't match the expected format - auto it = _fileMappings.begin(); - while (it != _fileMappings.end()) { - bool shouldDrop = false; - - if (!isValidFilePath(it.key())) { - qCWarning(asset_server) << "Will not keep mapping for" << it.key() << "since it is not a valid path."; - shouldDrop = true; - } - - if (!isValidHash(it.value().toString())) { - qCWarning(asset_server) << "Will not keep mapping for" << it.key() << "since it does not have a valid hash."; - shouldDrop = true; - } - - if (shouldDrop) { - it = _fileMappings.erase(it); - } else { - ++it; - } + if (!jsonDocument.isObject()) { + qCWarning(asset_server) << "Failed to read mapping file, root value in" << mapFilePath << "is not an object"; + return false; } - qCInfo(asset_server) << "Loaded" << _fileMappings.count() << "mappings from map file at" << mapFilePath; + //_fileMappings = jsonDocument.object().toVariantHash(); + auto root = jsonDocument.object(); + for (auto it = root.begin(); it != root.end(); ++it) { + auto key = it.key(); + auto value = it.value(); + + if (!value.isString()) { + qCWarning(asset_server) << "Skipping" << key << ":" << value << "because it is not a string"; + continue; + } + + if (!isValidFilePath(key)) { + qCWarning(asset_server) << "Will not keep mapping for" << key << "since it is not a valid path."; + continue; + } + + if (!isValidHash(value.toString())) { + qCWarning(asset_server) << "Will not keep mapping for" << key << "since it does not have a valid hash."; + continue; + } + + + qDebug() << "Added " << key << value.toString(); + _fileMappings[key] = value.toString(); + } + + qCInfo(asset_server) << "Loaded" << _fileMappings.size() << "mappings from map file at" << mapFilePath; return true; } } @@ -802,8 +822,13 @@ bool AssetServer::writeMappingsToFile() { QFile mapFile { mapFilePath }; if (mapFile.open(QIODevice::WriteOnly)) { - auto jsonObject = QJsonObject::fromVariantHash(_fileMappings); - QJsonDocument jsonDocument { jsonObject }; + QJsonObject root; + + for (auto it : _fileMappings) { + root[it.first] = it.second; + } + + QJsonDocument jsonDocument { root }; if (mapFile.write(jsonDocument.toJson()) != -1) { qCDebug(asset_server) << "Wrote JSON mappings to file at" << mapFilePath; @@ -832,7 +857,8 @@ bool AssetServer::setMapping(AssetPath path, AssetHash hash) { } // remember what the old mapping was in case persistence fails - auto oldMapping = _fileMappings.value(path).toString(); + auto it = _fileMappings.find(path); + auto oldMapping = it != _fileMappings.end() ? it->second : ""; // update the in memory QHash _fileMappings[path] = hash; @@ -846,7 +872,7 @@ bool AssetServer::setMapping(AssetPath path, AssetHash hash) { } else { // failed to persist this mapping to file - put back the old one in our in-memory representation if (oldMapping.isEmpty()) { - _fileMappings.remove(path); + _fileMappings.erase(_fileMappings.find(path)); } else { _fileMappings[path] = oldMapping; } @@ -879,9 +905,9 @@ bool AssetServer::deleteMappings(AssetPathList& paths) { auto sizeBefore = _fileMappings.size(); while (it != _fileMappings.end()) { - if (it.key().startsWith(path)) { + if (it->first.startsWith(path)) { // add this hash to the list we need to check for asset removal from the server - hashesToCheckForDeletion << it.value().toString(); + hashesToCheckForDeletion << it->second; it = _fileMappings.erase(it); } else { @@ -897,12 +923,14 @@ bool AssetServer::deleteMappings(AssetPathList& paths) { } } else { - auto oldMapping = _fileMappings.take(path); - if (!oldMapping.isNull()) { - // add this hash to the list we need to check for asset removal from server - hashesToCheckForDeletion << oldMapping.toString(); + auto it = _fileMappings.find(path); + if (it != _fileMappings.end()) { + _fileMappings.erase(it); - qCDebug(asset_server) << "Deleted a mapping:" << path << "=>" << oldMapping.toString(); + // add this hash to the list we need to check for asset removal from server + hashesToCheckForDeletion << it->second; + + qCDebug(asset_server) << "Deleted a mapping:" << path << "=>" << it->second; } else { qCDebug(asset_server) << "Unable to delete a mapping that was not found:" << path; } @@ -913,12 +941,9 @@ bool AssetServer::deleteMappings(AssetPathList& paths) { if (writeMappingsToFile()) { // persistence succeeded we are good to go - // grab the current mapped hashes - auto mappedHashes = _fileMappings.values(); - - // enumerate the mapped hashes and clear the list of hashes to check for anything that's present - for (auto& hashVariant : mappedHashes) { - auto it = hashesToCheckForDeletion.find(hashVariant.toString()); + // TODO iterate through hashesToCheckForDeletion instead + for (auto& pair : _fileMappings) { + auto it = hashesToCheckForDeletion.find(pair.second); if (it != hashesToCheckForDeletion.end()) { hashesToCheckForDeletion.erase(it); } @@ -974,16 +999,17 @@ bool AssetServer::renameMapping(AssetPath oldPath, AssetPath newPath) { auto it = oldMappings.begin(); while (it != oldMappings.end()) { - if (it.key().startsWith(oldPath)) { - auto newKey = it.key(); + auto& oldKey = it->first; + if (oldKey.startsWith(oldPath)) { + auto newKey = oldKey; newKey.replace(0, oldPath.size(), newPath); // remove the old version from the in memory file mappings - _fileMappings.remove(it.key()); - _fileMappings.insert(newKey, it.value()); + _fileMappings.erase(_fileMappings.find(oldKey)); + _fileMappings[newKey] = it->second; } - ++it; + it++; } if (writeMappingsToFile()) { @@ -1008,10 +1034,12 @@ bool AssetServer::renameMapping(AssetPath oldPath, AssetPath newPath) { } // take the old hash to remove the old mapping - auto oldSourceMapping = _fileMappings.take(oldPath).toString(); + auto it = _fileMappings.find(oldPath); + auto oldSourceMapping = it->second; + _fileMappings.erase(it); // in case we're overwriting, keep the current destination mapping for potential rollback - auto oldDestinationMapping = _fileMappings.value(newPath); + auto oldDestinationMapping = _fileMappings.find(newPath)->second; if (!oldSourceMapping.isEmpty()) { _fileMappings[newPath] = oldSourceMapping; @@ -1027,10 +1055,10 @@ bool AssetServer::renameMapping(AssetPath oldPath, AssetPath newPath) { if (!oldDestinationMapping.isNull()) { // put back the overwritten mapping for the destination path - _fileMappings[newPath] = oldDestinationMapping.toString(); + _fileMappings[newPath] = oldDestinationMapping; } else { // clear the new mapping - _fileMappings.remove(newPath); + _fileMappings.erase(_fileMappings.find(newPath)); } qCDebug(asset_server) << "Failed to persist renamed mapping:" << oldPath << "=>" << newPath; @@ -1052,6 +1080,19 @@ QString getBakeMapping(const AssetHash& hash, const QString& relativeFilePath) { return HIDDEN_BAKED_CONTENT_FOLDER + hash + "/" + relativeFilePath; } +void AssetServer::handleFailedBake(QString originalAssetHash, QString assetPath) { + qDebug() << "Failed: " << originalAssetHash << assetPath; + + bool loaded; + AssetMeta meta; + + std::tie(loaded, meta) = readMetaFile(originalAssetHash); + + meta.failedLastBake = true; + + writeMetaFile(originalAssetHash, meta); +} + void AssetServer::handleCompletedBake(QString originalAssetHash, QString originalAssetPath, QVector bakedFilePaths) { bool errorCompletingBake { false }; @@ -1121,20 +1162,68 @@ void AssetServer::handleCompletedBake(QString originalAssetHash, QString origina if (!errorCompletingBake) { // create the meta file to store which version of the baking process we just completed - createMetaFile(originalAssetHash); + writeMetaFile(originalAssetHash); } else { qWarning() << "Could not complete bake for" << originalAssetHash; } } -bool AssetServer::createMetaFile(AssetHash originalAssetHash) { +static const QString BAKE_VERSION_KEY = "bake_version"; +static const QString APP_VERSION_KEY = "app_version"; +static const QString FAILED_LAST_BAKE_KEY = "failed_last_bake"; + +std::pair AssetServer::readMetaFile(AssetHash hash) { + auto metaFilePath = HIDDEN_BAKED_CONTENT_FOLDER + hash + "/" + "meta.json"; + + auto it = _fileMappings.find(metaFilePath); + if (it == _fileMappings.end()) { + return { false, {} }; + } + + auto metaFileHash = it->second; + + QFile metaFile(_filesDirectory.absoluteFilePath(metaFileHash)); + + if (metaFile.open(QIODevice::ReadOnly)) { + auto data = metaFile.readAll(); + metaFile.close(); + + QJsonParseError error; + auto doc = QJsonDocument::fromJson(data, &error); + + if (error.error == QJsonParseError::NoError && doc.isObject()) { + auto root = doc.object(); + + auto bakeVersion = root[BAKE_VERSION_KEY].toInt(-1); + auto appVersion = root[APP_VERSION_KEY].toInt(-1); + auto failedLastBake = root[FAILED_LAST_BAKE_KEY]; + + if (bakeVersion != -1 + && appVersion != -1 + && failedLastBake.isBool()) { + + AssetMeta meta; + meta.bakeVersion = bakeVersion; + meta.applicationVersion = appVersion; + meta.failedLastBake = failedLastBake.toBool(); + + return { true, meta }; + } else { + qCWarning(asset_server) << "Metafile for" << hash << "has either missing or malformed data."; + } + } + } + + return { false, {} }; +} + +bool AssetServer::writeMetaFile(AssetHash originalAssetHash, AssetMeta& meta) { // construct the JSON that will be in the meta file QJsonObject metaFileObject; - static const int BAKE_VERSION = 1; - static const QString VERSION_KEY = "version"; - - metaFileObject[VERSION_KEY] = BAKE_VERSION; + metaFileObject[BAKE_VERSION_KEY] = meta.bakeVersion; + metaFileObject[APP_VERSION_KEY] = meta.applicationVersion; + metaFileObject[FAILED_LAST_BAKE_KEY] = meta.failedLastBake; QJsonDocument metaFileDoc; metaFileDoc.setObject(metaFileObject); @@ -1164,7 +1253,7 @@ bool AssetServer::setBakingEnabled(const AssetPathList& paths, bool enabled) { for (const auto& path : paths) { auto it = _fileMappings.find(path); if (it != _fileMappings.end()) { - auto hash = it->toString(); + auto hash = it->second; auto dotIndex = path.lastIndexOf("."); if (dotIndex == -1) { diff --git a/assignment-client/src/assets/AssetServer.h b/assignment-client/src/assets/AssetServer.h index 94576b329d..f3fff466a3 100644 --- a/assignment-client/src/assets/AssetServer.h +++ b/assignment-client/src/assets/AssetServer.h @@ -22,6 +22,14 @@ #include "AutoBaker.h" #include "ReceivedMessage.h" + +namespace std { + template <> + struct hash { + size_t operator()(const QString& v) const { return qHash(v); } + }; +} + class BakeAssetTask : public QObject, public QRunnable { Q_OBJECT public: @@ -33,6 +41,7 @@ public: signals: void bakeComplete(QString assetHash, QString assetPath, QVector outputFiles); + void bakeFailed(QString assetHash, QString assetPath); private: std::atomic _isBaking { false }; @@ -41,6 +50,15 @@ private: QString _filePath; }; +struct AssetMeta { + AssetMeta() { + } + + int bakeVersion { 0 }; + int applicationVersion { 0 }; + bool failedLastBake { false }; +}; + class AssetServer : public ThreadedAssignment { Q_OBJECT public: @@ -60,7 +78,7 @@ private slots: void sendStatsPacket() override; private: - using Mappings = QVariantHash; + using Mappings = std::unordered_map; void handleGetMappingOperation(ReceivedMessage& message, SharedNodePointer senderNode, NLPacketList& replyPacket); void handleGetAllMappingOperation(ReceivedMessage& message, SharedNodePointer senderNode, NLPacketList& replyPacket); @@ -100,9 +118,11 @@ private: /// Move baked content for asset to baked directory and update baked status void handleCompletedBake(QString originalAssetHash, QString assetPath, QVector bakedFilePaths); + void handleFailedBake(QString originalAssetHash, QString assetPath); /// Create meta file to describe baked content for original asset - bool createMetaFile(AssetHash originalAssetHash); + std::pair readMetaFile(AssetHash hash); + bool writeMetaFile(AssetHash originalAssetHash, AssetMeta& meta = AssetMeta()); Mappings _fileMappings; From 2aa39e7da488dae25f90baac8f2b2e64b9efc0c3 Mon Sep 17 00:00:00 2001 From: Ryan Huffman Date: Wed, 30 Aug 2017 16:53:51 -0700 Subject: [PATCH 037/129] Fix baking library not automatically being built --- libraries/baking/CMakeLists.txt | 2 -- 1 file changed, 2 deletions(-) diff --git a/libraries/baking/CMakeLists.txt b/libraries/baking/CMakeLists.txt index 44c52ae0d8..ee2d861539 100644 --- a/libraries/baking/CMakeLists.txt +++ b/libraries/baking/CMakeLists.txt @@ -11,7 +11,5 @@ if (FBX_FOUND) target_include_directories(${TARGET_NAME} SYSTEM PRIVATE ${FBX_INCLUDE_DIR}) endif () -set_target_properties(${TARGET_NAME} PROPERTIES EXCLUDE_FROM_ALL TRUE EXCLUDE_FROM_DEFAULT_BUILD TRUE) - link_hifi_libraries(shared model networking ktx image) include_hifi_library_headers(gpu) From 84b5afaa4dd7244efb93a423ff8559e25977bd3c Mon Sep 17 00:00:00 2001 From: Stephen Birarda Date: Wed, 30 Aug 2017 17:28:17 -0700 Subject: [PATCH 038/129] mark task dispatcher as override --- libraries/image/src/image/Image.cpp | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/libraries/image/src/image/Image.cpp b/libraries/image/src/image/Image.cpp index eaa55d0315..c6baee56c9 100644 --- a/libraries/image/src/image/Image.cpp +++ b/libraries/image/src/image/Image.cpp @@ -449,7 +449,7 @@ void generateMips(gpu::Texture* texture, QImage& image, int face = -1) { class SequentialTaskDispatcher : public nvtt::TaskDispatcher { public: - virtual void dispatch(nvtt::Task* task, void* context, int count) { + virtual void dispatch(nvtt::Task* task, void* context, int count) override { for (int i = 0; i < count; i++) { task(context, i); } From da8ae237565abe0bf21496ef25bdff09d2b5ce90 Mon Sep 17 00:00:00 2001 From: Stephen Birarda Date: Thu, 31 Aug 2017 11:52:42 -0700 Subject: [PATCH 039/129] move default to the bottom of bakingStatusToString --- libraries/networking/src/AssetUtils.cpp | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/libraries/networking/src/AssetUtils.cpp b/libraries/networking/src/AssetUtils.cpp index df3aa7ad0b..4c26d2d633 100644 --- a/libraries/networking/src/AssetUtils.cpp +++ b/libraries/networking/src/AssetUtils.cpp @@ -97,8 +97,8 @@ QString bakingStatusToString(BakingStatus status) { return "Baked"; case Error: return "Error"; - default: case Unrelevant: + default: return "--"; } } From 9ce0f03aa2b6f3c7f061271cd0022362256894e6 Mon Sep 17 00:00:00 2001 From: Ryan Huffman Date: Thu, 31 Aug 2017 11:58:03 -0700 Subject: [PATCH 040/129] Fix mapping comparision in getAssetStatus being done against path --- assignment-client/src/assets/AssetServer.cpp | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/assignment-client/src/assets/AssetServer.cpp b/assignment-client/src/assets/AssetServer.cpp index b91c8795c2..5bdaec8d54 100644 --- a/assignment-client/src/assets/AssetServer.cpp +++ b/assignment-client/src/assets/AssetServer.cpp @@ -142,7 +142,7 @@ BakingStatus AssetServer::getAssetStatus(const AssetPath& path, const AssetHash& auto bakedPath = "/.baked/" + hash + "/" + bakedFilename; auto jt = _fileMappings.find(bakedPath); if (jt != _fileMappings.end()) { - if (jt->first == hash) { + if (jt->second == hash) { return NotBaked; } else { return Baked; From 3aee6db1e8c20c99fa3d9b858d720fd1bffad359 Mon Sep 17 00:00:00 2001 From: Atlante45 Date: Tue, 29 Aug 2017 16:43:09 -0700 Subject: [PATCH 041/129] Log baker errors --- assignment-client/src/assets/AssetServer.cpp | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/assignment-client/src/assets/AssetServer.cpp b/assignment-client/src/assets/AssetServer.cpp index f7a8ecf453..568891adc3 100644 --- a/assignment-client/src/assets/AssetServer.cpp +++ b/assignment-client/src/assets/AssetServer.cpp @@ -84,7 +84,7 @@ void BakeAssetTask::run() { loop.exec(); if (baker->hasErrors()) { - qDebug() << "Failed to bake: " << _assetHash << _assetPath; + qDebug() << "Failed to bake: " << _assetHash << _assetPath << baker->getErrors(); } else { auto vectorOutputFiles = QVector::fromStdVector(baker->getOutputFiles()); qDebug() << "Finished baking: " << _assetHash << _assetPath << vectorOutputFiles; From da2bede6fe897bda05f5088fdaf9963aaa15c1c8 Mon Sep 17 00:00:00 2001 From: Atlante45 Date: Tue, 29 Aug 2017 16:43:25 -0700 Subject: [PATCH 042/129] Hide .baked items from the Asset Browser --- assignment-client/src/assets/AssetServer.cpp | 8 +++++--- .../src/scripting/AssetMappingsScriptingInterface.cpp | 7 +++++++ 2 files changed, 12 insertions(+), 3 deletions(-) diff --git a/assignment-client/src/assets/AssetServer.cpp b/assignment-client/src/assets/AssetServer.cpp index 568891adc3..c26600f0d9 100644 --- a/assignment-client/src/assets/AssetServer.cpp +++ b/assignment-client/src/assets/AssetServer.cpp @@ -521,9 +521,11 @@ void AssetServer::handleGetAllMappingOperation(ReceivedMessage& message, SharedN replyPacket.writePrimitive(count); for (auto it = _fileMappings.cbegin(); it != _fileMappings.cend(); ++ it) { - replyPacket.writeString(it.key()); - replyPacket.write(QByteArray::fromHex(it.value().toString().toUtf8())); - replyPacket.writePrimitive(getAssetStatus(it.key(), it.value().toString())); + auto mapping = it.key(); + auto hash = it.value().toString(); + replyPacket.writeString(mapping); + replyPacket.write(QByteArray::fromHex(hash.toUtf8())); + replyPacket.writePrimitive(getAssetStatus(mapping, hash)); } } diff --git a/interface/src/scripting/AssetMappingsScriptingInterface.cpp b/interface/src/scripting/AssetMappingsScriptingInterface.cpp index f2929501ab..71ea2de4c5 100644 --- a/interface/src/scripting/AssetMappingsScriptingInterface.cpp +++ b/interface/src/scripting/AssetMappingsScriptingInterface.cpp @@ -202,6 +202,13 @@ void AssetMappingModel::refresh() { auto existingPaths = _pathToItemMap.keys(); for (auto& mapping : mappings) { auto& path = mapping.first; + + const QString HIDDEN_BAKED_CONTENT_FOLDER = "/.baked/"; + if (path.startsWith(HIDDEN_BAKED_CONTENT_FOLDER)) { + // Hide baked mappings + continue; + } + auto parts = path.split("/"); auto length = parts.length(); From 764e5d2e4c8720006cfca98e3cd4d2b0926e0619 Mon Sep 17 00:00:00 2001 From: Atlante45 Date: Thu, 31 Aug 2017 13:01:56 -0700 Subject: [PATCH 043/129] Move constant to central location --- assignment-client/src/assets/AssetServer.cpp | 2 -- interface/src/scripting/AssetMappingsScriptingInterface.cpp | 1 - libraries/networking/src/AssetUtils.h | 2 ++ 3 files changed, 2 insertions(+), 3 deletions(-) diff --git a/assignment-client/src/assets/AssetServer.cpp b/assignment-client/src/assets/AssetServer.cpp index c26600f0d9..1565415600 100644 --- a/assignment-client/src/assets/AssetServer.cpp +++ b/assignment-client/src/assets/AssetServer.cpp @@ -53,8 +53,6 @@ static QStringList BAKEABLE_TEXTURE_EXTENSIONS; static const QString BAKED_MODEL_SIMPLE_NAME = "asset.fbx"; static const QString BAKED_TEXTURE_SIMPLE_NAME = "texture.ktx"; -static const QString HIDDEN_BAKED_CONTENT_FOLDER = "/.baked/"; - BakeAssetTask::BakeAssetTask(const AssetHash& assetHash, const AssetPath& assetPath, const QString& filePath) : _assetHash(assetHash), _assetPath(assetPath), _filePath(filePath) { } diff --git a/interface/src/scripting/AssetMappingsScriptingInterface.cpp b/interface/src/scripting/AssetMappingsScriptingInterface.cpp index 71ea2de4c5..d116c336d7 100644 --- a/interface/src/scripting/AssetMappingsScriptingInterface.cpp +++ b/interface/src/scripting/AssetMappingsScriptingInterface.cpp @@ -203,7 +203,6 @@ void AssetMappingModel::refresh() { for (auto& mapping : mappings) { auto& path = mapping.first; - const QString HIDDEN_BAKED_CONTENT_FOLDER = "/.baked/"; if (path.startsWith(HIDDEN_BAKED_CONTENT_FOLDER)) { // Hide baked mappings continue; diff --git a/libraries/networking/src/AssetUtils.h b/libraries/networking/src/AssetUtils.h index f8f0171a5d..48a2d2510d 100644 --- a/libraries/networking/src/AssetUtils.h +++ b/libraries/networking/src/AssetUtils.h @@ -33,6 +33,8 @@ const QString ASSET_FILE_PATH_REGEX_STRING = "^(\\/[^\\/\\0]+)+$"; const QString ASSET_PATH_REGEX_STRING = "^\\/([^\\/\\0]+(\\/)?)+$"; const QString ASSET_HASH_REGEX_STRING = QString("^[a-fA-F0-9]{%1}$").arg(SHA256_HASH_HEX_LENGTH); +const QString HIDDEN_BAKED_CONTENT_FOLDER = "/.baked/"; + enum AssetServerError : uint8_t { NoError = 0, AssetNotFound, From aad0be269380c161fa2872ce163ccb0ec5a96aa6 Mon Sep 17 00:00:00 2001 From: Atlante45 Date: Tue, 29 Aug 2017 17:28:06 -0700 Subject: [PATCH 044/129] Set baking enabled mappings --- assignment-client/src/assets/AssetServer.cpp | 8 +++++--- 1 file changed, 5 insertions(+), 3 deletions(-) diff --git a/assignment-client/src/assets/AssetServer.cpp b/assignment-client/src/assets/AssetServer.cpp index f7a8ecf453..c5f52b4a82 100644 --- a/assignment-client/src/assets/AssetServer.cpp +++ b/assignment-client/src/assets/AssetServer.cpp @@ -1224,9 +1224,11 @@ bool AssetServer::setBakingEnabled(const AssetPathList& paths, bool enabled) { auto bakedMapping = getBakeMapping(hash, bakedFilename); if (enabled) { - // TODO - } else { - // TODO + removeBakedPathsForDeletedAsset(hash); + setMapping(bakedMapping, hash); + } else if (_fileMappings.value(bakedMapping) == hash) { + deleteMappings({ bakedMapping }); + maybeBake(path, hash); } } } From a9cfc01df36a25e45f3f9c0f943179955c65d7ab Mon Sep 17 00:00:00 2001 From: Atlante45 Date: Thu, 31 Aug 2017 09:49:46 -0700 Subject: [PATCH 045/129] Set Baking Enabled API --- assignment-client/src/assets/AssetServer.cpp | 17 +++++++----- interface/resources/qml/AssetServer.qml | 13 +++++++++- .../AssetMappingsScriptingInterface.cpp | 26 ++++++++++++++++--- .../AssetMappingsScriptingInterface.h | 3 ++- libraries/networking/src/AssetUtils.cpp | 1 - libraries/networking/src/AssetUtils.h | 2 +- 6 files changed, 48 insertions(+), 14 deletions(-) diff --git a/assignment-client/src/assets/AssetServer.cpp b/assignment-client/src/assets/AssetServer.cpp index c5f52b4a82..a044ea172b 100644 --- a/assignment-client/src/assets/AssetServer.cpp +++ b/assignment-client/src/assets/AssetServer.cpp @@ -124,7 +124,7 @@ BakingStatus AssetServer::getAssetStatus(const AssetPath& path, const AssetHash& auto dotIndex = path.lastIndexOf("."); if (dotIndex == -1) { - return Unrelevant; + return Irrelevant; } auto extension = path.mid(dotIndex + 1); @@ -136,7 +136,7 @@ BakingStatus AssetServer::getAssetStatus(const AssetPath& path, const AssetHash& } else if (BAKEABLE_TEXTURE_EXTENSIONS.contains(extension.toLocal8Bit()) && hasMetaFile(hash)) { bakedFilename = BAKED_TEXTURE_SIMPLE_NAME; } else { - return Unrelevant; + return Irrelevant; } auto bakedPath = HIDDEN_BAKED_CONTENT_FOLDER + hash + "/" + bakedFilename; @@ -1223,12 +1223,17 @@ bool AssetServer::setBakingEnabled(const AssetPathList& paths, bool enabled) { auto bakedMapping = getBakeMapping(hash, bakedFilename); - if (enabled) { + bool currentlyDisabled = (_fileMappings.value(bakedMapping) == hash); + + if (enabled && currentlyDisabled) { + QStringList bakedMappings{ bakedMapping }; + deleteMappings(bakedMappings); + maybeBake(path, hash); + qDebug() << "Enabled baking for" << path; + } else if (!enabled && !currentlyDisabled) { removeBakedPathsForDeletedAsset(hash); setMapping(bakedMapping, hash); - } else if (_fileMappings.value(bakedMapping) == hash) { - deleteMappings({ bakedMapping }); - maybeBake(path, hash); + qDebug() << "Disabled baking for" << path; } } } diff --git a/interface/resources/qml/AssetServer.qml b/interface/resources/qml/AssetServer.qml index a34de72270..294ad1a36c 100644 --- a/interface/resources/qml/AssetServer.qml +++ b/interface/resources/qml/AssetServer.qml @@ -606,7 +606,18 @@ ScrollingWindow { HifiControls.CheckBox { text: "Use baked (optimized) versions" colorScheme: root.colorScheme - enabled: selectedItems > 0 + enabled: selectedItems === 1 && assetProxyModel.data(treeView.selection.currentIndex, 0x105) !== "--" + checked: selectedItems === 1 && assetProxyModel.data(treeView.selection.currentIndex, 0x105) === "Baked"; + onClicked: { + var mappings = []; + for (var i in treeView.selection.selectedIndexes) { + var index = treeView.selection.selectedIndexes[i]; + var path = assetProxyModel.data(index, 0x100); + mappings.push(path); + } + print("Setting baking enabled:" + mappings + checked); + Assets.setBakingEnabled(mappings, checked); + } } } diff --git a/interface/src/scripting/AssetMappingsScriptingInterface.cpp b/interface/src/scripting/AssetMappingsScriptingInterface.cpp index f2929501ab..076a9f5ada 100644 --- a/interface/src/scripting/AssetMappingsScriptingInterface.cpp +++ b/interface/src/scripting/AssetMappingsScriptingInterface.cpp @@ -161,7 +161,23 @@ void AssetMappingsScriptingInterface::renameMapping(QString oldPath, QString new connect(request, &RenameMappingRequest::finished, this, [this, callback](RenameMappingRequest* request) mutable { if (callback.isCallable()) { - QJSValueList args { request->getErrorString() }; + QJSValueList args{ request->getErrorString() }; + callback.call(args); + } + + request->deleteLater(); + }); + + request->start(); +} + +void AssetMappingsScriptingInterface::setBakingEnabled(QStringList paths, bool enabled, QJSValue callback) { + auto assetClient = DependencyManager::get(); + auto request = assetClient->createSetBakingEnabledRequest(paths, enabled); + + connect(request, &SetBakingEnabledRequest::finished, this, [this, callback](SetBakingEnabledRequest* request) mutable { + if (callback.isCallable()) { + QJSValueList args{ request->getErrorString() }; callback.call(args); } @@ -214,18 +230,16 @@ void AssetMappingModel::refresh() { // start index at 1 to avoid empty string from leading slash for (int i = 1; i < length; ++i) { fullPath += (i == 1 ? "" : "/") + parts[i]; + bool isFolder = i < length - 1; auto it = _pathToItemMap.find(fullPath); if (it == _pathToItemMap.end()) { auto item = new QStandardItem(parts[i]); - bool isFolder = i < length - 1; - auto statusString = isFolder ? "--" : bakingStatusToString(mapping.second.status); item->setData(isFolder ? fullPath + "/" : fullPath, Qt::UserRole); item->setData(isFolder, Qt::UserRole + 1); item->setData(parts[i], Qt::UserRole + 2); item->setData("atp:" + fullPath, Qt::UserRole + 3); item->setData(fullPath, Qt::UserRole + 4); - item->setData(statusString, Qt::UserRole + 5); if (lastItem) { lastItem->appendRow(item); @@ -237,6 +251,10 @@ void AssetMappingModel::refresh() { } else { lastItem = it.value(); } + + // update status + auto statusString = isFolder ? "--" : bakingStatusToString(mapping.second.status); + lastItem->setData(statusString, Qt::UserRole + 5); } Q_ASSERT(fullPath == path); diff --git a/interface/src/scripting/AssetMappingsScriptingInterface.h b/interface/src/scripting/AssetMappingsScriptingInterface.h index 0615c5e84d..e4059c1ff7 100644 --- a/interface/src/scripting/AssetMappingsScriptingInterface.h +++ b/interface/src/scripting/AssetMappingsScriptingInterface.h @@ -66,10 +66,11 @@ public: Q_INVOKABLE void setMapping(QString path, QString hash, QJSValue callback = QJSValue()); Q_INVOKABLE void getMapping(QString path, QJSValue callback = QJSValue()); Q_INVOKABLE void uploadFile(QString path, QString mapping, QJSValue startedCallback = QJSValue(), QJSValue completedCallback = QJSValue(), bool dropEvent = false); - Q_INVOKABLE void deleteMappings(QStringList paths, QJSValue callback); + Q_INVOKABLE void deleteMappings(QStringList paths, QJSValue callback = QJSValue()); Q_INVOKABLE void deleteMapping(QString path, QJSValue callback) { deleteMappings(QStringList(path), callback = QJSValue()); } Q_INVOKABLE void getAllMappings(QJSValue callback = QJSValue()); Q_INVOKABLE void renameMapping(QString oldPath, QString newPath, QJSValue callback = QJSValue()); + Q_INVOKABLE void setBakingEnabled(QStringList paths, bool enabled, QJSValue callback = QJSValue()); protected: QSet _pendingRequests; diff --git a/libraries/networking/src/AssetUtils.cpp b/libraries/networking/src/AssetUtils.cpp index 4c26d2d633..3af0b1df47 100644 --- a/libraries/networking/src/AssetUtils.cpp +++ b/libraries/networking/src/AssetUtils.cpp @@ -97,7 +97,6 @@ QString bakingStatusToString(BakingStatus status) { return "Baked"; case Error: return "Error"; - case Unrelevant: default: return "--"; } diff --git a/libraries/networking/src/AssetUtils.h b/libraries/networking/src/AssetUtils.h index f8f0171a5d..be403e93b9 100644 --- a/libraries/networking/src/AssetUtils.h +++ b/libraries/networking/src/AssetUtils.h @@ -53,7 +53,7 @@ enum AssetMappingOperationType : uint8_t { }; enum BakingStatus { - Unrelevant, + Irrelevant, NotBaked, Pending, Baking, From d2c7235fed3d1346693cc38c2f4431488bc4fd93 Mon Sep 17 00:00:00 2001 From: Ryan Huffman Date: Thu, 31 Aug 2017 14:30:00 -0700 Subject: [PATCH 046/129] Fix update to count width in GetAllMappings --- assignment-client/src/assets/AssetServer.cpp | 2 +- libraries/networking/src/MappingRequest.cpp | 2 +- 2 files changed, 2 insertions(+), 2 deletions(-) diff --git a/assignment-client/src/assets/AssetServer.cpp b/assignment-client/src/assets/AssetServer.cpp index 07239fbec4..a4211d4aa8 100644 --- a/assignment-client/src/assets/AssetServer.cpp +++ b/assignment-client/src/assets/AssetServer.cpp @@ -529,7 +529,7 @@ void AssetServer::handleGetMappingOperation(ReceivedMessage& message, SharedNode void AssetServer::handleGetAllMappingOperation(ReceivedMessage& message, SharedNodePointer senderNode, NLPacketList& replyPacket) { replyPacket.writePrimitive(AssetServerError::NoError); - uint32_t count = _fileMappings.size(); + uint32_t count = (uint32_t)_fileMappings.size(); replyPacket.writePrimitive(count); diff --git a/libraries/networking/src/MappingRequest.cpp b/libraries/networking/src/MappingRequest.cpp index 258b7c6359..2d6684ec88 100644 --- a/libraries/networking/src/MappingRequest.cpp +++ b/libraries/networking/src/MappingRequest.cpp @@ -131,7 +131,7 @@ void GetAllMappingsRequest::doStart() { if (!_error) { - int numberOfMappings; + uint32_t numberOfMappings; message->readPrimitive(&numberOfMappings); for (auto i = 0; i < numberOfMappings; ++i) { auto path = message->readString(); From cdfae714b55bb1fffdcb12d529f2a4378c12a09f Mon Sep 17 00:00:00 2001 From: Atlante45 Date: Thu, 31 Aug 2017 15:51:47 -0700 Subject: [PATCH 047/129] Remove baking tasks for hash once done --- assignment-client/src/assets/AssetServer.cpp | 2 ++ 1 file changed, 2 insertions(+) diff --git a/assignment-client/src/assets/AssetServer.cpp b/assignment-client/src/assets/AssetServer.cpp index a044ea172b..5b0343ca72 100644 --- a/assignment-client/src/assets/AssetServer.cpp +++ b/assignment-client/src/assets/AssetServer.cpp @@ -1163,6 +1163,8 @@ void AssetServer::handleCompletedBake(QString originalAssetHash, QString origina } else { qWarning() << "Could not complete bake for" << originalAssetHash; } + + _pendingBakes.remove(originalAssetHash); } bool AssetServer::createMetaFile(AssetHash originalAssetHash) { From bf53161a0b21dee4d0da0081967f669b63a1ba2c Mon Sep 17 00:00:00 2001 From: Ryan Huffman Date: Thu, 31 Aug 2017 15:55:27 -0700 Subject: [PATCH 048/129] Fix comparison between signed and unsigned int in MappingRequest --- libraries/networking/src/MappingRequest.cpp | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/libraries/networking/src/MappingRequest.cpp b/libraries/networking/src/MappingRequest.cpp index 2d6684ec88..588d8ecc88 100644 --- a/libraries/networking/src/MappingRequest.cpp +++ b/libraries/networking/src/MappingRequest.cpp @@ -133,7 +133,7 @@ void GetAllMappingsRequest::doStart() { if (!_error) { uint32_t numberOfMappings; message->readPrimitive(&numberOfMappings); - for (auto i = 0; i < numberOfMappings; ++i) { + for (uint32_t i = 0; i < numberOfMappings; ++i) { auto path = message->readString(); auto hash = message->read(SHA256_HASH_LENGTH).toHex(); BakingStatus status; From 2fd2bb1e92c4327b0445065b3e523e4de8db3cef Mon Sep 17 00:00:00 2001 From: Atlante45 Date: Thu, 31 Aug 2017 16:00:55 -0700 Subject: [PATCH 049/129] Auto reload after changing baking enabled --- interface/resources/qml/AssetServer.qml | 4 +++- 1 file changed, 3 insertions(+), 1 deletion(-) diff --git a/interface/resources/qml/AssetServer.qml b/interface/resources/qml/AssetServer.qml index 294ad1a36c..36ef3e69f8 100644 --- a/interface/resources/qml/AssetServer.qml +++ b/interface/resources/qml/AssetServer.qml @@ -616,7 +616,9 @@ ScrollingWindow { mappings.push(path); } print("Setting baking enabled:" + mappings + checked); - Assets.setBakingEnabled(mappings, checked); + Assets.setBakingEnabled(mappings, checked, function() { + reload(); + }); } } } From 9b857eb53bd2b136d9ea5f62916934540e1297b6 Mon Sep 17 00:00:00 2001 From: Atlante45 Date: Thu, 31 Aug 2017 16:16:43 -0700 Subject: [PATCH 050/129] Ensure "enabled" binding stays in place --- interface/resources/qml/AssetServer.qml | 10 +++++++++- 1 file changed, 9 insertions(+), 1 deletion(-) diff --git a/interface/resources/qml/AssetServer.qml b/interface/resources/qml/AssetServer.qml index 36ef3e69f8..f4b434ae5a 100644 --- a/interface/resources/qml/AssetServer.qml +++ b/interface/resources/qml/AssetServer.qml @@ -604,10 +604,16 @@ ScrollingWindow { } HifiControls.CheckBox { + function isChecked() { + var status = assetProxyModel.data(treeView.selection.currentIndex, 0x105); + var bakingDisabled = (status === "Not Baked" || status === "--"); + return selectedItems === 1 && !bakingDisabled; + } + text: "Use baked (optimized) versions" colorScheme: root.colorScheme enabled: selectedItems === 1 && assetProxyModel.data(treeView.selection.currentIndex, 0x105) !== "--" - checked: selectedItems === 1 && assetProxyModel.data(treeView.selection.currentIndex, 0x105) === "Baked"; + checked: isChecked() onClicked: { var mappings = []; for (var i in treeView.selection.selectedIndexes) { @@ -619,6 +625,8 @@ ScrollingWindow { Assets.setBakingEnabled(mappings, checked, function() { reload(); }); + + checked = Qt.binding(isChecked); } } } From 7a68788c8c9b8646c3fe72f1efa84ac950b693ae Mon Sep 17 00:00:00 2001 From: Ryan Huffman Date: Thu, 31 Aug 2017 16:22:46 -0700 Subject: [PATCH 051/129] Fix warning for non-const default in AssetServer --- assignment-client/src/assets/AssetServer.cpp | 2 +- assignment-client/src/assets/AssetServer.h | 2 +- 2 files changed, 2 insertions(+), 2 deletions(-) diff --git a/assignment-client/src/assets/AssetServer.cpp b/assignment-client/src/assets/AssetServer.cpp index 5b9cec9346..6653842032 100644 --- a/assignment-client/src/assets/AssetServer.cpp +++ b/assignment-client/src/assets/AssetServer.cpp @@ -1255,7 +1255,7 @@ std::pair AssetServer::readMetaFile(AssetHash hash) { return { false, {} }; } -bool AssetServer::writeMetaFile(AssetHash originalAssetHash, AssetMeta& meta) { +bool AssetServer::writeMetaFile(AssetHash originalAssetHash, const AssetMeta& meta) { // construct the JSON that will be in the meta file QJsonObject metaFileObject; diff --git a/assignment-client/src/assets/AssetServer.h b/assignment-client/src/assets/AssetServer.h index 531c588762..44e16272b2 100644 --- a/assignment-client/src/assets/AssetServer.h +++ b/assignment-client/src/assets/AssetServer.h @@ -122,7 +122,7 @@ private: /// Create meta file to describe baked content for original asset std::pair readMetaFile(AssetHash hash); - bool writeMetaFile(AssetHash originalAssetHash, AssetMeta& meta = AssetMeta()); + bool writeMetaFile(AssetHash originalAssetHash, const AssetMeta& meta = AssetMeta()); /// Remove baked paths when the original asset is deleteds void removeBakedPathsForDeletedAsset(AssetHash originalAssetHash); From 5ee33085a571661a8757bfdfc73275a035be97a5 Mon Sep 17 00:00:00 2001 From: Ryan Huffman Date: Fri, 1 Sep 2017 10:37:28 -0700 Subject: [PATCH 052/129] Add handling of failedLastBake error in getAssetStatus --- assignment-client/src/assets/AssetServer.cpp | 8 ++++++++ 1 file changed, 8 insertions(+) diff --git a/assignment-client/src/assets/AssetServer.cpp b/assignment-client/src/assets/AssetServer.cpp index 6653842032..903322a8ea 100644 --- a/assignment-client/src/assets/AssetServer.cpp +++ b/assignment-client/src/assets/AssetServer.cpp @@ -145,6 +145,14 @@ BakingStatus AssetServer::getAssetStatus(const AssetPath& path, const AssetHash& auto jt = _fileMappings.find(bakedPath); if (jt != _fileMappings.end()) { if (jt->second == hash) { + bool loaded; + AssetMeta meta; + + std::tie(loaded, meta) = readMetaFile(hash); + if (loaded && meta.failedLastBake) { + return Error; + } + return NotBaked; } else { return Baked; From c67fcc13844c9b28e8eb2b304e87aa5f3033e2ae Mon Sep 17 00:00:00 2001 From: Ryan Huffman Date: Fri, 1 Sep 2017 10:41:31 -0700 Subject: [PATCH 053/129] Add RedirectFail to ResourceRequest::getResultString() --- libraries/networking/src/ResourceRequest.cpp | 1 + 1 file changed, 1 insertion(+) diff --git a/libraries/networking/src/ResourceRequest.cpp b/libraries/networking/src/ResourceRequest.cpp index aeeab2232a..f028535e2f 100644 --- a/libraries/networking/src/ResourceRequest.cpp +++ b/libraries/networking/src/ResourceRequest.cpp @@ -36,6 +36,7 @@ QString ResourceRequest::getResultString() const { case AccessDenied: return "Access Denied"; case InvalidURL: return "Invalid URL"; case NotFound: return "Not Found"; + case RedirectFail: return "Redirect Fail"; default: return "Unspecified Error"; } } From eb2e87a7982897f1612257e5e111bf9f49e3fe0e Mon Sep 17 00:00:00 2001 From: Ryan Huffman Date: Fri, 1 Sep 2017 13:02:25 -0700 Subject: [PATCH 054/129] Change it++ to ++it --- assignment-client/src/assets/AssetServer.cpp | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/assignment-client/src/assets/AssetServer.cpp b/assignment-client/src/assets/AssetServer.cpp index 903322a8ea..489e94f3e7 100644 --- a/assignment-client/src/assets/AssetServer.cpp +++ b/assignment-client/src/assets/AssetServer.cpp @@ -1056,7 +1056,7 @@ bool AssetServer::renameMapping(AssetPath oldPath, AssetPath newPath) { _fileMappings[newKey] = it->second; } - it++; + ++it; } if (writeMappingsToFile()) { From 28402be37c7345cda5a2f3b3f1f4f9b3834fa02d Mon Sep 17 00:00:00 2001 From: Atlante45 Date: Thu, 31 Aug 2017 17:39:36 -0700 Subject: [PATCH 055/129] Fix status glyphs --- interface/resources/qml/AssetServer.qml | 126 +++++++++++++++++- interface/resources/qml/controls-uit/Tree.qml | 105 +-------------- 2 files changed, 131 insertions(+), 100 deletions(-) diff --git a/interface/resources/qml/AssetServer.qml b/interface/resources/qml/AssetServer.qml index f4b434ae5a..5e564f8626 100644 --- a/interface/resources/qml/AssetServer.qml +++ b/interface/resources/qml/AssetServer.qml @@ -10,6 +10,7 @@ import QtQuick 2.5 import QtQuick.Controls 1.4 +import QtQuick.Controls.Styles 1.4 import QtQuick.Dialogs 1.2 as OriginalDialogs import Qt.labs.settings 1.0 @@ -527,7 +528,6 @@ ScrollingWindow { headerVisible: true sortIndicatorVisible: true - canEdit: true colorScheme: root.colorScheme modifyEl: renameEl @@ -542,8 +542,130 @@ ScrollingWindow { id: bakedColumn title: "Use Baked?" role: "baked" - width: 140 + width: 100 } + + itemDelegate: Loader { + id: itemDelegateLoader + + anchors { + left: parent ? parent.left : undefined + leftMargin: (styleData.column === 0 ? (2 + styleData.depth) : 1) * hifi.dimensions.tablePadding + right: parent ? parent.right : undefined + rightMargin: hifi.dimensions.tablePadding + verticalCenter: parent ? parent.verticalCenter : undefined + } + + function convertToGlyph(text) { + switch (text) { + case "Not Baked": + return hifi.glyphs.circleSlash; + case "Baked": + return hifi.glyphs.placemark; + case "Error": + return hifi.glyphs.alert; + default: + return ""; + } + } + + function getComponent() { + if ((styleData.column === 0) && styleData.selected) { + return textFieldComponent; + } else if (convertToGlyph(styleData.value) != "") { + return glyphComponent; + } else { + return labelComponent; + } + + } + sourceComponent: getComponent() + + Component { + id: labelComponent + FiraSansSemiBold { + text: styleData.value + size: hifi.fontSizes.tableText + color: colorScheme == hifi.colorSchemes.light + ? (styleData.selected ? hifi.colors.black : hifi.colors.baseGrayHighlight) + : (styleData.selected ? hifi.colors.black : hifi.colors.lightGrayText) + + elide: Text.ElideRight + horizontalAlignment: styleData.column === 1 ? TextInput.AlignHCenter : TextInput.AlignLeft + } + } + Component { + id: glyphComponent + + HiFiGlyphs { + text: convertToGlyph(styleData.value) + size: hifi.dimensions.frameIconSize + color: colorScheme == hifi.colorSchemes.light + ? (styleData.selected ? hifi.colors.black : hifi.colors.baseGrayHighlight) + : (styleData.selected ? hifi.colors.black : hifi.colors.lightGrayText) + + elide: Text.ElideRight + horizontalAlignment: TextInput.AlignHCenter + } + } + Component { + id: textFieldComponent + + TextField { + id: textField + readOnly: !activeFocus + + text: styleData.value + + FontLoader { id: firaSansSemiBold; source: "../../fonts/FiraSans-SemiBold.ttf"; } + font.family: firaSansSemiBold.name + font.pixelSize: hifi.fontSizes.textFieldInput + height: hifi.dimensions.tableRowHeight + + style: TextFieldStyle { + textColor: readOnly + ? hifi.colors.black + : (treeView.isLightColorScheme ? hifi.colors.black : hifi.colors.white) + background: Rectangle { + visible: !readOnly + color: treeView.isLightColorScheme ? hifi.colors.white : hifi.colors.black + border.color: hifi.colors.primaryHighlight + border.width: 1 + } + selectedTextColor: hifi.colors.black + selectionColor: hifi.colors.primaryHighlight + padding.left: readOnly ? 0 : hifi.dimensions.textPadding + padding.right: readOnly ? 0 : hifi.dimensions.textPadding + } + + validator: RegExpValidator { + regExp: /[^/]+/ + } + + Keys.onPressed: { + if (event.key == Qt.Key_Escape) { + text = styleData.value; + unfocusHelper.forceActiveFocus(); + event.accepted = true; + } + } + onAccepted: { + if (acceptableInput && styleData.selected) { + if (!modifyEl(selection.currentIndex, text)) { + text = styleData.value; + } + unfocusHelper.forceActiveFocus(); + } + } + + onReadOnlyChanged: { + // Have to explicily set keyboardRaised because automatic setting fails because readOnly is true at the time. + keyboardRaised = activeFocus; + } + } + } + } + MouseArea { propagateComposedEvents: true diff --git a/interface/resources/qml/controls-uit/Tree.qml b/interface/resources/qml/controls-uit/Tree.qml index ce7edfcaab..d94a8cdceb 100644 --- a/interface/resources/qml/controls-uit/Tree.qml +++ b/interface/resources/qml/controls-uit/Tree.qml @@ -19,7 +19,6 @@ TreeView { id: treeView property var treeModel: ListModel { } - property var canEdit: false property bool centerHeaderText: false property int colorScheme: hifi.colorSchemes.light readonly property bool isLightColorScheme: colorScheme == hifi.colorSchemes.light @@ -198,9 +197,7 @@ TreeView { : (styleData.alternate ? hifi.colors.tableRowDarkEven : hifi.colors.tableRowDarkOdd) } - itemDelegate: Loader { - id: itemDelegateLoader - + itemDelegate: FiraSansSemiBold { anchors { left: parent ? parent.left : undefined leftMargin: (2 + styleData.depth) * hifi.dimensions.tablePadding @@ -209,101 +206,13 @@ TreeView { verticalCenter: parent ? parent.verticalCenter : undefined } - function getComponent() { - if (treeView.canEdit && styleData.selected) { - return textFieldComponent; - } else { - if (styleData.value.startsWith("HifiGlyphs#")) { - return glyphComponent; - } else { - return labelComponent; - } - } - - } - sourceComponent: getComponent() - - Component { - id: labelComponent - FiraSansSemiBold { - - text: styleData.value - size: hifi.fontSizes.tableText - color: colorScheme == hifi.colorSchemes.light - ? (styleData.selected ? hifi.colors.black : hifi.colors.baseGrayHighlight) - : (styleData.selected ? hifi.colors.black : hifi.colors.lightGrayText) + text: styleData.value + size: hifi.fontSizes.tableText + color: colorScheme == hifi.colorSchemes.light + ? (styleData.selected ? hifi.colors.black : hifi.colors.baseGrayHighlight) + : (styleData.selected ? hifi.colors.black : hifi.colors.lightGrayText) - elide: Text.ElideRight - } - } - Component { - id: glyphComponent - HiFiGlyphs { - text: styleData.value.replace("HifiGlyphs#", "") - size: hifi.fontSizes.tableText - color: colorScheme == hifi.colorSchemes.light - ? (styleData.selected ? hifi.colors.black : hifi.colors.baseGrayHighlight) - : (styleData.selected ? hifi.colors.black : hifi.colors.lightGrayText) - - elide: Text.ElideRight - } - } - Component { - id: textFieldComponent - - TextField { - id: textField - readOnly: !activeFocus - - text: styleData.value - - FontLoader { id: firaSansSemiBold; source: "../../fonts/FiraSans-SemiBold.ttf"; } - font.family: firaSansSemiBold.name - font.pixelSize: hifi.fontSizes.textFieldInput - height: hifi.dimensions.tableRowHeight - - style: TextFieldStyle { - textColor: readOnly - ? hifi.colors.black - : (treeView.isLightColorScheme ? hifi.colors.black : hifi.colors.white) - background: Rectangle { - visible: !readOnly - color: treeView.isLightColorScheme ? hifi.colors.white : hifi.colors.black - border.color: hifi.colors.primaryHighlight - border.width: 1 - } - selectedTextColor: hifi.colors.black - selectionColor: hifi.colors.primaryHighlight - padding.left: readOnly ? 0 : hifi.dimensions.textPadding - padding.right: readOnly ? 0 : hifi.dimensions.textPadding - } - - validator: RegExpValidator { - regExp: /[^/]+/ - } - - Keys.onPressed: { - if (event.key == Qt.Key_Escape) { - text = styleData.value; - unfocusHelper.forceActiveFocus(); - event.accepted = true; - } - } - onAccepted: { - if (acceptableInput && styleData.selected) { - if (!modifyEl(selection.currentIndex, text)) { - text = styleData.value; - } - unfocusHelper.forceActiveFocus(); - } - } - - onReadOnlyChanged: { - // Have to explicily set keyboardRaised because automatic setting fails because readOnly is true at the time. - keyboardRaised = activeFocus; - } - } - } + elide: Text.ElideRight } Item { From 26b74db8641da504eca295c5b73d3dec2ea9abff Mon Sep 17 00:00:00 2001 From: Stephen Birarda Date: Thu, 31 Aug 2017 11:31:58 -0700 Subject: [PATCH 056/129] enable compression in image library from Asset Server --- assignment-client/src/assets/AssetServer.cpp | 6 ++++++ tools/oven/src/Oven.cpp | 2 +- 2 files changed, 7 insertions(+), 1 deletion(-) diff --git a/assignment-client/src/assets/AssetServer.cpp b/assignment-client/src/assets/AssetServer.cpp index 43689a5b08..cacbb25e5b 100644 --- a/assignment-client/src/assets/AssetServer.cpp +++ b/assignment-client/src/assets/AssetServer.cpp @@ -266,6 +266,12 @@ AssetServer::AssetServer(ReceivedMessage& message) : _transferTaskPool(this), _bakingTaskPool(this) { + // enable compression in image library + image::setColorTexturesCompressionEnabled(true); + image::setGrayscaleTexturesCompressionEnabled(true); + image::setNormalTexturesCompressionEnabled(true); + image::setCubeTexturesCompressionEnabled(true); + BAKEABLE_TEXTURE_EXTENSIONS = TextureBaker::getSupportedFormats(); qDebug() << "Supported baking texture formats:" << BAKEABLE_MODEL_EXTENSIONS; diff --git a/tools/oven/src/Oven.cpp b/tools/oven/src/Oven.cpp index d0b8c3cd65..c9d3b4e5f0 100644 --- a/tools/oven/src/Oven.cpp +++ b/tools/oven/src/Oven.cpp @@ -44,7 +44,7 @@ Oven::Oven(int argc, char* argv[]) : parser.addHelpOption(); parser.process(*this); - // enable compression in image library, except for cube maps + // enable compression in image library image::setColorTexturesCompressionEnabled(true); image::setGrayscaleTexturesCompressionEnabled(true); image::setNormalTexturesCompressionEnabled(true); From e25b4700d99b8e8093af370d17aa6c5187573e47 Mon Sep 17 00:00:00 2001 From: Stephen Birarda Date: Thu, 31 Aug 2017 11:48:05 -0700 Subject: [PATCH 057/129] re-set defaults in texture baking when AssetServer done --- assignment-client/src/assets/AssetServer.cpp | 14 ++++++++++++++ assignment-client/src/assets/AssetServer.h | 7 +++++++ 2 files changed, 21 insertions(+) diff --git a/assignment-client/src/assets/AssetServer.cpp b/assignment-client/src/assets/AssetServer.cpp index cacbb25e5b..b2ebdaccc9 100644 --- a/assignment-client/src/assets/AssetServer.cpp +++ b/assignment-client/src/assets/AssetServer.cpp @@ -266,6 +266,12 @@ AssetServer::AssetServer(ReceivedMessage& message) : _transferTaskPool(this), _bakingTaskPool(this) { + // store the current state of image compression so we can reset it when this assignment is complete + _wasColorTextureCompressionEnabled = image::isColorTexturesCompressionEnabled(); + _wasGrayscaleTextureCompressionEnabled = image::isGrayscaleTexturesCompressionEnabled(); + _wasNormalTextureCompressionEnabled = image::isNormalTexturesCompressionEnabled(); + _wasCubeTextureCompressionEnabled = image::isCubeTexturesCompressionEnabled(); + // enable compression in image library image::setColorTexturesCompressionEnabled(true); image::setGrayscaleTexturesCompressionEnabled(true); @@ -302,6 +308,14 @@ AssetServer::AssetServer(ReceivedMessage& message) : #endif } +void AssetServer::aboutToFinish() { + // re-set defaults in image library + image::setColorTexturesCompressionEnabled(_wasCubeTextureCompressionEnabled); + image::setGrayscaleTexturesCompressionEnabled(_wasGrayscaleTextureCompressionEnabled); + image::setNormalTexturesCompressionEnabled(_wasNormalTextureCompressionEnabled); + image::setCubeTexturesCompressionEnabled(_wasCubeTextureCompressionEnabled); +} + void AssetServer::run() { qCDebug(asset_server) << "Waiting for connection to domain to request settings from domain-server."; diff --git a/assignment-client/src/assets/AssetServer.h b/assignment-client/src/assets/AssetServer.h index 44e16272b2..907726e1ab 100644 --- a/assignment-client/src/assets/AssetServer.h +++ b/assignment-client/src/assets/AssetServer.h @@ -64,6 +64,8 @@ class AssetServer : public ThreadedAssignment { public: AssetServer(ReceivedMessage& message); + void aboutToFinish() override; + public slots: void run() override; @@ -137,6 +139,11 @@ private: QHash> _pendingBakes; QThreadPool _bakingTaskPool; + + bool _wasColorTextureCompressionEnabled { false }; + bool _wasGrayscaleTextureCompressionEnabled { false }; + bool _wasNormalTextureCompressionEnabled { false }; + bool _wasCubeTextureCompressionEnabled { false }; }; #endif From 7859b3b11f58ec72737e00037941cba1bdd8423b Mon Sep 17 00:00:00 2001 From: Stephen Birarda Date: Fri, 1 Sep 2017 15:34:42 -0700 Subject: [PATCH 058/129] remove image compression choices from Interface settings --- interface/src/ui/PreferencesDialog.cpp | 24 ----------------- libraries/image/src/image/Image.cpp | 36 ++++++++++---------------- 2 files changed, 13 insertions(+), 47 deletions(-) diff --git a/interface/src/ui/PreferencesDialog.cpp b/interface/src/ui/PreferencesDialog.cpp index 1c3df3210c..287abd9033 100644 --- a/interface/src/ui/PreferencesDialog.cpp +++ b/interface/src/ui/PreferencesDialog.cpp @@ -333,30 +333,6 @@ void setupPreferences() { preferences->addPreference(preference); } } - { - auto getter = []()->bool { return image::isColorTexturesCompressionEnabled(); }; - auto setter = [](bool value) { return image::setColorTexturesCompressionEnabled(value); }; - auto preference = new CheckPreference(RENDER, "Compress Color Textures", getter, setter); - preferences->addPreference(preference); - } - { - auto getter = []()->bool { return image::isNormalTexturesCompressionEnabled(); }; - auto setter = [](bool value) { return image::setNormalTexturesCompressionEnabled(value); }; - auto preference = new CheckPreference(RENDER, "Compress Normal Textures", getter, setter); - preferences->addPreference(preference); - } - { - auto getter = []()->bool { return image::isGrayscaleTexturesCompressionEnabled(); }; - auto setter = [](bool value) { return image::setGrayscaleTexturesCompressionEnabled(value); }; - auto preference = new CheckPreference(RENDER, "Compress Grayscale Textures", getter, setter); - preferences->addPreference(preference); - } - { - auto getter = []()->bool { return image::isCubeTexturesCompressionEnabled(); }; - auto setter = [](bool value) { return image::setCubeTexturesCompressionEnabled(value); }; - auto preference = new CheckPreference(RENDER, "Compress Cube Textures", getter, setter); - preferences->addPreference(preference); - } } { static const QString RENDER("Networking"); diff --git a/libraries/image/src/image/Image.cpp b/libraries/image/src/image/Image.cpp index c6baee56c9..d3f40ead05 100644 --- a/libraries/image/src/image/Image.cpp +++ b/libraries/image/src/image/Image.cpp @@ -22,7 +22,6 @@ #include #include #include -#include #include "ImageLogging.h" @@ -30,18 +29,17 @@ using namespace gpu; #define CPU_MIPMAPS 1 -static std::mutex settingsMutex; -static Setting::Handle compressColorTextures("hifi.graphics.compressColorTextures", false); -static Setting::Handle compressNormalTextures("hifi.graphics.compressNormalTextures", false); -static Setting::Handle compressGrayscaleTextures("hifi.graphics.compressGrayscaleTextures", false); -static Setting::Handle compressCubeTextures("hifi.graphics.compressCubeTextures", false); - static const glm::uvec2 SPARSE_PAGE_SIZE(128); static const glm::uvec2 MAX_TEXTURE_SIZE(4096); bool DEV_DECIMATE_TEXTURES = false; std::atomic DECIMATED_TEXTURE_COUNT{ 0 }; std::atomic RECTIFIED_TEXTURE_COUNT{ 0 }; +static std::atomic compressColorTextures { false }; +static std::atomic compressNormalTextures { false }; +static std::atomic compressGrayscaleTextures { false }; +static std::atomic compressCubeTextures { false }; + bool needsSparseRectification(const glm::uvec2& size) { // Don't attempt to rectify small textures (textures less than the sparse page size in any dimension) if (glm::any(glm::lessThan(size, SPARSE_PAGE_SIZE))) { @@ -150,8 +148,7 @@ gpu::TexturePointer TextureUsage::createCubeTextureFromImageWithoutIrradiance(co bool isColorTexturesCompressionEnabled() { #if CPU_MIPMAPS - std::lock_guard guard(settingsMutex); - return compressColorTextures.get(); + return compressColorTextures; #else return false; #endif @@ -159,8 +156,7 @@ bool isColorTexturesCompressionEnabled() { bool isNormalTexturesCompressionEnabled() { #if CPU_MIPMAPS - std::lock_guard guard(settingsMutex); - return compressNormalTextures.get(); + return compressNormalTextures; #else return false; #endif @@ -168,8 +164,7 @@ bool isNormalTexturesCompressionEnabled() { bool isGrayscaleTexturesCompressionEnabled() { #if CPU_MIPMAPS - std::lock_guard guard(settingsMutex); - return compressGrayscaleTextures.get(); + return compressGrayscaleTextures; #else return false; #endif @@ -177,31 +172,26 @@ bool isGrayscaleTexturesCompressionEnabled() { bool isCubeTexturesCompressionEnabled() { #if CPU_MIPMAPS - std::lock_guard guard(settingsMutex); - return compressCubeTextures.get(); + return compressCubeTextures; #else return false; #endif } void setColorTexturesCompressionEnabled(bool enabled) { - std::lock_guard guard(settingsMutex); - compressColorTextures.set(enabled); + compressColorTextures = enabled; } void setNormalTexturesCompressionEnabled(bool enabled) { - std::lock_guard guard(settingsMutex); - compressNormalTextures.set(enabled); + compressNormalTextures = enabled; } void setGrayscaleTexturesCompressionEnabled(bool enabled) { - std::lock_guard guard(settingsMutex); - compressGrayscaleTextures.set(enabled); + compressGrayscaleTextures = enabled; } void setCubeTexturesCompressionEnabled(bool enabled) { - std::lock_guard guard(settingsMutex); - compressCubeTextures.set(enabled); + compressCubeTextures = enabled; } From 1508edb459fdea1fa4ad8375559a2654d0df091d Mon Sep 17 00:00:00 2001 From: Stephen Birarda Date: Fri, 1 Sep 2017 15:41:20 -0700 Subject: [PATCH 059/129] use store/load to be clearer about atomics --- libraries/image/src/image/Image.cpp | 16 ++++++++-------- 1 file changed, 8 insertions(+), 8 deletions(-) diff --git a/libraries/image/src/image/Image.cpp b/libraries/image/src/image/Image.cpp index d3f40ead05..30299663de 100644 --- a/libraries/image/src/image/Image.cpp +++ b/libraries/image/src/image/Image.cpp @@ -148,7 +148,7 @@ gpu::TexturePointer TextureUsage::createCubeTextureFromImageWithoutIrradiance(co bool isColorTexturesCompressionEnabled() { #if CPU_MIPMAPS - return compressColorTextures; + return compressColorTextures.load(); #else return false; #endif @@ -156,7 +156,7 @@ bool isColorTexturesCompressionEnabled() { bool isNormalTexturesCompressionEnabled() { #if CPU_MIPMAPS - return compressNormalTextures; + return compressNormalTextures.load(); #else return false; #endif @@ -164,7 +164,7 @@ bool isNormalTexturesCompressionEnabled() { bool isGrayscaleTexturesCompressionEnabled() { #if CPU_MIPMAPS - return compressGrayscaleTextures; + return compressGrayscaleTextures.load(); #else return false; #endif @@ -172,26 +172,26 @@ bool isGrayscaleTexturesCompressionEnabled() { bool isCubeTexturesCompressionEnabled() { #if CPU_MIPMAPS - return compressCubeTextures; + return compressCubeTextures.load(); #else return false; #endif } void setColorTexturesCompressionEnabled(bool enabled) { - compressColorTextures = enabled; + compressColorTextures.store(enabled); } void setNormalTexturesCompressionEnabled(bool enabled) { - compressNormalTextures = enabled; + compressNormalTextures.store(enabled); } void setGrayscaleTexturesCompressionEnabled(bool enabled) { - compressGrayscaleTextures = enabled; + compressGrayscaleTextures.store(enabled); } void setCubeTexturesCompressionEnabled(bool enabled) { - compressCubeTextures = enabled; + compressCubeTextures.store(enabled); } From e4de869db6b6df2840b3fa5bb3eb928767347105 Mon Sep 17 00:00:00 2001 From: Stephen Birarda Date: Wed, 6 Sep 2017 13:56:50 -0700 Subject: [PATCH 060/129] fix iterator find from merge --- assignment-client/src/assets/AssetServer.cpp | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/assignment-client/src/assets/AssetServer.cpp b/assignment-client/src/assets/AssetServer.cpp index b2ebdaccc9..9e65ea3d96 100644 --- a/assignment-client/src/assets/AssetServer.cpp +++ b/assignment-client/src/assets/AssetServer.cpp @@ -1342,7 +1342,8 @@ bool AssetServer::setBakingEnabled(const AssetPathList& paths, bool enabled) { auto bakedMapping = getBakeMapping(hash, bakedFilename); - bool currentlyDisabled = (_fileMappings.value(bakedMapping) == hash); + auto it = _fileMappings.find(bakedMapping); + bool currentlyDisabled = (it != _fileMappings.end() && it->second == hash); if (enabled && currentlyDisabled) { QStringList bakedMappings{ bakedMapping }; From bd36ba19b32775b3e4079091452b1bd8bea14321 Mon Sep 17 00:00:00 2001 From: Stephen Birarda Date: Thu, 7 Sep 2017 12:02:07 -0700 Subject: [PATCH 061/129] fix dereference of end iterator --- assignment-client/src/assets/AssetServer.cpp | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/assignment-client/src/assets/AssetServer.cpp b/assignment-client/src/assets/AssetServer.cpp index 9e65ea3d96..cee4c77170 100644 --- a/assignment-client/src/assets/AssetServer.cpp +++ b/assignment-client/src/assets/AssetServer.cpp @@ -1106,7 +1106,7 @@ bool AssetServer::renameMapping(AssetPath oldPath, AssetPath newPath) { _fileMappings.erase(it); // in case we're overwriting, keep the current destination mapping for potential rollback - auto oldDestinationMapping = _fileMappings.find(newPath)->second; + auto oldDestinationIt = _fileMappings.find(newPath); if (!oldSourceMapping.isEmpty()) { _fileMappings[newPath] = oldSourceMapping; @@ -1120,9 +1120,9 @@ bool AssetServer::renameMapping(AssetPath oldPath, AssetPath newPath) { // we couldn't persist the renamed mapping, rollback and return failure _fileMappings[oldPath] = oldSourceMapping; - if (!oldDestinationMapping.isNull()) { + if (oldDestinationIt != _fileMappings.end()) { // put back the overwritten mapping for the destination path - _fileMappings[newPath] = oldDestinationMapping; + _fileMappings[newPath] = oldDestinationIt->second; } else { // clear the new mapping _fileMappings.erase(_fileMappings.find(newPath)); From aa1aad0a0995fbe7db54209aa7a347e5bf228033 Mon Sep 17 00:00:00 2001 From: Ryan Huffman Date: Thu, 7 Sep 2017 10:36:19 -0700 Subject: [PATCH 062/129] Add FBXWriter for serializing FBXNode --- libraries/fbx/src/FBX.cpp | 10 + libraries/fbx/src/FBX.h | 333 +++++++++++++++++++++++++++ libraries/fbx/src/FBXReader.cpp | 11 +- libraries/fbx/src/FBXReader.h | 303 +----------------------- libraries/fbx/src/FBXReader_Node.cpp | 20 +- libraries/fbx/src/FBXWriter.cpp | 253 ++++++++++++++++++++ libraries/fbx/src/FBXWriter.h | 28 +++ 7 files changed, 641 insertions(+), 317 deletions(-) create mode 100644 libraries/fbx/src/FBX.cpp create mode 100644 libraries/fbx/src/FBX.h create mode 100644 libraries/fbx/src/FBXWriter.cpp create mode 100644 libraries/fbx/src/FBXWriter.h diff --git a/libraries/fbx/src/FBX.cpp b/libraries/fbx/src/FBX.cpp new file mode 100644 index 0000000000..3f18b3e678 --- /dev/null +++ b/libraries/fbx/src/FBX.cpp @@ -0,0 +1,10 @@ +// +// FBX.cpp +// libraries/fbx/src +// +// Created by Ryan Huffman on 9/5/17. +// Copyright 2017 High Fidelity, Inc. +// +// Distributed under the Apache License, Version 2.0. +// See the accompanying file LICENSE or http://www.apache.org/licenses/LICENSE-2.0.html +// diff --git a/libraries/fbx/src/FBX.h b/libraries/fbx/src/FBX.h new file mode 100644 index 0000000000..9e1982f9e0 --- /dev/null +++ b/libraries/fbx/src/FBX.h @@ -0,0 +1,333 @@ +// +// FBX.h +// libraries/fbx/src +// +// Created by Ryan Huffman on 9/5/17. +// Copyright 2017 High Fidelity, Inc. +// +// Distributed under the Apache License, Version 2.0. +// See the accompanying file LICENSE or http://www.apache.org/licenses/LICENSE-2.0.html +// + +#ifndef hifi_FBX_h_ +#define hifi_FBX_h_ + +#include +#include +#include +#include +#include +#include + +#include +#include + +#include +#include + +#include +#include + +static const QByteArray FBX_BINARY_PROLOG = "Kaydara FBX Binary "; +static const int FBX_HEADER_BYTES_BEFORE_VERSION = 23; +static const quint32 FBX_VERSION_2016 = 7500; + +class FBXNode; +using FBXNodeList = QList; + + +/// A node within an FBX document. +class FBXNode { +public: + QByteArray name; + QVariantList properties; + FBXNodeList children; +}; + + +/// A single blendshape extracted from an FBX document. +class FBXBlendshape { +public: + QVector indices; + QVector vertices; + QVector normals; +}; + +struct FBXJointShapeInfo { + // same units and frame as FBXJoint.translation + glm::vec3 avgPoint; + std::vector dots; + std::vector points; + std::vector debugLines; +}; + +/// A single joint (transformation node) extracted from an FBX document. +class FBXJoint { +public: + + FBXJointShapeInfo shapeInfo; + QVector freeLineage; + bool isFree; + int parentIndex; + float distanceToParent; + + // http://download.autodesk.com/us/fbx/20112/FBX_SDK_HELP/SDKRef/a00209.html + + glm::vec3 translation; // T + glm::mat4 preTransform; // Roff * Rp + glm::quat preRotation; // Rpre + glm::quat rotation; // R + glm::quat postRotation; // Rpost + glm::mat4 postTransform; // Rp-1 * Soff * Sp * S * Sp-1 + + // World = ParentWorld * T * (Roff * Rp) * Rpre * R * Rpost * (Rp-1 * Soff * Sp * S * Sp-1) + + glm::mat4 transform; + glm::vec3 rotationMin; // radians + glm::vec3 rotationMax; // radians + glm::quat inverseDefaultRotation; + glm::quat inverseBindRotation; + glm::mat4 bindTransform; + QString name; + bool isSkeletonJoint; + bool bindTransformFoundInCluster; + + // geometric offset is applied in local space but does NOT affect children. + bool hasGeometricOffset; + glm::vec3 geometricTranslation; + glm::quat geometricRotation; + glm::vec3 geometricScaling; +}; + + +/// A single binding to a joint in an FBX document. +class FBXCluster { +public: + + int jointIndex; + glm::mat4 inverseBindMatrix; +}; + +const int MAX_NUM_PIXELS_FOR_FBX_TEXTURE = 2048 * 2048; + +/// A texture map in an FBX document. +class FBXTexture { +public: + QString name; + QByteArray filename; + QByteArray content; + + Transform transform; + int maxNumPixels { MAX_NUM_PIXELS_FOR_FBX_TEXTURE }; + int texcoordSet; + QString texcoordSetName; + + bool isBumpmap{ false }; + + bool isNull() const { return name.isEmpty() && filename.isEmpty() && content.isEmpty(); } +}; + +/// A single part of a mesh (with the same material). +class FBXMeshPart { +public: + + QVector quadIndices; // original indices from the FBX mesh + QVector quadTrianglesIndices; // original indices from the FBX mesh of the quad converted as triangles + QVector triangleIndices; // original indices from the FBX mesh + + QString materialID; +}; + +class FBXMaterial { +public: + FBXMaterial() {}; + FBXMaterial(const glm::vec3& diffuseColor, const glm::vec3& specularColor, const glm::vec3& emissiveColor, + float shininess, float opacity) : + diffuseColor(diffuseColor), + specularColor(specularColor), + emissiveColor(emissiveColor), + shininess(shininess), + opacity(opacity) {} + + void getTextureNames(QSet& textureList) const; + void setMaxNumPixelsPerTexture(int maxNumPixels); + + glm::vec3 diffuseColor{ 1.0f }; + float diffuseFactor{ 1.0f }; + glm::vec3 specularColor{ 0.02f }; + float specularFactor{ 1.0f }; + + glm::vec3 emissiveColor{ 0.0f }; + float emissiveFactor{ 0.0f }; + + float shininess{ 23.0f }; + float opacity{ 1.0f }; + + float metallic{ 0.0f }; + float roughness{ 1.0f }; + float emissiveIntensity{ 1.0f }; + float ambientFactor{ 1.0f }; + + QString materialID; + QString name; + QString shadingModel; + model::MaterialPointer _material; + + FBXTexture normalTexture; + FBXTexture albedoTexture; + FBXTexture opacityTexture; + FBXTexture glossTexture; + FBXTexture roughnessTexture; + FBXTexture specularTexture; + FBXTexture metallicTexture; + FBXTexture emissiveTexture; + FBXTexture occlusionTexture; + FBXTexture scatteringTexture; + FBXTexture lightmapTexture; + glm::vec2 lightmapParams{ 0.0f, 1.0f }; + + + bool isPBSMaterial{ false }; + // THe use XXXMap are not really used to drive which map are going or not, debug only + bool useNormalMap{ false }; + bool useAlbedoMap{ false }; + bool useOpacityMap{ false }; + bool useRoughnessMap{ false }; + bool useSpecularMap{ false }; + bool useMetallicMap{ false }; + bool useEmissiveMap{ false }; + bool useOcclusionMap{ false }; + + bool needTangentSpace() const; +}; + +/// A single mesh (with optional blendshapes) extracted from an FBX document. +class FBXMesh { +public: + + QVector parts; + + QVector vertices; + QVector normals; + QVector tangents; + QVector colors; + QVector texCoords; + QVector texCoords1; + QVector clusterIndices; + QVector clusterWeights; + + QVector clusters; + + Extents meshExtents; + glm::mat4 modelTransform; + + QVector blendshapes; + + unsigned int meshIndex; // the order the meshes appeared in the object file + + model::MeshPointer _mesh; +}; + +class ExtractedMesh { +public: + FBXMesh mesh; + QMultiHash newIndices; + QVector > blendshapeIndexMaps; + QVector > partMaterialTextures; + QHash texcoordSetMap; +}; + +/// A single animation frame extracted from an FBX document. +class FBXAnimationFrame { +public: + QVector rotations; + QVector translations; +}; + +/// A light in an FBX document. +class FBXLight { +public: + QString name; + Transform transform; + float intensity; + float fogValue; + glm::vec3 color; + + FBXLight() : + name(), + transform(), + intensity(1.0f), + fogValue(0.0f), + color(1.0f) + {} +}; + +Q_DECLARE_METATYPE(FBXAnimationFrame) +Q_DECLARE_METATYPE(QVector) + +/// A set of meshes extracted from an FBX document. +class FBXGeometry { +public: + using Pointer = std::shared_ptr; + + QString originalURL; + QString author; + QString applicationName; ///< the name of the application that generated the model + + QVector joints; + QHash jointIndices; ///< 1-based, so as to more easily detect missing indices + bool hasSkeletonJoints; + + QVector meshes; + + QHash materials; + + glm::mat4 offset; // This includes offset, rotation, and scale as specified by the FST file + + int leftEyeJointIndex = -1; + int rightEyeJointIndex = -1; + int neckJointIndex = -1; + int rootJointIndex = -1; + int leanJointIndex = -1; + int headJointIndex = -1; + int leftHandJointIndex = -1; + int rightHandJointIndex = -1; + int leftToeJointIndex = -1; + int rightToeJointIndex = -1; + + float leftEyeSize = 0.0f; // Maximum mesh extents dimension + float rightEyeSize = 0.0f; + + QVector humanIKJointIndices; + + glm::vec3 palmDirection; + + glm::vec3 neckPivot; + + Extents bindExtents; + Extents meshExtents; + + QVector animationFrames; + + int getJointIndex(const QString& name) const { return jointIndices.value(name) - 1; } + QStringList getJointNames() const; + + bool hasBlendedMeshes() const; + + /// Returns the unscaled extents of the model's mesh + Extents getUnscaledMeshExtents() const; + + bool convexHullContains(const glm::vec3& point) const; + + QHash meshIndicesToModelNames; + + /// given a meshIndex this will return the name of the model that mesh belongs to if known + QString getModelNameOfMesh(int meshIndex) const; + + QList blendshapeChannelNames; +}; + +Q_DECLARE_METATYPE(FBXGeometry) +Q_DECLARE_METATYPE(FBXGeometry::Pointer) + +#endif // hifi_FBX_h_ diff --git a/libraries/fbx/src/FBXReader.cpp b/libraries/fbx/src/FBXReader.cpp index 450aa296d8..ea4b00523e 100644 --- a/libraries/fbx/src/FBXReader.cpp +++ b/libraries/fbx/src/FBXReader.cpp @@ -168,7 +168,8 @@ QString getID(const QVariantList& properties, int index = 0) { return processID(properties.at(index).toString()); } -const char* HUMANIK_JOINTS[] = { +/// The names of the joints in the Maya HumanIK rig +static const std::array HUMANIK_JOINTS = { "RightHand", "RightForeArm", "RightArm", @@ -184,8 +185,7 @@ const char* HUMANIK_JOINTS[] = { "RightLeg", "LeftLeg", "RightFoot", - "LeftFoot", - "" + "LeftFoot" }; class FBXModel { @@ -512,11 +512,8 @@ FBXGeometry* FBXReader::extractFBXGeometry(const QVariantHash& mapping, const QS QVector humanIKJointNames; - for (int i = 0;; i++) { + for (int i = 0; i < HUMANIK_JOINTS.size(); i++) { QByteArray jointName = HUMANIK_JOINTS[i]; - if (jointName.isEmpty()) { - break; - } humanIKJointNames.append(processID(getString(joints.value(jointName, jointName)))); } QVector humanIKJointIDs(humanIKJointNames.size()); diff --git a/libraries/fbx/src/FBXReader.h b/libraries/fbx/src/FBXReader.h index 170bbbf366..a600ac6bab 100644 --- a/libraries/fbx/src/FBXReader.h +++ b/libraries/fbx/src/FBXReader.h @@ -12,6 +12,8 @@ #ifndef hifi_FBXReader_h #define hifi_FBXReader_h +#include "FBX.h" + #include #include #include @@ -31,305 +33,6 @@ class QIODevice; class FBXNode; -typedef QList FBXNodeList; - -/// The names of the joints in the Maya HumanIK rig, terminated with an empty string. -extern const char* HUMANIK_JOINTS[]; - -/// A node within an FBX document. -class FBXNode { -public: - - QByteArray name; - QVariantList properties; - FBXNodeList children; -}; - -/// A single blendshape extracted from an FBX document. -class FBXBlendshape { -public: - - QVector indices; - QVector vertices; - QVector normals; -}; - -struct FBXJointShapeInfo { - // same units and frame as FBXJoint.translation - glm::vec3 avgPoint; - std::vector dots; - std::vector points; - std::vector debugLines; -}; - -/// A single joint (transformation node) extracted from an FBX document. -class FBXJoint { -public: - - FBXJointShapeInfo shapeInfo; - QVector freeLineage; - bool isFree; - int parentIndex; - float distanceToParent; - - // http://download.autodesk.com/us/fbx/20112/FBX_SDK_HELP/SDKRef/a00209.html - - glm::vec3 translation; // T - glm::mat4 preTransform; // Roff * Rp - glm::quat preRotation; // Rpre - glm::quat rotation; // R - glm::quat postRotation; // Rpost - glm::mat4 postTransform; // Rp-1 * Soff * Sp * S * Sp-1 - - // World = ParentWorld * T * (Roff * Rp) * Rpre * R * Rpost * (Rp-1 * Soff * Sp * S * Sp-1) - - glm::mat4 transform; - glm::vec3 rotationMin; // radians - glm::vec3 rotationMax; // radians - glm::quat inverseDefaultRotation; - glm::quat inverseBindRotation; - glm::mat4 bindTransform; - QString name; - bool isSkeletonJoint; - bool bindTransformFoundInCluster; - - // geometric offset is applied in local space but does NOT affect children. - bool hasGeometricOffset; - glm::vec3 geometricTranslation; - glm::quat geometricRotation; - glm::vec3 geometricScaling; -}; - - -/// A single binding to a joint in an FBX document. -class FBXCluster { -public: - - int jointIndex; - glm::mat4 inverseBindMatrix; -}; - -const int MAX_NUM_PIXELS_FOR_FBX_TEXTURE = 2048 * 2048; - -/// A texture map in an FBX document. -class FBXTexture { -public: - QString name; - QByteArray filename; - QByteArray content; - - Transform transform; - int maxNumPixels { MAX_NUM_PIXELS_FOR_FBX_TEXTURE }; - int texcoordSet; - QString texcoordSetName; - - bool isBumpmap{ false }; - - bool isNull() const { return name.isEmpty() && filename.isEmpty() && content.isEmpty(); } -}; - -/// A single part of a mesh (with the same material). -class FBXMeshPart { -public: - - QVector quadIndices; // original indices from the FBX mesh - QVector quadTrianglesIndices; // original indices from the FBX mesh of the quad converted as triangles - QVector triangleIndices; // original indices from the FBX mesh - - QString materialID; -}; - -class FBXMaterial { -public: - FBXMaterial() {}; - FBXMaterial(const glm::vec3& diffuseColor, const glm::vec3& specularColor, const glm::vec3& emissiveColor, - float shininess, float opacity) : - diffuseColor(diffuseColor), - specularColor(specularColor), - emissiveColor(emissiveColor), - shininess(shininess), - opacity(opacity) {} - - void getTextureNames(QSet& textureList) const; - void setMaxNumPixelsPerTexture(int maxNumPixels); - - glm::vec3 diffuseColor{ 1.0f }; - float diffuseFactor{ 1.0f }; - glm::vec3 specularColor{ 0.02f }; - float specularFactor{ 1.0f }; - - glm::vec3 emissiveColor{ 0.0f }; - float emissiveFactor{ 0.0f }; - - float shininess{ 23.0f }; - float opacity{ 1.0f }; - - float metallic{ 0.0f }; - float roughness{ 1.0f }; - float emissiveIntensity{ 1.0f }; - float ambientFactor{ 1.0f }; - - QString materialID; - QString name; - QString shadingModel; - model::MaterialPointer _material; - - FBXTexture normalTexture; - FBXTexture albedoTexture; - FBXTexture opacityTexture; - FBXTexture glossTexture; - FBXTexture roughnessTexture; - FBXTexture specularTexture; - FBXTexture metallicTexture; - FBXTexture emissiveTexture; - FBXTexture occlusionTexture; - FBXTexture scatteringTexture; - FBXTexture lightmapTexture; - glm::vec2 lightmapParams{ 0.0f, 1.0f }; - - - bool isPBSMaterial{ false }; - // THe use XXXMap are not really used to drive which map are going or not, debug only - bool useNormalMap{ false }; - bool useAlbedoMap{ false }; - bool useOpacityMap{ false }; - bool useRoughnessMap{ false }; - bool useSpecularMap{ false }; - bool useMetallicMap{ false }; - bool useEmissiveMap{ false }; - bool useOcclusionMap{ false }; - - bool needTangentSpace() const; -}; - -/// A single mesh (with optional blendshapes) extracted from an FBX document. -class FBXMesh { -public: - - QVector parts; - - QVector vertices; - QVector normals; - QVector tangents; - QVector colors; - QVector texCoords; - QVector texCoords1; - QVector clusterIndices; - QVector clusterWeights; - - QVector clusters; - - Extents meshExtents; - glm::mat4 modelTransform; - - QVector blendshapes; - - unsigned int meshIndex; // the order the meshes appeared in the object file - - model::MeshPointer _mesh; -}; - -class ExtractedMesh { -public: - FBXMesh mesh; - QMultiHash newIndices; - QVector > blendshapeIndexMaps; - QVector > partMaterialTextures; - QHash texcoordSetMap; -}; - -/// A single animation frame extracted from an FBX document. -class FBXAnimationFrame { -public: - QVector rotations; - QVector translations; -}; - -/// A light in an FBX document. -class FBXLight { -public: - QString name; - Transform transform; - float intensity; - float fogValue; - glm::vec3 color; - - FBXLight() : - name(), - transform(), - intensity(1.0f), - fogValue(0.0f), - color(1.0f) - {} -}; - -Q_DECLARE_METATYPE(FBXAnimationFrame) -Q_DECLARE_METATYPE(QVector) - -/// A set of meshes extracted from an FBX document. -class FBXGeometry { -public: - using Pointer = std::shared_ptr; - - QString originalURL; - QString author; - QString applicationName; ///< the name of the application that generated the model - - QVector joints; - QHash jointIndices; ///< 1-based, so as to more easily detect missing indices - bool hasSkeletonJoints; - - QVector meshes; - - QHash materials; - - glm::mat4 offset; // This includes offset, rotation, and scale as specified by the FST file - - int leftEyeJointIndex = -1; - int rightEyeJointIndex = -1; - int neckJointIndex = -1; - int rootJointIndex = -1; - int leanJointIndex = -1; - int headJointIndex = -1; - int leftHandJointIndex = -1; - int rightHandJointIndex = -1; - int leftToeJointIndex = -1; - int rightToeJointIndex = -1; - - float leftEyeSize = 0.0f; // Maximum mesh extents dimension - float rightEyeSize = 0.0f; - - QVector humanIKJointIndices; - - glm::vec3 palmDirection; - - glm::vec3 neckPivot; - - Extents bindExtents; - Extents meshExtents; - - QVector animationFrames; - - int getJointIndex(const QString& name) const { return jointIndices.value(name) - 1; } - QStringList getJointNames() const; - - bool hasBlendedMeshes() const; - - /// Returns the unscaled extents of the model's mesh - Extents getUnscaledMeshExtents() const; - - bool convexHullContains(const glm::vec3& point) const; - - QHash meshIndicesToModelNames; - - /// given a meshIndex this will return the name of the model that mesh belongs to if known - QString getModelNameOfMesh(int meshIndex) const; - - QList blendshapeChannelNames; -}; - -Q_DECLARE_METATYPE(FBXGeometry) -Q_DECLARE_METATYPE(FBXGeometry::Pointer) /// Reads FBX geometry from the supplied model and mapping data. /// \exception QString if an error occurs in parsing @@ -402,7 +105,7 @@ class FBXReader { public: FBXGeometry* _fbxGeometry; - FBXNode _fbxNode; + FBXNode _rootNode; static FBXNode parseFBX(QIODevice* device); FBXGeometry* extractFBXGeometry(const QVariantHash& mapping, const QString& url); diff --git a/libraries/fbx/src/FBXReader_Node.cpp b/libraries/fbx/src/FBXReader_Node.cpp index d987f885eb..111b4a295a 100644 --- a/libraries/fbx/src/FBXReader_Node.cpp +++ b/libraries/fbx/src/FBXReader_Node.cpp @@ -24,15 +24,18 @@ #include #include "ModelFormatLogging.h" -template int streamSize() { +template +int streamSize() { return sizeof(T); } -template int streamSize() { +template +int streamSize() { return 1; } -template QVariant readBinaryArray(QDataStream& in, int& position) { +template +QVariant readBinaryArray(QDataStream& in, int& position) { quint32 arrayLength; quint32 encoding; quint32 compressedLength; @@ -350,8 +353,7 @@ FBXNode parseTextFBXNode(Tokenizer& tokenizer) { FBXNode FBXReader::parseFBX(QIODevice* device) { PROFILE_RANGE_EX(resource_parse, __FUNCTION__, 0xff0000ff, device); // verify the prolog - const QByteArray BINARY_PROLOG = "Kaydara FBX Binary "; - if (device->peek(BINARY_PROLOG.size()) != BINARY_PROLOG) { + if (device->peek(FBX_BINARY_PROLOG.size()) != FBX_BINARY_PROLOG) { // parse as a text file FBXNode top; Tokenizer tokenizer(device); @@ -377,15 +379,13 @@ FBXNode FBXReader::parseFBX(QIODevice* device) { // Bytes 0 - 20: Kaydara FBX Binary \x00(file - magic, with 2 spaces at the end, then a NULL terminator). // Bytes 21 - 22: [0x1A, 0x00](unknown but all observed files show these bytes). // Bytes 23 - 26 : unsigned int, the version number. 7300 for version 7.3 for example. - const int HEADER_BEFORE_VERSION = 23; - const quint32 VERSION_FBX2016 = 7500; - in.skipRawData(HEADER_BEFORE_VERSION); - int position = HEADER_BEFORE_VERSION; + in.skipRawData(FBX_HEADER_BYTES_BEFORE_VERSION); + int position = FBX_HEADER_BYTES_BEFORE_VERSION; quint32 fileVersion; in >> fileVersion; position += sizeof(fileVersion); qCDebug(modelformat) << "fileVersion:" << fileVersion; - bool has64BitPositions = (fileVersion >= VERSION_FBX2016); + bool has64BitPositions = (fileVersion >= FBX_VERSION_2016); // parse the top-level node FBXNode top; diff --git a/libraries/fbx/src/FBXWriter.cpp b/libraries/fbx/src/FBXWriter.cpp new file mode 100644 index 0000000000..a084d94a46 --- /dev/null +++ b/libraries/fbx/src/FBXWriter.cpp @@ -0,0 +1,253 @@ +// +// FBXWriter.cpp +// libraries/fbx/src +// +// Created by Ryan Huffman on 9/5/17. +// Copyright 2017 High Fidelity, Inc. +// +// Distributed under the Apache License, Version 2.0. +// See the accompanying file LICENSE or http://www.apache.org/licenses/LICENSE-2.0.html +// + +#include "FBXWriter.h" + +#include + + +QByteArray FBXWriter::encodeFBX(const FBXNode& root) { + QByteArray data; + QDataStream out(&data, QIODevice::WriteOnly); + out.setByteOrder(QDataStream::LittleEndian); + out.setVersion(QDataStream::Qt_4_5); + + out.writeRawData(FBX_BINARY_PROLOG, FBX_BINARY_PROLOG.size()); + auto bytes = QByteArray(FBX_HEADER_BYTES_BEFORE_VERSION - FBX_BINARY_PROLOG.size(), '\0'); + out.writeRawData(bytes, bytes.size()); + + out << FBX_VERSION_2016; + + for (auto& child : root.children) { + encodeNode(out, child); + } + encodeNode(out, FBXNode()); + + return data; +} + +void FBXWriter::encodeNode(QDataStream& out, const FBXNode& node) { + qDebug() << "Encoding " << node.name; + + auto device = out.device(); + auto nodeStartPos = device->pos(); + + // endOffset (temporary, updated later) + out << (qint64)0; + + // Property count + out << (quint64)node.properties.size(); + + // Property list length (temporary, updated later) + out << (quint64)0; + + out << (quint8)node.name.size(); + out.writeRawData(node.name, node.name.size()); + + if (node.name == "Vertices") { + for (auto& prop : node.properties) { + qDebug() << "Properties: " << prop; + } + } + + auto nodePropertiesStartPos = device->pos(); + + for (const auto& prop : node.properties) { + encodeFBXProperty(out, prop); + } + + // Go back and write property list length + auto nodePropertiesEndPos = device->pos(); + device->seek(nodeStartPos + sizeof(qint64) + sizeof(quint64)); + out << (quint64)(nodePropertiesEndPos - nodePropertiesStartPos); + + device->seek(nodePropertiesEndPos); + + for (auto& child : node.children) { + encodeNode(out, child); + } + + if (node.children.length() > 0) { + encodeNode(out, FBXNode()); + } + + // Go back and write actual endOffset + auto nodeEndPos = device->pos(); + device->seek(nodeStartPos); + out << (qint64)(nodeEndPos); + + device->seek(nodeEndPos); +} + +void FBXWriter::encodeFBXProperty(QDataStream& out, const QVariant& prop) { + auto type = prop.userType(); + switch (type) { + case QVariant::Type::Bool: + + out.device()->write("C", 1); + out << prop.toBool(); + break; + + case QMetaType::Int: + out.device()->write("I", 1); + out << prop.toInt(); + break; + + encodeNode(out, FBXNode()); + case QMetaType::Float: + out.device()->write("F", 1); + out << prop.toFloat(); + break; + + case QMetaType::Double: + out.device()->write("D", 1); + out << prop.toDouble(); + break; + + case QMetaType::LongLong: + out.device()->write("L", 1); + out << prop.toLongLong(); + break; + + case QMetaType::QString: + { + auto& bytes = prop.toString().toUtf8(); + out << 'S'; + out << bytes.length(); + out << bytes; + out << (int32_t)bytes.size(); + out.writeRawData(bytes, bytes.size()); + break; + } + + case QMetaType::QByteArray: + { + auto& bytes = prop.toByteArray(); + out.device()->write("S", 1); + out << (int32_t)bytes.size(); + out.writeRawData(bytes, bytes.size()); + break; + } + + // TODO Delete? Do we ever use QList instead of QVector? + case QVariant::Type::List: + { + auto& list = prop.toList(); + auto listType = prop.userType(); + + switch (listType) { + case QMetaType::Float: + out.device()->write("f", 1); + out << (int32_t)list.length(); + out << (int32_t)0; + out << (int32_t)0; + for (auto& innerProp : list) { + out << prop.toFloat(); + } + break; + + case QMetaType::Double: + out.device()->write("d", 1); + out << (int32_t)list.length(); + out << (int32_t)0; + out << (int32_t)0; + for (auto& innerProp : list) { + out << prop.toDouble(); + } + break; + + case QMetaType::LongLong: + out.device()->write("l", 1); + out << (int32_t)list.length(); + out << (int32_t)0; + out << (int32_t)0; + for (auto& innerProp : list) { + out << prop.toLongLong(); + } + break; + + case QMetaType::Int: + out.device()->write("i", 1); + out << (int32_t)list.length(); + out << (int32_t)0; + out << (int32_t)0; + for (auto& innerProp : list) { + out << prop.toInt(); + } + break; + + case QMetaType::Bool: + out.device()->write("b", 1); + out << (int32_t)list.length(); + out << (int32_t)0; + out << (int32_t)0; + for (auto& innerProp : list) { + out << prop.toBool(); + } + break; + } + } + break; + + default: + { + if (prop.canConvert>()) { + auto list = prop.value>(); + out.device()->write("f", 1); + out << (int32_t)list.length(); + out << (int32_t)0; + out << (int32_t)0; + for (auto& value : list) { + out << value; + } + } else if (prop.canConvert>()) { + auto list = prop.value>(); + out.device()->write("d", 1); + out << (int32_t)list.length(); + out << (int32_t)0; + out << (int32_t)0; + for (auto& value : list) { + out << value; + } + } else if (prop.canConvert>()) { + auto list = prop.value>(); + out.device()->write("l", 1); + out << (int32_t)list.length(); + out << (int32_t)0; + out << (int32_t)0; + for (auto& value : list) { + out << value; + } + } else if (prop.canConvert>()) { + auto list = prop.value>(); + out.device()->write("i", 1); + out << (int32_t)list.length(); + out << (int32_t)0; + out << (int32_t)0; + for (auto& value : list) { + out << value; + } + } else if (prop.canConvert>()) { + auto list = prop.value>(); + out.device()->write("b", 1); + out << (int32_t)list.length(); + out << (int32_t)0; + out << (int32_t)0; + for (auto& value : list) { + out << value; + } + } else { + qDebug() << "Unsupported property type in FBXWriter::encodeNode: " << type << prop; + } + } + + } +} diff --git a/libraries/fbx/src/FBXWriter.h b/libraries/fbx/src/FBXWriter.h new file mode 100644 index 0000000000..fa33983345 --- /dev/null +++ b/libraries/fbx/src/FBXWriter.h @@ -0,0 +1,28 @@ +// +// FBXWriter.h +// libraries/fbx/src +// +// Created by Ryan Huffman on 9/5/17. +// Copyright 2017 High Fidelity, Inc. +// +// Distributed under the Apache License, Version 2.0. +// See the accompanying file LICENSE or http://www.apache.org/licenses/LICENSE-2.0.html +// + +#ifndef hifi_FBXWriter_h +#define hifi_FBXWriter_h + +#include "FBX.h" + +#include +#include + +class FBXWriter { +public: + static QByteArray encodeFBX(const FBXNode& root); + + static void encodeNode(QDataStream& out, const FBXNode& node); + static void encodeFBXProperty(QDataStream& out, const QVariant& property); +}; + +#endif // hifi_FBXWriter_h From 7a55c867be3ee6b7a3c55455ff1533abf8daed3e Mon Sep 17 00:00:00 2001 From: Ryan Huffman Date: Thu, 7 Sep 2017 11:49:37 -0700 Subject: [PATCH 063/129] Fix _fbxNode rename to _rootNode in FBXReader --- libraries/fbx/src/FBXReader.cpp | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/libraries/fbx/src/FBXReader.cpp b/libraries/fbx/src/FBXReader.cpp index ea4b00523e..d212ec820f 100644 --- a/libraries/fbx/src/FBXReader.cpp +++ b/libraries/fbx/src/FBXReader.cpp @@ -468,7 +468,7 @@ QByteArray fileOnUrl(const QByteArray& filepath, const QString& url) { } FBXGeometry* FBXReader::extractFBXGeometry(const QVariantHash& mapping, const QString& url) { - const FBXNode& node = _fbxNode; + const FBXNode& node = _rootNode; QMap meshes; QHash modelIDsToNames; QHash meshIDsToMeshIndices; @@ -1839,7 +1839,7 @@ FBXGeometry* readFBX(const QByteArray& model, const QVariantHash& mapping, const FBXGeometry* readFBX(QIODevice* device, const QVariantHash& mapping, const QString& url, bool loadLightmaps, float lightmapLevel) { FBXReader reader; - reader._fbxNode = FBXReader::parseFBX(device); + reader._rootNode = FBXReader::parseFBX(device); reader._loadLightmaps = loadLightmaps; reader._lightmapLevel = lightmapLevel; From 49e09f483872c5ccca2d1137470a87099a29abfc Mon Sep 17 00:00:00 2001 From: Ryan Huffman Date: Thu, 7 Sep 2017 13:12:07 -0700 Subject: [PATCH 064/129] Remove FBXBaker and TextureBaker from oven --- tools/oven/src/FBXBaker.cpp | 568 -------------------------------- tools/oven/src/FBXBaker.h | 103 ------ tools/oven/src/TextureBaker.cpp | 131 -------- tools/oven/src/TextureBaker.h | 59 ---- 4 files changed, 861 deletions(-) delete mode 100644 tools/oven/src/FBXBaker.cpp delete mode 100644 tools/oven/src/FBXBaker.h delete mode 100644 tools/oven/src/TextureBaker.cpp delete mode 100644 tools/oven/src/TextureBaker.h diff --git a/tools/oven/src/FBXBaker.cpp b/tools/oven/src/FBXBaker.cpp deleted file mode 100644 index 8ece76b6c4..0000000000 --- a/tools/oven/src/FBXBaker.cpp +++ /dev/null @@ -1,568 +0,0 @@ -// -// FBXBaker.cpp -// tools/oven/src -// -// Created by Stephen Birarda on 3/30/17. -// Copyright 2017 High Fidelity, Inc. -// -// Distributed under the Apache License, Version 2.0. -// See the accompanying file LICENSE or http://www.apache.org/licenses/LICENSE-2.0.html -// - -#include // need this include so we don't get an error looking for std::isnan - -#include - -#include -#include -#include -#include -#include -#include - -#include - -#include -#include - -#include - -#include "ModelBakingLoggingCategory.h" -#include "TextureBaker.h" - -#include "FBXBaker.h" - -std::once_flag onceFlag; -FBXSDKManagerUniquePointer FBXBaker::_sdkManager { nullptr }; - -FBXBaker::FBXBaker(const QUrl& fbxURL, TextureBakerThreadGetter textureThreadGetter, - const QString& bakedOutputDir, const QString& originalOutputDir) : - _fbxURL(fbxURL), - _bakedOutputDir(bakedOutputDir), - _originalOutputDir(originalOutputDir), - _textureThreadGetter(textureThreadGetter) -{ - std::call_once(onceFlag, [](){ - // create the static FBX SDK manager - _sdkManager = FBXSDKManagerUniquePointer(FbxManager::Create(), [](FbxManager* manager){ - manager->Destroy(); - }); - }); -} - -void FBXBaker::bake() { - auto tempDir = PathUtils::generateTemporaryDir(); - - if (tempDir.isEmpty()) { - handleError("Failed to create a temporary directory."); - return; - } - - _tempDir = tempDir; - - _originalFBXFilePath = _tempDir.filePath(_fbxURL.fileName()); - qDebug() << "Made temporary dir " << _tempDir; - qDebug() << "Origin file path: " << _originalFBXFilePath; - - // setup the output folder for the results of this bake - setupOutputFolder(); - - if (hasErrors()) { - return; - } - - connect(this, &FBXBaker::sourceCopyReadyToLoad, this, &FBXBaker::bakeSourceCopy); - - // make a local copy of the FBX file - loadSourceFBX(); -} - -void FBXBaker::bakeSourceCopy() { - // load the scene from the FBX file - importScene(); - - if (hasErrors()) { - return; - } - - // enumerate the textures found in the scene and start a bake for them - rewriteAndBakeSceneTextures(); - - if (hasErrors()) { - return; - } - - // export the FBX with re-written texture references - exportScene(); - - if (hasErrors()) { - return; - } - - // check if we're already done with textures (in case we had none to re-write) - checkIfTexturesFinished(); -} - -void FBXBaker::setupOutputFolder() { - // make sure there isn't already an output directory using the same name - int iteration = 0; - - if (QDir(_bakedOutputDir).exists()) { - qWarning() << "Output path" << _bakedOutputDir << "already exists. Continuing."; - //_bakedOutputDir = _baseOutputPath + "/" + _fbxName + "-" + QString::number(++iteration) + "/"; - } else { - qCDebug(model_baking) << "Creating FBX output folder" << _bakedOutputDir; - - // attempt to make the output folder - if (!QDir().mkpath(_bakedOutputDir)) { - handleError("Failed to create FBX output folder " + _bakedOutputDir); - return; - } - // attempt to make the output folder - if (!QDir().mkpath(_originalOutputDir)) { - handleError("Failed to create FBX output folder " + _bakedOutputDir); - return; - } - } -} - -void FBXBaker::loadSourceFBX() { - // check if the FBX is local or first needs to be downloaded - if (_fbxURL.isLocalFile()) { - // load up the local file - QFile localFBX { _fbxURL.toLocalFile() }; - - qDebug() << "Local file url: " << _fbxURL << _fbxURL.toString() << _fbxURL.toLocalFile() << ", copying to: " << _originalFBXFilePath; - - if (!localFBX.exists()) { - //QMessageBox::warning(this, "Could not find " + _fbxURL.toString(), ""); - handleError("Could not find " + _fbxURL.toString()); - return; - } - - // make a copy in the output folder - if (!_originalOutputDir.isEmpty()) { - qDebug() << "Copying to: " << _originalOutputDir << "/" << _fbxURL.fileName(); - localFBX.copy(_originalOutputDir + "/" + _fbxURL.fileName()); - } - - localFBX.copy(_originalFBXFilePath); - - // emit our signal to start the import of the FBX source copy - emit sourceCopyReadyToLoad(); - } else { - // remote file, kick off a download - auto& networkAccessManager = NetworkAccessManager::getInstance(); - - QNetworkRequest networkRequest; - - // setup the request to follow re-directs and always hit the network - networkRequest.setAttribute(QNetworkRequest::FollowRedirectsAttribute, true); - networkRequest.setAttribute(QNetworkRequest::CacheLoadControlAttribute, QNetworkRequest::AlwaysNetwork); - networkRequest.setHeader(QNetworkRequest::UserAgentHeader, HIGH_FIDELITY_USER_AGENT); - - - networkRequest.setUrl(_fbxURL); - - qCDebug(model_baking) << "Downloading" << _fbxURL; - auto networkReply = networkAccessManager.get(networkRequest); - - connect(networkReply, &QNetworkReply::finished, this, &FBXBaker::handleFBXNetworkReply); - } -} - -void FBXBaker::handleFBXNetworkReply() { - auto requestReply = qobject_cast(sender()); - - if (requestReply->error() == QNetworkReply::NoError) { - qCDebug(model_baking) << "Downloaded" << _fbxURL; - - // grab the contents of the reply and make a copy in the output folder - QFile copyOfOriginal(_originalFBXFilePath); - - qDebug(model_baking) << "Writing copy of original FBX to" << _originalFBXFilePath << copyOfOriginal.fileName(); - - if (!copyOfOriginal.open(QIODevice::WriteOnly)) { - // add an error to the error list for this FBX stating that a duplicate of the original FBX could not be made - handleError("Could not create copy of " + _fbxURL.toString() + " (Failed to open " + _originalFBXFilePath + ")"); - return; - } - if (copyOfOriginal.write(requestReply->readAll()) == -1) { - handleError("Could not create copy of " + _fbxURL.toString() + " (Failed to write)"); - return; - } - - // close that file now that we are done writing to it - copyOfOriginal.close(); - - if (!_originalOutputDir.isEmpty()) { - copyOfOriginal.copy(_originalOutputDir + "/" + _fbxURL.fileName()); - } - - // emit our signal to start the import of the FBX source copy - emit sourceCopyReadyToLoad(); - } else { - // add an error to our list stating that the FBX could not be downloaded - handleError("Failed to download " + _fbxURL.toString()); - } -} - -void FBXBaker::importScene() { - // create an FBX SDK importer - FbxImporter* importer = FbxImporter::Create(_sdkManager.get(), ""); - - qDebug() << "file path: " << _originalFBXFilePath.toLocal8Bit().data() << QDir(_originalFBXFilePath).exists(); - // import the copy of the original FBX file - bool importStatus = importer->Initialize(_originalFBXFilePath.toLocal8Bit().data()); - - if (!importStatus) { - // failed to initialize importer, print an error and return - handleError("Failed to import " + _fbxURL.toString() + " - " + importer->GetStatus().GetErrorString()); - return; - } else { - qCDebug(model_baking) << "Imported" << _fbxURL << "to FbxScene"; - } - - // setup a new scene to hold the imported file - _scene = FbxScene::Create(_sdkManager.get(), "bakeScene"); - - // import the file to the created scene - importer->Import(_scene); - - // destroy the importer that is no longer needed - importer->Destroy(); -} - -QString texturePathRelativeToFBX(QUrl fbxURL, QUrl textureURL) { - auto fbxPath = fbxURL.toString(QUrl::RemoveFilename | QUrl::RemoveQuery | QUrl::RemoveFragment); - auto texturePath = textureURL.toString(QUrl::RemoveFilename | QUrl::RemoveQuery | QUrl::RemoveFragment); - - if (texturePath.startsWith(fbxPath)) { - // texture path is a child of the FBX path, return the texture path without the fbx path - return texturePath.mid(fbxPath.length()); - } else { - // the texture path was not a child of the FBX path, return the empty string - return ""; - } -} - -QString FBXBaker::createBakedTextureFileName(const QFileInfo& textureFileInfo) { - // first make sure we have a unique base name for this texture - // in case another texture referenced by this model has the same base name - auto nameMatches = _textureNameMatchCount[textureFileInfo.baseName()]; - - QString bakedTextureFileName { textureFileInfo.completeBaseName() }; - - if (nameMatches > 0) { - // there are already nameMatches texture with this name - // append - and that number to our baked texture file name so that it is unique - bakedTextureFileName += "-" + QString::number(nameMatches); - } - - bakedTextureFileName += BAKED_TEXTURE_EXT; - - // increment the number of name matches - ++nameMatches; - - return bakedTextureFileName; -} - -QUrl FBXBaker::getTextureURL(const QFileInfo& textureFileInfo, FbxFileTexture* fileTexture) { - QUrl urlToTexture; - - if (textureFileInfo.exists() && textureFileInfo.isFile()) { - // set the texture URL to the local texture that we have confirmed exists - urlToTexture = QUrl::fromLocalFile(textureFileInfo.absoluteFilePath()); - } else { - // external texture that we'll need to download or find - - // first check if it the RelativePath to the texture in the FBX was relative - QString relativeFileName = fileTexture->GetRelativeFileName(); - auto apparentRelativePath = QFileInfo(relativeFileName.replace("\\", "/")); - - // this is a relative file path which will require different handling - // depending on the location of the original FBX - if (_fbxURL.isLocalFile() && apparentRelativePath.exists() && apparentRelativePath.isFile()) { - // the absolute path we ran into for the texture in the FBX exists on this machine - // so use that file - urlToTexture = QUrl::fromLocalFile(apparentRelativePath.absoluteFilePath()); - } else { - // we didn't find the texture on this machine at the absolute path - // so assume that it is right beside the FBX to match the behaviour of interface - urlToTexture = _fbxURL.resolved(apparentRelativePath.fileName()); - } - } - - return urlToTexture; -} - -image::TextureUsage::Type textureTypeForMaterialProperty(FbxProperty& property, FbxSurfaceMaterial* material) { - using namespace image::TextureUsage; - - // this is a property we know has a texture, we need to match it to a High Fidelity known texture type - // since that information is passed to the baking process - - // grab the hierarchical name for this property and lowercase it for case-insensitive compare - auto propertyName = QString(property.GetHierarchicalName()).toLower(); - - // figure out the type of the property based on what known value string it matches - if ((propertyName.contains("diffuse") && !propertyName.contains("tex_global_diffuse")) - || propertyName.contains("tex_color_map")) { - return ALBEDO_TEXTURE; - } else if (propertyName.contains("transparentcolor") || propertyName.contains("transparencyfactor")) { - return ALBEDO_TEXTURE; - } else if (propertyName.contains("bump")) { - return BUMP_TEXTURE; - } else if (propertyName.contains("normal")) { - return NORMAL_TEXTURE; - } else if ((propertyName.contains("specular") && !propertyName.contains("tex_global_specular")) - || propertyName.contains("reflection")) { - return SPECULAR_TEXTURE; - } else if (propertyName.contains("tex_metallic_map")) { - return METALLIC_TEXTURE; - } else if (propertyName.contains("shininess")) { - return GLOSS_TEXTURE; - } else if (propertyName.contains("tex_roughness_map")) { - return ROUGHNESS_TEXTURE; - } else if (propertyName.contains("emissive")) { - return EMISSIVE_TEXTURE; - } else if (propertyName.contains("ambientcolor")) { - return LIGHTMAP_TEXTURE; - } else if (propertyName.contains("ambientfactor")) { - // we need to check what the ambient factor is, since that tells Interface to process this texture - // either as an occlusion texture or a light map - auto lambertMaterial = FbxCast(material); - - if (lambertMaterial->AmbientFactor == 0) { - return LIGHTMAP_TEXTURE; - } else if (lambertMaterial->AmbientFactor > 0) { - return OCCLUSION_TEXTURE; - } else { - return UNUSED_TEXTURE; - } - - } else if (propertyName.contains("tex_ao_map")) { - return OCCLUSION_TEXTURE; - } - - return UNUSED_TEXTURE; -} - -void FBXBaker::rewriteAndBakeSceneTextures() { - - // enumerate the surface materials to find the textures used in the scene - int numMaterials = _scene->GetMaterialCount(); - for (int i = 0; i < numMaterials; i++) { - FbxSurfaceMaterial* material = _scene->GetMaterial(i); - - if (material) { - // enumerate the properties of this material to see what texture channels it might have - FbxProperty property = material->GetFirstProperty(); - - while (property.IsValid()) { - // first check if this property has connected textures, if not we don't need to bother with it here - if (property.GetSrcObjectCount() > 0) { - - // figure out the type of texture from the material property - auto textureType = textureTypeForMaterialProperty(property, material); - - if (textureType != image::TextureUsage::UNUSED_TEXTURE) { - int numTextures = property.GetSrcObjectCount(); - - for (int j = 0; j < numTextures; j++) { - FbxFileTexture* fileTexture = property.GetSrcObject(j); - - // use QFileInfo to easily split up the existing texture filename into its components - QString fbxTextureFileName { fileTexture->GetFileName() }; - QFileInfo textureFileInfo { fbxTextureFileName.replace("\\", "/") }; - - // make sure this texture points to something and isn't one we've already re-mapped - if (!textureFileInfo.filePath().isEmpty() - && textureFileInfo.suffix() != BAKED_TEXTURE_EXT.mid(1)) { - - // construct the new baked texture file name and file path - // ensuring that the baked texture will have a unique name - // even if there was another texture with the same name at a different path - auto bakedTextureFileName = createBakedTextureFileName(textureFileInfo); - QString bakedTextureFilePath { - _bakedOutputDir + "/" + bakedTextureFileName - }; - _outputFiles.push_back(bakedTextureFilePath); - - qCDebug(model_baking).noquote() << "Re-mapping" << fileTexture->GetFileName() - << "to" << bakedTextureFilePath; - - // figure out the URL to this texture, embedded or external - auto urlToTexture = getTextureURL(textureFileInfo, fileTexture); - - // write the new filename into the FBX scene - fileTexture->SetFileName(bakedTextureFilePath.toUtf8().data()); - - // write the relative filename to be the baked texture file name since it will - // be right beside the FBX - fileTexture->SetRelativeFileName(bakedTextureFileName.toLocal8Bit().constData()); - - if (!_bakingTextures.contains(urlToTexture)) { - // bake this texture asynchronously - bakeTexture(urlToTexture, textureType, _bakedOutputDir); - } - } - } - } - } - - property = material->GetNextProperty(property); - } - } - } -} - -void FBXBaker::bakeTexture(const QUrl& textureURL, image::TextureUsage::Type textureType, const QDir& outputDir) { - // start a bake for this texture and add it to our list to keep track of - QSharedPointer bakingTexture { - new TextureBaker(textureURL, textureType, outputDir), - &TextureBaker::deleteLater - }; - - // make sure we hear when the baking texture is done - connect(bakingTexture.data(), &Baker::finished, this, &FBXBaker::handleBakedTexture); - - // keep a shared pointer to the baking texture - _bakingTextures.insert(textureURL, bakingTexture); - - // start baking the texture on one of our available worker threads - bakingTexture->moveToThread(_textureThreadGetter()); - QMetaObject::invokeMethod(bakingTexture.data(), "bake"); -} - -void FBXBaker::handleBakedTexture() { - TextureBaker* bakedTexture = qobject_cast(sender()); - - // make sure we haven't already run into errors, and that this is a valid texture - if (bakedTexture) { - if (!hasErrors()) { - if (!bakedTexture->hasErrors()) { - if (!_originalOutputDir.isEmpty()) { - // we've been asked to make copies of the originals, so we need to make copies of this if it is a linked texture - - // use the path to the texture being baked to determine if this was an embedded or a linked texture - - // it is embeddded if the texure being baked was inside the original output folder - // since that is where the FBX SDK places the .fbm folder it generates when importing the FBX - - auto originalOutputFolder = QUrl::fromLocalFile(_originalOutputDir); - - if (!originalOutputFolder.isParentOf(bakedTexture->getTextureURL())) { - // for linked textures we want to save a copy of original texture beside the original FBX - - qCDebug(model_baking) << "Saving original texture for" << bakedTexture->getTextureURL(); - - // check if we have a relative path to use for the texture - auto relativeTexturePath = texturePathRelativeToFBX(_fbxURL, bakedTexture->getTextureURL()); - - QFile originalTextureFile { - _originalOutputDir + "/" + relativeTexturePath + bakedTexture->getTextureURL().fileName() - }; - - if (relativeTexturePath.length() > 0) { - // make the folders needed by the relative path - } - - if (originalTextureFile.open(QIODevice::WriteOnly) && originalTextureFile.write(bakedTexture->getOriginalTexture()) != -1) { - qCDebug(model_baking) << "Saved original texture file" << originalTextureFile.fileName() - << "for" << _fbxURL; - } else { - handleError("Could not save original external texture " + originalTextureFile.fileName() - + " for " + _fbxURL.toString()); - return; - } - } - } - - - // now that this texture has been baked and handled, we can remove that TextureBaker from our hash - _bakingTextures.remove(bakedTexture->getTextureURL()); - - checkIfTexturesFinished(); - } else { - // there was an error baking this texture - add it to our list of errors - _errorList.append(bakedTexture->getErrors()); - - // we don't emit finished yet so that the other textures can finish baking first - _pendingErrorEmission = true; - - // now that this texture has been baked, even though it failed, we can remove that TextureBaker from our list - _bakingTextures.remove(bakedTexture->getTextureURL()); - - checkIfTexturesFinished(); - } - } else { - // we have errors to attend to, so we don't do extra processing for this texture - // but we do need to remove that TextureBaker from our list - // and then check if we're done with all textures - _bakingTextures.remove(bakedTexture->getTextureURL()); - - checkIfTexturesFinished(); - } - } -} - -void FBXBaker::exportScene() { - // setup the exporter - FbxExporter* exporter = FbxExporter::Create(_sdkManager.get(), ""); - - // save the relative path to this FBX inside our passed output folder - - auto fileName = _fbxURL.fileName(); - auto baseName = fileName.left(fileName.lastIndexOf('.')); - auto bakedFilename = baseName + BAKED_FBX_EXTENSION; - - _bakedFBXFilePath = _bakedOutputDir + "/" + bakedFilename; - - bool exportStatus = exporter->Initialize(_bakedFBXFilePath.toLocal8Bit().data()); - - if (!exportStatus) { - // failed to initialize exporter, print an error and return - handleError("Failed to export FBX file at " + _fbxURL.toString() + " to " + _bakedFBXFilePath - + "- error: " + exporter->GetStatus().GetErrorString()); - } - - _outputFiles.push_back(_bakedFBXFilePath); - - // export the scene - exporter->Export(_scene); - - qCDebug(model_baking) << "Exported" << _fbxURL << "with re-written paths to" << _bakedFBXFilePath; -} - - -void FBXBaker::removeEmbeddedMediaFolder() { - // now that the bake is complete, remove the embedded media folder produced by the FBX SDK when it imports an FBX - //auto embeddedMediaFolderName = _fbxURL.fileName().replace(".fbx", ".fbm"); - //QDir(_bakedOutputDir + ORIGINAL_OUTPUT_SUBFOLDER + embeddedMediaFolderName).removeRecursively(); -} - -void FBXBaker::checkIfTexturesFinished() { - // check if we're done everything we need to do for this FBX - // and emit our finished signal if we're done - - if (_bakingTextures.isEmpty()) { - // remove the embedded media folder that the FBX SDK produces when reading the original - removeEmbeddedMediaFolder(); - - if (hasErrors()) { - // if we're checking for completion but we have errors - // that means one or more of our texture baking operations failed - - if (_pendingErrorEmission) { - emit finished(); - } - - return; - } else { - qCDebug(model_baking) << "Finished baking" << _fbxURL; - - emit finished(); - } - } -} diff --git a/tools/oven/src/FBXBaker.h b/tools/oven/src/FBXBaker.h deleted file mode 100644 index faf56ed46b..0000000000 --- a/tools/oven/src/FBXBaker.h +++ /dev/null @@ -1,103 +0,0 @@ -// -// FBXBaker.h -// tools/oven/src -// -// Created by Stephen Birarda on 3/30/17. -// Copyright 2017 High Fidelity, Inc. -// -// Distributed under the Apache License, Version 2.0. -// See the accompanying file LICENSE or http://www.apache.org/licenses/LICENSE-2.0.html -// - -#ifndef hifi_FBXBaker_h -#define hifi_FBXBaker_h - -#include -#include -#include -#include - -#include "Baker.h" -#include "TextureBaker.h" - -#include - -namespace fbxsdk { - class FbxManager; - class FbxProperty; - class FbxScene; - class FbxFileTexture; -} - -static const QString BAKED_FBX_EXTENSION = ".baked.fbx"; -using FBXSDKManagerUniquePointer = std::unique_ptr>; - -using TextureBakerThreadGetter = std::function; - -class FBXBaker : public Baker { - Q_OBJECT -public: - FBXBaker(const QUrl& fbxURL, TextureBakerThreadGetter textureThreadGetter, - const QString& bakedOutputDir, const QString& originalOutputDir = ""); - - QUrl getFBXUrl() const { return _fbxURL; } - QString getBakedFBXFilePath() const { return _bakedFBXFilePath; } - std::vector getOutputFiles() const { return _outputFiles; } - -public slots: - // all calls to FBXBaker::bake for FBXBaker instances must be from the same thread - // because the Autodesk SDK will cause a crash if it is called from multiple threads - virtual void bake() override; - -signals: - void sourceCopyReadyToLoad(); - -private slots: - void bakeSourceCopy(); - void handleFBXNetworkReply(); - void handleBakedTexture(); - -private: - void setupOutputFolder(); - - void loadSourceFBX(); - - void importScene(); - void rewriteAndBakeSceneTextures(); - void exportScene(); - void removeEmbeddedMediaFolder(); - - void checkIfTexturesFinished(); - - QString createBakedTextureFileName(const QFileInfo& textureFileInfo); - QUrl getTextureURL(const QFileInfo& textureFileInfo, fbxsdk::FbxFileTexture* fileTexture); - - void bakeTexture(const QUrl& textureURL, image::TextureUsage::Type textureType, const QDir& outputDir); - - QUrl _fbxURL; - - QString _bakedFBXFilePath; - - QString _bakedOutputDir; - - // If set, the original FBX and textures will also be copied here - QString _originalOutputDir; - - QDir _tempDir; - QString _originalFBXFilePath; - - // List of baked output files, includes the FBX and textures - std::vector _outputFiles; - - static FBXSDKManagerUniquePointer _sdkManager; - fbxsdk::FbxScene* _scene { nullptr }; - - QMultiHash> _bakingTextures; - QHash _textureNameMatchCount; - - TextureBakerThreadGetter _textureThreadGetter; - - bool _pendingErrorEmission { false }; -}; - -#endif // hifi_FBXBaker_h diff --git a/tools/oven/src/TextureBaker.cpp b/tools/oven/src/TextureBaker.cpp deleted file mode 100644 index 70df511d2c..0000000000 --- a/tools/oven/src/TextureBaker.cpp +++ /dev/null @@ -1,131 +0,0 @@ -// -// TextureBaker.cpp -// tools/oven/src -// -// Created by Stephen Birarda on 4/5/17. -// Copyright 2017 High Fidelity, Inc. -// -// Distributed under the Apache License, Version 2.0. -// See the accompanying file LICENSE or http://www.apache.org/licenses/LICENSE-2.0.html -// - -#include -#include -#include -#include - -#include -#include -#include -#include - -#include "ModelBakingLoggingCategory.h" - -#include "TextureBaker.h" - -const QString BAKED_TEXTURE_EXT = ".ktx"; - -TextureBaker::TextureBaker(const QUrl& textureURL, image::TextureUsage::Type textureType, const QDir& outputDirectory) : - _textureURL(textureURL), - _textureType(textureType), - _outputDirectory(outputDirectory) -{ - // figure out the baked texture filename - auto originalFilename = textureURL.fileName(); - _bakedTextureFileName = originalFilename.left(originalFilename.lastIndexOf('.')) + BAKED_TEXTURE_EXT; -} - -void TextureBaker::bake() { - // once our texture is loaded, kick off a the processing - connect(this, &TextureBaker::originalTextureLoaded, this, &TextureBaker::processTexture); - - // first load the texture (either locally or remotely) - loadTexture(); -} - -void TextureBaker::loadTexture() { - // check if the texture is local or first needs to be downloaded - if (_textureURL.isLocalFile()) { - // load up the local file - QFile localTexture { _textureURL.toLocalFile() }; - - if (!localTexture.open(QIODevice::ReadOnly)) { - handleError("Unable to open texture " + _textureURL.toString()); - return; - } - - _originalTexture = localTexture.readAll(); - - emit originalTextureLoaded(); - } else { - // remote file, kick off a download - auto& networkAccessManager = NetworkAccessManager::getInstance(); - - QNetworkRequest networkRequest; - - // setup the request to follow re-directs and always hit the network - networkRequest.setAttribute(QNetworkRequest::FollowRedirectsAttribute, true); - networkRequest.setAttribute(QNetworkRequest::CacheLoadControlAttribute, QNetworkRequest::AlwaysNetwork); - networkRequest.setHeader(QNetworkRequest::UserAgentHeader, HIGH_FIDELITY_USER_AGENT); - - networkRequest.setUrl(_textureURL); - - qCDebug(model_baking) << "Downloading" << _textureURL; - - // kickoff the download, wait for slot to tell us it is done - auto networkReply = networkAccessManager.get(networkRequest); - connect(networkReply, &QNetworkReply::finished, this, &TextureBaker::handleTextureNetworkReply); - } -} - -void TextureBaker::handleTextureNetworkReply() { - auto requestReply = qobject_cast(sender()); - - if (requestReply->error() == QNetworkReply::NoError) { - qCDebug(model_baking) << "Downloaded texture" << _textureURL; - - // store the original texture so it can be passed along for the bake - _originalTexture = requestReply->readAll(); - - emit originalTextureLoaded(); - } else { - // add an error to our list stating that this texture could not be downloaded - handleError("Error downloading " + _textureURL.toString() + " - " + requestReply->errorString()); - } -} - -void TextureBaker::processTexture() { - auto processedTexture = image::processImage(_originalTexture, _textureURL.toString().toStdString(), - ABSOLUTE_MAX_TEXTURE_NUM_PIXELS, _textureType); - - if (!processedTexture) { - handleError("Could not process texture " + _textureURL.toString()); - return; - } - - // the baked textures need to have the source hash added for cache checks in Interface - // so we add that to the processed texture before handling it off to be serialized - auto hashData = QCryptographicHash::hash(_originalTexture, QCryptographicHash::Md5); - std::string hash = hashData.toHex().toStdString(); - processedTexture->setSourceHash(hash); - - auto memKTX = gpu::Texture::serialize(*processedTexture); - - if (!memKTX) { - handleError("Could not serialize " + _textureURL.toString() + " to KTX"); - return; - } - - const char* data = reinterpret_cast(memKTX->_storage->data()); - const size_t length = memKTX->_storage->size(); - - // attempt to write the baked texture to the destination file path - QFile bakedTextureFile { _outputDirectory.absoluteFilePath(_bakedTextureFileName) }; - - if (!bakedTextureFile.open(QIODevice::WriteOnly) || bakedTextureFile.write(data, length) == -1) { - handleError("Could not write baked texture for " + _textureURL.toString()); - } - - qCDebug(model_baking) << "Baked texture" << _textureURL; - emit finished(); -} diff --git a/tools/oven/src/TextureBaker.h b/tools/oven/src/TextureBaker.h deleted file mode 100644 index ee1e968f20..0000000000 --- a/tools/oven/src/TextureBaker.h +++ /dev/null @@ -1,59 +0,0 @@ -// -// TextureBaker.h -// tools/oven/src -// -// Created by Stephen Birarda on 4/5/17. -// Copyright 2017 High Fidelity, Inc. -// -// Distributed under the Apache License, Version 2.0. -// See the accompanying file LICENSE or http://www.apache.org/licenses/LICENSE-2.0.html -// - -#ifndef hifi_TextureBaker_h -#define hifi_TextureBaker_h - -#include -#include -#include - -#include - -#include "Baker.h" - -extern const QString BAKED_TEXTURE_EXT; - -class TextureBaker : public Baker { - Q_OBJECT - -public: - TextureBaker(const QUrl& textureURL, image::TextureUsage::Type textureType, const QDir& outputDirectory); - - const QByteArray& getOriginalTexture() const { return _originalTexture; } - - QUrl getTextureURL() const { return _textureURL; } - - QString getDestinationFilePath() const { return _outputDirectory.absoluteFilePath(_bakedTextureFileName); } - QString getBakedTextureFileName() const { return _bakedTextureFileName; } - -public slots: - virtual void bake() override; - -signals: - void originalTextureLoaded(); - -private slots: - void processTexture(); - -private: - void loadTexture(); - void handleTextureNetworkReply(); - - QUrl _textureURL; - QByteArray _originalTexture; - image::TextureUsage::Type _textureType; - - QDir _outputDirectory; - QString _bakedTextureFileName; -}; - -#endif // hifi_TextureBaker_h From b60d68c7149487e5d824ee9ffd0705df6401c3a5 Mon Sep 17 00:00:00 2001 From: Ryan Huffman Date: Thu, 7 Sep 2017 14:00:09 -0700 Subject: [PATCH 065/129] Replace FBX SDK loading of file in FBXBaker with FBXReader --- libraries/baking/CMakeLists.txt | 2 +- libraries/baking/src/FBXBaker.cpp | 28 +++++++++------------------- libraries/baking/src/FBXBaker.h | 6 +++++- tools/oven/CMakeLists.txt | 2 +- tools/oven/src/ui/BakeWidget.h | 2 +- tools/oven/src/ui/ModelBakeWidget.h | 2 +- tools/oven/src/ui/SkyboxBakeWidget.h | 2 +- 7 files changed, 19 insertions(+), 25 deletions(-) diff --git a/libraries/baking/CMakeLists.txt b/libraries/baking/CMakeLists.txt index 806220ec30..da3389c862 100644 --- a/libraries/baking/CMakeLists.txt +++ b/libraries/baking/CMakeLists.txt @@ -15,5 +15,5 @@ if (FBX_FOUND) endif (UNIX) endif () -link_hifi_libraries(shared model networking ktx image) +link_hifi_libraries(shared model networking ktx image fbx) include_hifi_library_headers(gpu) diff --git a/libraries/baking/src/FBXBaker.cpp b/libraries/baking/src/FBXBaker.cpp index 81b5de7546..9a28e6a788 100644 --- a/libraries/baking/src/FBXBaker.cpp +++ b/libraries/baking/src/FBXBaker.cpp @@ -1,6 +1,6 @@ // // FBXBaker.cpp -// tools/oven/src +// tools/baking/src // // Created by Stephen Birarda on 3/30/17. // Copyright 2017 High Fidelity, Inc. @@ -27,6 +27,9 @@ #include +#include +#include + #include "ModelBakingLoggingCategory.h" #include "TextureBaker.h" @@ -205,29 +208,16 @@ void FBXBaker::handleFBXNetworkReply() { } void FBXBaker::importScene() { - // create an FBX SDK importer - FbxImporter* importer = FbxImporter::Create(_sdkManager.get(), ""); - qDebug() << "file path: " << _originalFBXFilePath.toLocal8Bit().data() << QDir(_originalFBXFilePath).exists(); - // import the copy of the original FBX file - bool importStatus = importer->Initialize(_originalFBXFilePath.toLocal8Bit().data()); - if (!importStatus) { - // failed to initialize importer, print an error and return - handleError("Failed to import " + _fbxURL.toString() + " - " + importer->GetStatus().GetErrorString()); + QFile fbxFile(_originalFBXFilePath); + if (!fbxFile.open(QIODevice::ReadOnly)) { + handleError("Error opening " + _originalFBXFilePath); return; - } else { - qCDebug(model_baking) << "Imported" << _fbxURL << "to FbxScene"; } - // setup a new scene to hold the imported file - _scene = FbxScene::Create(_sdkManager.get(), "bakeScene"); - - // import the file to the created scene - importer->Import(_scene); - - // destroy the importer that is no longer needed - importer->Destroy(); + qCDebug(model_baking) << "Imported" << _fbxURL << "to FbxScene"; + _rootNode = FBXReader::parseFBX(&fbxFile); } QString texturePathRelativeToFBX(QUrl fbxURL, QUrl textureURL) { diff --git a/libraries/baking/src/FBXBaker.h b/libraries/baking/src/FBXBaker.h index 00e7987422..0eb25f510b 100644 --- a/libraries/baking/src/FBXBaker.h +++ b/libraries/baking/src/FBXBaker.h @@ -1,6 +1,6 @@ // // FBXBaker.h -// tools/oven/src +// tools/baking/src // // Created by Stephen Birarda on 3/30/17. // Copyright 2017 High Fidelity, Inc. @@ -24,6 +24,8 @@ #include +#include + namespace fbxsdk { class FbxManager; class FbxProperty; @@ -76,6 +78,8 @@ private: void bakeTexture(const QUrl& textureURL, image::TextureUsage::Type textureType, const QDir& outputDir); QUrl _fbxURL; + + FBXNode _rootNode; QString _bakedFBXFilePath; diff --git a/tools/oven/CMakeLists.txt b/tools/oven/CMakeLists.txt index 0d692b5465..010b1c25b1 100644 --- a/tools/oven/CMakeLists.txt +++ b/tools/oven/CMakeLists.txt @@ -2,7 +2,7 @@ set(TARGET_NAME oven) setup_hifi_project(Widgets Gui Concurrent) -link_hifi_libraries(networking shared image gpu ktx fbx baking) +link_hifi_libraries(networking shared image gpu ktx fbx baking model) setup_memory_debugger() diff --git a/tools/oven/src/ui/BakeWidget.h b/tools/oven/src/ui/BakeWidget.h index e7ab8d1840..00996128ed 100644 --- a/tools/oven/src/ui/BakeWidget.h +++ b/tools/oven/src/ui/BakeWidget.h @@ -14,7 +14,7 @@ #include -#include "../Baker.h" +#include class BakeWidget : public QWidget { Q_OBJECT diff --git a/tools/oven/src/ui/ModelBakeWidget.h b/tools/oven/src/ui/ModelBakeWidget.h index ed08990ba5..b42b8725f6 100644 --- a/tools/oven/src/ui/ModelBakeWidget.h +++ b/tools/oven/src/ui/ModelBakeWidget.h @@ -16,7 +16,7 @@ #include -#include "../FBXBaker.h" +#include #include "BakeWidget.h" diff --git a/tools/oven/src/ui/SkyboxBakeWidget.h b/tools/oven/src/ui/SkyboxBakeWidget.h index 4063a5459b..f00ab07f33 100644 --- a/tools/oven/src/ui/SkyboxBakeWidget.h +++ b/tools/oven/src/ui/SkyboxBakeWidget.h @@ -16,7 +16,7 @@ #include -#include "../TextureBaker.h" +#include #include "BakeWidget.h" From 7214f5737625d777ebdecb7bd6046e71f7d6f0bb Mon Sep 17 00:00:00 2001 From: Ryan Huffman Date: Thu, 7 Sep 2017 14:12:15 -0700 Subject: [PATCH 066/129] Update FBXBaker::exportScene with FBXWriter --- libraries/baking/src/FBXBaker.cpp | 22 +++++++++------------- 1 file changed, 9 insertions(+), 13 deletions(-) diff --git a/libraries/baking/src/FBXBaker.cpp b/libraries/baking/src/FBXBaker.cpp index 9a28e6a788..35d981f337 100644 --- a/libraries/baking/src/FBXBaker.cpp +++ b/libraries/baking/src/FBXBaker.cpp @@ -212,7 +212,7 @@ void FBXBaker::importScene() { QFile fbxFile(_originalFBXFilePath); if (!fbxFile.open(QIODevice::ReadOnly)) { - handleError("Error opening " + _originalFBXFilePath); + handleError("Error opening " + _originalFBXFilePath + " for reading"); return; } @@ -502,29 +502,25 @@ void FBXBaker::handleBakedTexture() { } void FBXBaker::exportScene() { - // setup the exporter - FbxExporter* exporter = FbxExporter::Create(_sdkManager.get(), ""); - // save the relative path to this FBX inside our passed output folder - auto fileName = _fbxURL.fileName(); auto baseName = fileName.left(fileName.lastIndexOf('.')); auto bakedFilename = baseName + BAKED_FBX_EXTENSION; _bakedFBXFilePath = _bakedOutputDir + "/" + bakedFilename; - bool exportStatus = exporter->Initialize(_bakedFBXFilePath.toLocal8Bit().data()); + auto fbxData = FBXWriter::encodeFBX(_rootNode); - if (!exportStatus) { - // failed to initialize exporter, print an error and return - handleError("Failed to export FBX file at " + _fbxURL.toString() + " to " + _bakedFBXFilePath - + "- error: " + exporter->GetStatus().GetErrorString()); + QFile bakedFile(_bakedFBXFilePath); + + if (!bakedFile.open(QIODevice::WriteOnly)) { + handleError("Error opening " + _bakedFBXFilePath + " for writing"); + return; } - _outputFiles.push_back(_bakedFBXFilePath); + bakedFile.write(fbxData); - // export the scene - exporter->Export(_scene); + _outputFiles.push_back(_bakedFBXFilePath); qCDebug(model_baking) << "Exported" << _fbxURL << "with re-written paths to" << _bakedFBXFilePath; } From b153d1e1779aa53372a22e7e6f8a247b4cf0b675 Mon Sep 17 00:00:00 2001 From: Stephen Birarda Date: Thu, 7 Sep 2017 18:38:29 -0700 Subject: [PATCH 067/129] use FBXReader/FBXWriter for texture baking in FBXBaker --- libraries/baking/src/FBXBaker.cpp | 108 ++++++++++++++--------- libraries/baking/src/FBXBaker.h | 18 ++-- libraries/baking/src/TextureBaker.cpp | 13 ++- libraries/baking/src/TextureBaker.h | 3 +- libraries/fbx/src/FBX.h | 1 + libraries/fbx/src/FBXReader.cpp | 2 +- libraries/fbx/src/FBXReader_Material.cpp | 1 + libraries/fbx/src/FBXWriter.cpp | 6 +- 8 files changed, 87 insertions(+), 65 deletions(-) diff --git a/libraries/baking/src/FBXBaker.cpp b/libraries/baking/src/FBXBaker.cpp index 35d981f337..73209ec7cf 100644 --- a/libraries/baking/src/FBXBaker.cpp +++ b/libraries/baking/src/FBXBaker.cpp @@ -35,9 +35,6 @@ #include "FBXBaker.h" -std::once_flag onceFlag; -FBXSDKManagerUniquePointer FBXBaker::_sdkManager { nullptr }; - FBXBaker::FBXBaker(const QUrl& fbxURL, TextureBakerThreadGetter textureThreadGetter, const QString& bakedOutputDir, const QString& originalOutputDir) : _fbxURL(fbxURL), @@ -45,12 +42,7 @@ FBXBaker::FBXBaker(const QUrl& fbxURL, TextureBakerThreadGetter textureThreadGet _originalOutputDir(originalOutputDir), _textureThreadGetter(textureThreadGetter) { - std::call_once(onceFlag, [](){ - // create the static FBX SDK manager - _sdkManager = FBXSDKManagerUniquePointer(FbxManager::Create(), [](FbxManager* manager){ - manager->Destroy(); - }); - }); + } void FBXBaker::bake() { @@ -216,8 +208,12 @@ void FBXBaker::importScene() { return; } - qCDebug(model_baking) << "Imported" << _fbxURL << "to FbxScene"; - _rootNode = FBXReader::parseFBX(&fbxFile); + FBXReader reader; + + qCDebug(model_baking) << "Parsing" << _fbxURL; + _rootNode = reader._rootNode = reader.parseFBX(&fbxFile); + _geometry = *reader.extractFBXGeometry({}, _fbxURL.toString()); + _textureContent = reader._textureContent; } QString texturePathRelativeToFBX(QUrl fbxURL, QUrl textureURL) { @@ -254,7 +250,7 @@ QString FBXBaker::createBakedTextureFileName(const QFileInfo& textureFileInfo) { return bakedTextureFileName; } -QUrl FBXBaker::getTextureURL(const QFileInfo& textureFileInfo, FbxFileTexture* fileTexture) { +QUrl FBXBaker::getTextureURL(const QFileInfo& textureFileInfo, QString relativeFileName) { QUrl urlToTexture; if (textureFileInfo.exists() && textureFileInfo.isFile()) { @@ -264,7 +260,6 @@ QUrl FBXBaker::getTextureURL(const QFileInfo& textureFileInfo, FbxFileTexture* f // external texture that we'll need to download or find // first check if it the RelativePath to the texture in the FBX was relative - QString relativeFileName = fileTexture->GetRelativeFileName(); auto apparentRelativePath = QFileInfo(relativeFileName.replace("\\", "/")); // this is a relative file path which will require different handling @@ -336,31 +331,44 @@ image::TextureUsage::Type textureTypeForMaterialProperty(FbxProperty& property, } void FBXBaker::rewriteAndBakeSceneTextures() { + using namespace image::TextureUsage; + QHash textureTypes; - // enumerate the surface materials to find the textures used in the scene - int numMaterials = _scene->GetMaterialCount(); - for (int i = 0; i < numMaterials; i++) { - FbxSurfaceMaterial* material = _scene->GetMaterial(i); + // enumerate the materials in the extracted geometry so we can determine the texture type for each texture ID + for (const auto& material : _geometry.materials) { + if (material.normalTexture.isBumpmap) { + textureTypes[material.normalTexture.id] = BUMP_TEXTURE; + } else { + textureTypes[material.normalTexture.id] = NORMAL_TEXTURE; + } - if (material) { - // enumerate the properties of this material to see what texture channels it might have - FbxProperty property = material->GetFirstProperty(); + textureTypes[material.albedoTexture.id] = ALBEDO_TEXTURE; + textureTypes[material.glossTexture.id] = GLOSS_TEXTURE; + textureTypes[material.roughnessTexture.id] = ROUGHNESS_TEXTURE; + textureTypes[material.specularTexture.id] = SPECULAR_TEXTURE; + textureTypes[material.metallicTexture.id] = METALLIC_TEXTURE; + textureTypes[material.emissiveTexture.id] = EMISSIVE_TEXTURE; + textureTypes[material.occlusionTexture.id] = OCCLUSION_TEXTURE; + textureTypes[material.lightmapTexture.id] = LIGHTMAP_TEXTURE; + } - while (property.IsValid()) { - // first check if this property has connected textures, if not we don't need to bother with it here - if (property.GetSrcObjectCount() > 0) { + // enumerate the children of the root node + for (FBXNode& rootChild : _rootNode.children) { - // figure out the type of texture from the material property - auto textureType = textureTypeForMaterialProperty(property, material); + if (rootChild.name == "Objects") { - if (textureType != image::TextureUsage::UNUSED_TEXTURE) { - int numTextures = property.GetSrcObjectCount(); + // enumerate the objects + auto object = rootChild.children.begin(); + while (object != rootChild.children.end()) { + if (object->name == "Texture") { - for (int j = 0; j < numTextures; j++) { - FbxFileTexture* fileTexture = property.GetSrcObject(j); + // enumerate the texture children + for (FBXNode& textureChild : object->children) { + + if (textureChild.name == "RelativeFilename") { // use QFileInfo to easily split up the existing texture filename into its components - QString fbxTextureFileName { fileTexture->GetFileName() }; + QString fbxTextureFileName { textureChild.properties.at(0).toByteArray() }; QFileInfo textureFileInfo { fbxTextureFileName.replace("\\", "/") }; // make sure this texture points to something and isn't one we've already re-mapped @@ -383,38 +391,50 @@ void FBXBaker::rewriteAndBakeSceneTextures() { }; _outputFiles.push_back(bakedTextureFilePath); - qCDebug(model_baking).noquote() << "Re-mapping" << fileTexture->GetFileName() - << "to" << bakedTextureFilePath; + qCDebug(model_baking).noquote() << "Re-mapping" << fbxTextureFileName + << "to" << bakedTextureFileName; // figure out the URL to this texture, embedded or external - auto urlToTexture = getTextureURL(textureFileInfo, fileTexture); + auto urlToTexture = getTextureURL(textureFileInfo, fbxTextureFileName); // write the new filename into the FBX scene - fileTexture->SetFileName(bakedTextureFilePath.toUtf8().data()); - - // write the relative filename to be the baked texture file name since it will - // be right beside the FBX - fileTexture->SetRelativeFileName(bakedTextureFileName.toLocal8Bit().constData()); + textureChild.properties[0] = bakedTextureFileName.toLocal8Bit(); if (!_bakingTextures.contains(urlToTexture)) { + + // grab the ID for this texture so we can figure out the + // texture type from the loaded materials + QString textureID { object->properties[0].toByteArray() }; + auto textureType = textureTypes[textureID]; + + // check if this was an embedded texture we have already have in-memory content for + auto textureContent = _textureContent.value(fbxTextureFileName.toLocal8Bit()); + // bake this texture asynchronously - bakeTexture(urlToTexture, textureType, _bakedOutputDir); + bakeTexture(urlToTexture, textureType, _bakedOutputDir, textureContent); } } } } - } - property = material->GetNextProperty(property); + ++object; + + } else if (object->name == "Video") { + // this is an embedded texture, we need to remove it from the FBX + object = rootChild.children.erase(object); + } else { + ++object; + } } } } } -void FBXBaker::bakeTexture(const QUrl& textureURL, image::TextureUsage::Type textureType, const QDir& outputDir) { +void FBXBaker::bakeTexture(const QUrl& textureURL, image::TextureUsage::Type textureType, + const QDir& outputDir, const QByteArray& textureContent) { // start a bake for this texture and add it to our list to keep track of QSharedPointer bakingTexture { - new TextureBaker(textureURL, textureType, outputDir), + new TextureBaker(textureURL, textureType, outputDir, textureContent), &TextureBaker::deleteLater }; @@ -464,7 +484,7 @@ void FBXBaker::handleBakedTexture() { if (originalTextureFile.open(QIODevice::WriteOnly) && originalTextureFile.write(bakedTexture->getOriginalTexture()) != -1) { qCDebug(model_baking) << "Saved original texture file" << originalTextureFile.fileName() - << "for" << _fbxURL; + << "for" << _fbxURL; } else { handleError("Could not save original external texture " + originalTextureFile.fileName() + " for " + _fbxURL.toString()); diff --git a/libraries/baking/src/FBXBaker.h b/libraries/baking/src/FBXBaker.h index 0eb25f510b..2dc31828cb 100644 --- a/libraries/baking/src/FBXBaker.h +++ b/libraries/baking/src/FBXBaker.h @@ -26,15 +26,7 @@ #include -namespace fbxsdk { - class FbxManager; - class FbxProperty; - class FbxScene; - class FbxFileTexture; -} - static const QString BAKED_FBX_EXTENSION = ".baked.fbx"; -using FBXSDKManagerUniquePointer = std::unique_ptr>; using TextureBakerThreadGetter = std::function; @@ -73,13 +65,16 @@ private: void checkIfTexturesFinished(); QString createBakedTextureFileName(const QFileInfo& textureFileInfo); - QUrl getTextureURL(const QFileInfo& textureFileInfo, fbxsdk::FbxFileTexture* fileTexture); + QUrl getTextureURL(const QFileInfo& textureFileInfo, QString relativeFileName); - void bakeTexture(const QUrl& textureURL, image::TextureUsage::Type textureType, const QDir& outputDir); + void bakeTexture(const QUrl& textureURL, image::TextureUsage::Type textureType, const QDir& outputDir, + const QByteArray& textureContent = QByteArray()); QUrl _fbxURL; FBXNode _rootNode; + FBXGeometry _geometry; + QHash _textureContent; QString _bakedFBXFilePath; @@ -91,9 +86,6 @@ private: QDir _tempDir; QString _originalFBXFilePath; - static FBXSDKManagerUniquePointer _sdkManager; - fbxsdk::FbxScene* _scene { nullptr }; - QMultiHash> _bakingTextures; QHash _textureNameMatchCount; diff --git a/libraries/baking/src/TextureBaker.cpp b/libraries/baking/src/TextureBaker.cpp index 548e3921e4..cdf21a0290 100644 --- a/libraries/baking/src/TextureBaker.cpp +++ b/libraries/baking/src/TextureBaker.cpp @@ -25,8 +25,10 @@ const QString BAKED_TEXTURE_EXT = ".ktx"; -TextureBaker::TextureBaker(const QUrl& textureURL, image::TextureUsage::Type textureType, const QDir& outputDirectory) : +TextureBaker::TextureBaker(const QUrl& textureURL, image::TextureUsage::Type textureType, + const QDir& outputDirectory, const QByteArray& textureContent) : _textureURL(textureURL), + _originalTexture(textureContent), _textureType(textureType), _outputDirectory(outputDirectory) { @@ -39,8 +41,13 @@ void TextureBaker::bake() { // once our texture is loaded, kick off a the processing connect(this, &TextureBaker::originalTextureLoaded, this, &TextureBaker::processTexture); - // first load the texture (either locally or remotely) - loadTexture(); + if (_originalTexture.isEmpty()) { + // first load the texture (either locally or remotely) + loadTexture(); + } else { + // we already have a texture passed to us, use that + emit originalTextureLoaded(); + } } const QStringList TextureBaker::getSupportedFormats() { diff --git a/libraries/baking/src/TextureBaker.h b/libraries/baking/src/TextureBaker.h index 76d0a69823..5ea696dda0 100644 --- a/libraries/baking/src/TextureBaker.h +++ b/libraries/baking/src/TextureBaker.h @@ -27,7 +27,8 @@ class TextureBaker : public Baker { Q_OBJECT public: - TextureBaker(const QUrl& textureURL, image::TextureUsage::Type textureType, const QDir& outputDirectory); + TextureBaker(const QUrl& textureURL, image::TextureUsage::Type textureType, + const QDir& outputDirectory, const QByteArray& textureContent = QByteArray()); static const QStringList getSupportedFormats(); diff --git a/libraries/fbx/src/FBX.h b/libraries/fbx/src/FBX.h index 9e1982f9e0..fc45e9a792 100644 --- a/libraries/fbx/src/FBX.h +++ b/libraries/fbx/src/FBX.h @@ -113,6 +113,7 @@ const int MAX_NUM_PIXELS_FOR_FBX_TEXTURE = 2048 * 2048; /// A texture map in an FBX document. class FBXTexture { public: + QString id; QString name; QByteArray filename; QByteArray content; diff --git a/libraries/fbx/src/FBXReader.cpp b/libraries/fbx/src/FBXReader.cpp index d212ec820f..d4a58a1126 100644 --- a/libraries/fbx/src/FBXReader.cpp +++ b/libraries/fbx/src/FBXReader.cpp @@ -939,7 +939,7 @@ FBXGeometry* FBXReader::extractFBXGeometry(const QVariantHash& mapping, const QS QByteArray content; foreach (const FBXNode& subobject, object.children) { if (subobject.name == "RelativeFilename") { - filepath= subobject.properties.at(0).toByteArray(); + filepath = subobject.properties.at(0).toByteArray(); filepath = filepath.replace('\\', '/'); } else if (subobject.name == "Content" && !subobject.properties.isEmpty()) { diff --git a/libraries/fbx/src/FBXReader_Material.cpp b/libraries/fbx/src/FBXReader_Material.cpp index ca2ec557b4..ef6496cd10 100644 --- a/libraries/fbx/src/FBXReader_Material.cpp +++ b/libraries/fbx/src/FBXReader_Material.cpp @@ -92,6 +92,7 @@ FBXTexture FBXReader::getTexture(const QString& textureID) { texture.filename = filepath; } + texture.id = textureID; texture.name = _textureNames.value(textureID); texture.transform.setIdentity(); texture.texcoordSet = 0; diff --git a/libraries/fbx/src/FBXWriter.cpp b/libraries/fbx/src/FBXWriter.cpp index a084d94a46..f6bb92a8b5 100644 --- a/libraries/fbx/src/FBXWriter.cpp +++ b/libraries/fbx/src/FBXWriter.cpp @@ -119,7 +119,7 @@ void FBXWriter::encodeFBXProperty(QDataStream& out, const QVariant& prop) { case QMetaType::QString: { - auto& bytes = prop.toString().toUtf8(); + auto bytes = prop.toString().toUtf8(); out << 'S'; out << bytes.length(); out << bytes; @@ -130,7 +130,7 @@ void FBXWriter::encodeFBXProperty(QDataStream& out, const QVariant& prop) { case QMetaType::QByteArray: { - auto& bytes = prop.toByteArray(); + auto bytes = prop.toByteArray(); out.device()->write("S", 1); out << (int32_t)bytes.size(); out.writeRawData(bytes, bytes.size()); @@ -140,7 +140,7 @@ void FBXWriter::encodeFBXProperty(QDataStream& out, const QVariant& prop) { // TODO Delete? Do we ever use QList instead of QVector? case QVariant::Type::List: { - auto& list = prop.toList(); + auto list = prop.toList(); auto listType = prop.userType(); switch (listType) { From 3cf77f377c149717545eea83fe9e988f792dd24f Mon Sep 17 00:00:00 2001 From: Stephen Birarda Date: Thu, 7 Sep 2017 18:46:44 -0700 Subject: [PATCH 068/129] remove FBX SDK from CMake files --- cmake/modules/FindFBX.cmake | 111 -------------------------------- libraries/baking/CMakeLists.txt | 14 ---- tools/oven/CMakeLists.txt | 12 ---- 3 files changed, 137 deletions(-) delete mode 100644 cmake/modules/FindFBX.cmake diff --git a/cmake/modules/FindFBX.cmake b/cmake/modules/FindFBX.cmake deleted file mode 100644 index 3963f4d2e6..0000000000 --- a/cmake/modules/FindFBX.cmake +++ /dev/null @@ -1,111 +0,0 @@ -# Locate the FBX SDK -# -# Defines the following variables: -# -# FBX_FOUND - Found the FBX SDK -# FBX_VERSION - Version number -# FBX_INCLUDE_DIRS - Include directories -# FBX_LIBRARIES - The libraries to link to -# -# Accepts the following variables as input: -# -# FBX_VERSION - as a CMake variable, e.g. 2017.0.1 -# FBX_ROOT - (as a CMake or environment variable) -# The root directory of the FBX SDK install - -# adapted from https://github.com/ufz-vislab/VtkFbxConverter/blob/master/FindFBX.cmake -# which uses the MIT license (https://github.com/ufz-vislab/VtkFbxConverter/blob/master/LICENSE.txt) - -if (NOT FBX_VERSION) - set(FBX_VERSION 2017.1) -endif() - -string(REGEX REPLACE "^([0-9]+).*$" "\\1" FBX_VERSION_MAJOR "${FBX_VERSION}") -string(REGEX REPLACE "^[0-9]+\\.([0-9]+).*$" "\\1" FBX_VERSION_MINOR "${FBX_VERSION}") -string(REGEX REPLACE "^[0-9]+\\.[0-9]+\\.([0-9]+).*$" "\\1" FBX_VERSION_PATCH "${FBX_VERSION}") - -set(FBX_MAC_LOCATIONS "/Applications/Autodesk/FBX\ SDK/${FBX_VERSION}") -set(FBX_LINUX_LOCATIONS "/usr/local/fbxsdk") - -if (WIN32) - string(REGEX REPLACE "\\\\" "/" WIN_PROGRAM_FILES_X64_DIRECTORY $ENV{ProgramW6432}) -endif() - -set(FBX_WIN_LOCATIONS "${WIN_PROGRAM_FILES_X64_DIRECTORY}/Autodesk/FBX/FBX SDK/${FBX_VERSION}") - -set(FBX_SEARCH_LOCATIONS $ENV{FBX_ROOT} ${FBX_ROOT} ${FBX_MAC_LOCATIONS} ${FBX_WIN_LOCATIONS} ${FBX_LINUX_LOCATIONS}) - -function(_fbx_append_debugs _endvar _library) - if (${_library} AND ${_library}_DEBUG) - set(_output optimized ${${_library}} debug ${${_library}_DEBUG}) - else() - set(_output ${${_library}}) - endif() - - set(${_endvar} ${_output} PARENT_SCOPE) -endfunction() - -if (${CMAKE_CXX_COMPILER_ID} MATCHES "Clang") - set(fbx_compiler clang) -elseif (${CMAKE_CXX_COMPILER_ID} MATCHES "GNU") - set(fbx_compiler gcc4) -endif() - -function(_fbx_find_library _name _lib _suffix) - if (MSVC_VERSION EQUAL 1910) - set(VS_PREFIX vs2015) - elseif (MSVC_VERSION EQUAL 1900) - set(VS_PREFIX vs2015) - elseif (MSVC_VERSION EQUAL 1800) - set(VS_PREFIX vs2013) - elseif (MSVC_VERSION EQUAL 1700) - set(VS_PREFIX vs2012) - elseif (MSVC_VERSION EQUAL 1600) - set(VS_PREFIX vs2010) - elseif (MSVC_VERSION EQUAL 1500) - set(VS_PREFIX vs2008) - endif() - - find_library(${_name} - NAMES ${_lib} - HINTS ${FBX_SEARCH_LOCATIONS} - PATH_SUFFIXES lib/${fbx_compiler}/${_suffix} lib/${fbx_compiler}/x64/${_suffix} lib/${fbx_compiler}/ub/${_suffix} lib/${VS_PREFIX}/x64/${_suffix} - ) - - mark_as_advanced(${_name}) -endfunction() - -find_path(FBX_INCLUDE_DIR fbxsdk.h - PATHS ${FBX_SEARCH_LOCATIONS} - PATH_SUFFIXES include -) -mark_as_advanced(FBX_INCLUDE_DIR) - -if (WIN32) - _fbx_find_library(FBX_LIBRARY libfbxsdk-md release) - _fbx_find_library(FBX_LIBRARY_DEBUG libfbxsdk-md debug) -elseif (APPLE) - find_library(CARBON NAMES Carbon) - find_library(SYSTEM_CONFIGURATION NAMES SystemConfiguration) - _fbx_find_library(FBX_LIBRARY libfbxsdk.a release) - _fbx_find_library(FBX_LIBRARY_DEBUG libfbxsdk.a debug) -else () - _fbx_find_library(FBX_LIBRARY libfbxsdk.a release) -endif() - -include(FindPackageHandleStandardArgs) -FIND_PACKAGE_HANDLE_STANDARD_ARGS(FBX DEFAULT_MSG FBX_LIBRARY FBX_INCLUDE_DIR) - -if (FBX_FOUND) - set(FBX_INCLUDE_DIRS ${FBX_INCLUDE_DIR}) - _fbx_append_debugs(FBX_LIBRARIES FBX_LIBRARY) - add_definitions(-DFBXSDK_NEW_API) - - if (WIN32) - add_definitions(-DK_PLUGIN -DK_FBXSDK -DK_NODLL) - set(CMAKE_EXE_LINKER_FLAGS /NODEFAULTLIB:\"LIBCMT\") - set(FBX_LIBRARIES ${FBX_LIBRARIES} Wininet.lib) - elseif (APPLE) - set(FBX_LIBRARIES ${FBX_LIBRARIES} ${CARBON} ${SYSTEM_CONFIGURATION}) - endif() -endif() diff --git a/libraries/baking/CMakeLists.txt b/libraries/baking/CMakeLists.txt index da3389c862..0805c0198c 100644 --- a/libraries/baking/CMakeLists.txt +++ b/libraries/baking/CMakeLists.txt @@ -1,19 +1,5 @@ set(TARGET_NAME baking) setup_hifi_library(Concurrent) -find_package(FBX) -if (FBX_FOUND) - if (CMAKE_THREAD_LIBS_INIT) - target_link_libraries(${TARGET_NAME} ${FBX_LIBRARIES} "${CMAKE_THREAD_LIBS_INIT}") - else () - target_link_libraries(${TARGET_NAME} ${FBX_LIBRARIES}) - endif () - target_include_directories(${TARGET_NAME} SYSTEM PRIVATE ${FBX_INCLUDE_DIR}) - - if (UNIX) - target_link_libraries(${TARGET_NAME} ${CMAKE_DL_LIBS}) - endif (UNIX) -endif () - link_hifi_libraries(shared model networking ktx image fbx) include_hifi_library_headers(gpu) diff --git a/tools/oven/CMakeLists.txt b/tools/oven/CMakeLists.txt index 010b1c25b1..1022c204c5 100644 --- a/tools/oven/CMakeLists.txt +++ b/tools/oven/CMakeLists.txt @@ -17,16 +17,4 @@ if (UNIX) endif() endif () -# try to find the FBX SDK but fail silently if we don't -# because this tool is not built by default -find_package(FBX) -if (FBX_FOUND) - if (CMAKE_THREAD_LIBS_INIT) - target_link_libraries(${TARGET_NAME} ${FBX_LIBRARIES} "${CMAKE_THREAD_LIBS_INIT}") - else () - target_link_libraries(${TARGET_NAME} ${FBX_LIBRARIES}) - endif () - target_include_directories(${TARGET_NAME} SYSTEM PRIVATE ${FBX_INCLUDE_DIR}) -endif () - set_target_properties(${TARGET_NAME} PROPERTIES EXCLUDE_FROM_ALL TRUE EXCLUDE_FROM_DEFAULT_BUILD TRUE) From 0d7b50cfb66370ddcca5b379b16f70430a92a573 Mon Sep 17 00:00:00 2001 From: Stephen Birarda Date: Fri, 8 Sep 2017 09:44:19 -0700 Subject: [PATCH 069/129] remove old texture type method --- libraries/baking/src/FBXBaker.cpp | 52 ------------------------------- 1 file changed, 52 deletions(-) diff --git a/libraries/baking/src/FBXBaker.cpp b/libraries/baking/src/FBXBaker.cpp index 73209ec7cf..e3b1269d37 100644 --- a/libraries/baking/src/FBXBaker.cpp +++ b/libraries/baking/src/FBXBaker.cpp @@ -278,58 +278,6 @@ QUrl FBXBaker::getTextureURL(const QFileInfo& textureFileInfo, QString relativeF return urlToTexture; } -image::TextureUsage::Type textureTypeForMaterialProperty(FbxProperty& property, FbxSurfaceMaterial* material) { - using namespace image::TextureUsage; - - // this is a property we know has a texture, we need to match it to a High Fidelity known texture type - // since that information is passed to the baking process - - // grab the hierarchical name for this property and lowercase it for case-insensitive compare - auto propertyName = QString(property.GetHierarchicalName()).toLower(); - - // figure out the type of the property based on what known value string it matches - if ((propertyName.contains("diffuse") && !propertyName.contains("tex_global_diffuse")) - || propertyName.contains("tex_color_map")) { - return ALBEDO_TEXTURE; - } else if (propertyName.contains("transparentcolor") || propertyName.contains("transparencyfactor")) { - return ALBEDO_TEXTURE; - } else if (propertyName.contains("bump")) { - return BUMP_TEXTURE; - } else if (propertyName.contains("normal")) { - return NORMAL_TEXTURE; - } else if ((propertyName.contains("specular") && !propertyName.contains("tex_global_specular")) - || propertyName.contains("reflection")) { - return SPECULAR_TEXTURE; - } else if (propertyName.contains("tex_metallic_map")) { - return METALLIC_TEXTURE; - } else if (propertyName.contains("shininess")) { - return GLOSS_TEXTURE; - } else if (propertyName.contains("tex_roughness_map")) { - return ROUGHNESS_TEXTURE; - } else if (propertyName.contains("emissive")) { - return EMISSIVE_TEXTURE; - } else if (propertyName.contains("ambientcolor")) { - return LIGHTMAP_TEXTURE; - } else if (propertyName.contains("ambientfactor")) { - // we need to check what the ambient factor is, since that tells Interface to process this texture - // either as an occlusion texture or a light map - auto lambertMaterial = FbxCast(material); - - if (lambertMaterial->AmbientFactor == 0) { - return LIGHTMAP_TEXTURE; - } else if (lambertMaterial->AmbientFactor > 0) { - return OCCLUSION_TEXTURE; - } else { - return UNUSED_TEXTURE; - } - - } else if (propertyName.contains("tex_ao_map")) { - return OCCLUSION_TEXTURE; - } - - return UNUSED_TEXTURE; -} - void FBXBaker::rewriteAndBakeSceneTextures() { using namespace image::TextureUsage; QHash textureTypes; From 91f455159808cbfc3b5b953b622f3048f22ee121 Mon Sep 17 00:00:00 2001 From: Stephen Birarda Date: Tue, 8 Aug 2017 11:24:16 -0700 Subject: [PATCH 070/129] add draco to the oven tool --- cmake/externals/draco/CMakeLists.txt | 35 ++++++++++++++++++++++++++++ cmake/modules/FindDraco.cmake | 30 ++++++++++++++++++++++++ tools/oven/CMakeLists.txt | 5 ++++ 3 files changed, 70 insertions(+) create mode 100644 cmake/externals/draco/CMakeLists.txt create mode 100644 cmake/modules/FindDraco.cmake diff --git a/cmake/externals/draco/CMakeLists.txt b/cmake/externals/draco/CMakeLists.txt new file mode 100644 index 0000000000..8ef1593dc1 --- /dev/null +++ b/cmake/externals/draco/CMakeLists.txt @@ -0,0 +1,35 @@ +set(EXTERNAL_NAME draco) + +if (ANDROID) + set(ANDROID_CMAKE_ARGS "-DCMAKE_TOOLCHAIN_FILE=${CMAKE_TOOLCHAIN_FILE}" "-DANDROID_NATIVE_API_LEVEL=19") +endif () + +include(ExternalProject) +ExternalProject_Add( + ${EXTERNAL_NAME} + URL http://hifi-public.s3.amazonaws.com/dependencies/draco-master-c8b6219.zip + URL_MD5 d60ed5fb2e1792445078f931875e3ee1 + CONFIGURE_COMMAND CMAKE_ARGS ${ANDROID_CMAKE_ARGS} -DCMAKE_BUILD_TYPE=${CMAKE_BUILD_TYPE} -DCMAKE_INSTALL_PREFIX:PATH= + LOG_DOWNLOAD 1 + LOG_CONFIGURE 1 + LOG_BUILD 1 +) + +# Hide this external target (for ide users) +set_target_properties(${EXTERNAL_NAME} PROPERTIES FOLDER "hidden/externals") + +ExternalProject_Get_Property(${EXTERNAL_NAME} INSTALL_DIR) + +string(TOUPPER ${EXTERNAL_NAME} EXTERNAL_NAME_UPPER) +set(${EXTERNAL_NAME_UPPER}_INCLUDE_DIRS ${INSTALL_DIR}/include/draco/src CACHE PATH "List of Draco include directories") + +if (UNIX) + set(LIB_PREFIX "lib") + set(LIB_EXT "a") +elseif (WIN32) + set(LIB_EXT "lib") +endif () + +set(${EXTERNAL_NAME_UPPER}_LIBRARY ${INSTALL_DIR}/lib/${LIB_PREFIX}draco.${LIB_EXT} CACHE FILEPATH "Path to Draco release library") +set(${EXTERNAL_NAME_UPPER}_ENCODER_LIBRARY ${INSTALL_DIR}/lib/${LIB_PREFIX}dracoenc.${LIB_EXT} CACHE FILEPATH "Path to Draco encoder release library") +set(${EXTERNAL_NAME_UPPER}_DECODER_LIBRARY ${INSTALL_DIR}/lib/${LIB_PREFIX}dracodec.${LIB_EXT} CACHE FILEPATH "Path to Draco decoder release library") diff --git a/cmake/modules/FindDraco.cmake b/cmake/modules/FindDraco.cmake new file mode 100644 index 0000000000..f549d410f9 --- /dev/null +++ b/cmake/modules/FindDraco.cmake @@ -0,0 +1,30 @@ +# +# FindDraco.cmake +# +# Try to find Draco libraries and include path. +# Once done this will define +# +# DRACO_FOUND +# DRACO_INCLUDE_DIRS +# DRACO_LIBRARY +# DRACO_ENCODER_LIBRARY +# DRACO_DECODER_LIBRARY +# +# Created on 8/8/2017 by Stephen Birarda +# Copyright 2017 High Fidelity, Inc. +# +# Distributed under the Apache License, Version 2.0. +# See the accompanying file LICENSE or http://www.apache.org/licenses/LICENSE-2.0.html +# + +include("${MACRO_DIR}/HifiLibrarySearchHints.cmake") +hifi_library_search_hints("draco") + +find_path(DRACO_INCLUDE_DIRS draco/core/draco_types.h PATH_SUFFIXES include/draco/src HINTS ${DRACO_SEARCH_DIRS}) + +find_library(DRACO_LIBRARY draco PATH_SUFFIXES "lib" HINTS ${DRACO_SEARCH_DIRS}) +find_library(DRACO_ENCODER_LIBRARY draco PATH_SUFFIXES "lib" HINTS ${DRACO_SEARCH_DIRS}) +find_library(DRACO_DECODER_LIBRARY draco PATH_SUFFIXES "lib" HINTS ${DRACO_SEARCH_DIRS}) + +include(FindPackageHandleStandardArgs) +find_package_handle_standard_args(DRACO DEFAULT_MSG DRACO_INCLUDE_DIRS DRACO_LIBRARY DRACO_ENCODER_LIBRARY DRACO_DECODER_LIBRARY) diff --git a/tools/oven/CMakeLists.txt b/tools/oven/CMakeLists.txt index 1022c204c5..9d5d6c1aad 100644 --- a/tools/oven/CMakeLists.txt +++ b/tools/oven/CMakeLists.txt @@ -6,6 +6,11 @@ link_hifi_libraries(networking shared image gpu ktx fbx baking model) setup_memory_debugger() +add_dependency_external_projects(draco) +find_package(Draco REQUIRED) +target_include_directories(${TARGET_NAME} SYSTEM PRIVATE ${DRACO_INCLUDE_DIRS}) +target_link_libraries(${TARGET_NAME} ${DRACO_LIBRARY} ${DRACO_ENCODER_LIBRARY}) + if (WIN32) package_libraries_for_deployment() endif () From 09d18b5ba91fb67774a759acf92cda0d93acd0fe Mon Sep 17 00:00:00 2001 From: Stephen Birarda Date: Tue, 8 Aug 2017 12:17:27 -0700 Subject: [PATCH 071/129] handle special include path for Draco on WIN --- cmake/externals/draco/CMakeLists.txt | 7 ++++++- cmake/modules/FindDraco.cmake | 2 +- 2 files changed, 7 insertions(+), 2 deletions(-) diff --git a/cmake/externals/draco/CMakeLists.txt b/cmake/externals/draco/CMakeLists.txt index 8ef1593dc1..03a6622a1b 100644 --- a/cmake/externals/draco/CMakeLists.txt +++ b/cmake/externals/draco/CMakeLists.txt @@ -21,7 +21,12 @@ set_target_properties(${EXTERNAL_NAME} PROPERTIES FOLDER "hidden/externals") ExternalProject_Get_Property(${EXTERNAL_NAME} INSTALL_DIR) string(TOUPPER ${EXTERNAL_NAME} EXTERNAL_NAME_UPPER) -set(${EXTERNAL_NAME_UPPER}_INCLUDE_DIRS ${INSTALL_DIR}/include/draco/src CACHE PATH "List of Draco include directories") + +if (UNIX) + set(${EXTERNAL_NAME_UPPER}_INCLUDE_DIRS ${INSTALL_DIR}/include/draco/src CACHE PATH "List of Draco include directories") +else () + set(${EXTERNAL_NAME_UPPER}_INCLUDE_DIRS ${INSTALL_DIR}/include CACHE PATH "List of Draco include directories") +endif () if (UNIX) set(LIB_PREFIX "lib") diff --git a/cmake/modules/FindDraco.cmake b/cmake/modules/FindDraco.cmake index f549d410f9..342797b62e 100644 --- a/cmake/modules/FindDraco.cmake +++ b/cmake/modules/FindDraco.cmake @@ -20,7 +20,7 @@ include("${MACRO_DIR}/HifiLibrarySearchHints.cmake") hifi_library_search_hints("draco") -find_path(DRACO_INCLUDE_DIRS draco/core/draco_types.h PATH_SUFFIXES include/draco/src HINTS ${DRACO_SEARCH_DIRS}) +find_path(DRACO_INCLUDE_DIRS draco/core/draco_types.h PATH_SUFFIXES include/draco/src include HINTS ${DRACO_SEARCH_DIRS}) find_library(DRACO_LIBRARY draco PATH_SUFFIXES "lib" HINTS ${DRACO_SEARCH_DIRS}) find_library(DRACO_ENCODER_LIBRARY draco PATH_SUFFIXES "lib" HINTS ${DRACO_SEARCH_DIRS}) From 9b462171f623b28cadf1c34d4f42bac3521eaaed Mon Sep 17 00:00:00 2001 From: Ryan Huffman Date: Fri, 8 Sep 2017 16:56:23 -0700 Subject: [PATCH 072/129] Add draco support to FBXBaker --- libraries/baking/CMakeLists.txt | 5 + libraries/baking/src/FBXBaker.cpp | 136 ++++++++++++++++++++++++++- libraries/baking/src/FBXBaker.h | 3 +- libraries/fbx/src/FBX.h | 7 ++ libraries/fbx/src/FBXReader.h | 2 +- libraries/fbx/src/FBXReader_Mesh.cpp | 3 +- tools/oven/CMakeLists.txt | 5 - 7 files changed, 148 insertions(+), 13 deletions(-) diff --git a/libraries/baking/CMakeLists.txt b/libraries/baking/CMakeLists.txt index 0805c0198c..66cf791776 100644 --- a/libraries/baking/CMakeLists.txt +++ b/libraries/baking/CMakeLists.txt @@ -3,3 +3,8 @@ setup_hifi_library(Concurrent) link_hifi_libraries(shared model networking ktx image fbx) include_hifi_library_headers(gpu) + +add_dependency_external_projects(draco) +find_package(Draco REQUIRED) +target_include_directories(${TARGET_NAME} SYSTEM PRIVATE ${DRACO_INCLUDE_DIRS}) +target_link_libraries(${TARGET_NAME} ${DRACO_LIBRARY} ${DRACO_ENCODER_LIBRARY}) diff --git a/libraries/baking/src/FBXBaker.cpp b/libraries/baking/src/FBXBaker.cpp index e3b1269d37..f504b3f03c 100644 --- a/libraries/baking/src/FBXBaker.cpp +++ b/libraries/baking/src/FBXBaker.cpp @@ -11,8 +11,6 @@ #include // need this include so we don't get an error looking for std::isnan -#include - #include #include #include @@ -35,6 +33,9 @@ #include "FBXBaker.h" +#include +#include + FBXBaker::FBXBaker(const QUrl& fbxURL, TextureBakerThreadGetter textureThreadGetter, const QString& bakedOutputDir, const QString& originalOutputDir) : _fbxURL(fbxURL), @@ -80,7 +81,8 @@ void FBXBaker::bakeSourceCopy() { return; } - // enumerate the textures found in the scene and start a bake for them + // enumerate the models and textures found in the scene and start a bake for them + rewriteAndBakeSceneModels(); rewriteAndBakeSceneTextures(); if (hasErrors()) { @@ -212,7 +214,7 @@ void FBXBaker::importScene() { qCDebug(model_baking) << "Parsing" << _fbxURL; _rootNode = reader._rootNode = reader.parseFBX(&fbxFile); - _geometry = *reader.extractFBXGeometry({}, _fbxURL.toString()); + _geometry = reader.extractFBXGeometry({}, _fbxURL.toString()); _textureContent = reader._textureContent; } @@ -278,12 +280,136 @@ QUrl FBXBaker::getTextureURL(const QFileInfo& textureFileInfo, QString relativeF return urlToTexture; } +void FBXBaker::rewriteAndBakeSceneModels() { + unsigned int meshIndex = 0; + for (FBXNode& rootChild : _rootNode.children) { + if (rootChild.name == "Objects") { + for (FBXNode& objectChild : rootChild.children) { + if (objectChild.name == "Geometry") { + + // TODO Pull this out of _geometry instead so we don't have to reprocess it + auto extractedMesh = FBXReader::extractMesh(objectChild, meshIndex); + auto mesh = extractedMesh.mesh; + + Q_ASSERT(mesh.normals.size() == 0 || mesh.normals.size() == mesh.vertices.size()); + Q_ASSERT(mesh.colors.size() == 0 || mesh.colors.size() == mesh.vertices.size()); + Q_ASSERT(mesh.texCoords.size() == 0 || mesh.texCoords.size() == mesh.vertices.size()); + + int64_t numTriangles { 0 }; + for (auto& part : mesh.parts) { + Q_ASSERT(part.quadTrianglesIndices.size() % 3 == 0); + Q_ASSERT(part.triangleIndices.size() % 3 == 0); + + numTriangles += part.quadTrianglesIndices.size() / 3; + numTriangles += part.triangleIndices.size() / 3; + } + + draco::TriangleSoupMeshBuilder meshBuilder; + + meshBuilder.Start(numTriangles); + + bool hasNormals { mesh.normals.size() > 0 }; + bool hasColors { mesh.colors.size() > 0 }; + bool hasTexCoords { mesh.texCoords.size() > 0 }; + //bool hasTexCoords1 { mesh.texCoords1.size() > 0 }; + + int normalsAttributeID { -1 }; + int colorsAttributeID { -1 }; + int texCoordsAttributeID { -1 }; + //int texCoords1AttributeID { -1 }; + + const int positionAttributeID = meshBuilder.AddAttribute(draco::GeometryAttribute::POSITION, + 3, draco::DT_FLOAT32); + + const int faceMaterialAttributeID = meshBuilder.AddAttribute( + (draco::GeometryAttribute::Type)DRACO_ATTRIBUTE_MATERIAL_ID, + 1, draco::DT_INT64); + + if (hasNormals) { + normalsAttributeID = meshBuilder.AddAttribute(draco::GeometryAttribute::NORMAL, + 3, draco::DT_FLOAT32); + } + if (hasColors) { + colorsAttributeID = meshBuilder.AddAttribute(draco::GeometryAttribute::COLOR, + 3, draco::DT_FLOAT32); + } + if (hasTexCoords) { + texCoordsAttributeID = meshBuilder.AddAttribute(draco::GeometryAttribute::TEX_COORD, + 2, draco::DT_FLOAT32); + } + + + for (auto& part : mesh.parts) { + //Q_ASSERT(part.quadTrianglesIndices % 3 == 0); + //Q_ASSERT(part.triangleIndices % 3 == 0); + + int64_t materialID = 0; + + for (int i = 0; (i + 2) < part.quadTrianglesIndices.size(); i += 3) { + auto idx0 = part.quadTrianglesIndices[i]; + auto idx1 = part.quadTrianglesIndices[i + 1]; + auto idx2 = part.quadTrianglesIndices[i + 2]; + + auto face = draco::FaceIndex(i / 3); + + meshBuilder.SetPerFaceAttributeValueForFace(faceMaterialAttributeID, face, &materialID); + + meshBuilder.SetAttributeValuesForFace(positionAttributeID, face, + &mesh.vertices[idx0], &mesh.vertices[idx1], + &mesh.vertices[idx2]); + + if (hasNormals) { + meshBuilder.SetAttributeValuesForFace(normalsAttributeID, face, + &mesh.normals[idx0],&mesh.normals[idx1], + &mesh.normals[idx2]); + } + if (hasColors) { + meshBuilder.SetAttributeValuesForFace(colorsAttributeID, face, + &mesh.colors[idx0], &mesh.colors[idx1], + &mesh.colors[idx2]); + } + if (hasTexCoords) { + meshBuilder.SetAttributeValuesForFace(texCoordsAttributeID, face, + &mesh.texCoords[idx0], &mesh.texCoords[idx1], + &mesh.texCoords[idx2]); + } + } + } + + auto dracoMesh = meshBuilder.Finalize(); + + draco::Encoder encoder; + draco::EncoderBuffer buffer; + encoder.EncodeMeshToBuffer(*dracoMesh, &buffer); + + FBXNode dracoMeshNode; + dracoMeshNode.name = "DracoMesh"; + auto value = QVariant::fromValue(QByteArray(buffer.data(), buffer.size())); + dracoMeshNode.properties.append(value); + + + QFile file("C:/Users/huffm/encodedFBX/" + this->_fbxURL.fileName() + "-" + QString::number(meshIndex) + ".drc"); + if (file.open(QIODevice::WriteOnly)) { + file.write(buffer.data(), buffer.size()); + file.close(); + } else { + qWarning() << "Failed to write to: " << file.fileName(); + + } + + objectChild.children.push_back(dracoMeshNode); + } + } + } + } +} + void FBXBaker::rewriteAndBakeSceneTextures() { using namespace image::TextureUsage; QHash textureTypes; // enumerate the materials in the extracted geometry so we can determine the texture type for each texture ID - for (const auto& material : _geometry.materials) { + for (const auto& material : _geometry->materials) { if (material.normalTexture.isBumpmap) { textureTypes[material.normalTexture.id] = BUMP_TEXTURE; } else { diff --git a/libraries/baking/src/FBXBaker.h b/libraries/baking/src/FBXBaker.h index 2dc31828cb..26c1ff2dcc 100644 --- a/libraries/baking/src/FBXBaker.h +++ b/libraries/baking/src/FBXBaker.h @@ -58,6 +58,7 @@ private: void loadSourceFBX(); void importScene(); + void rewriteAndBakeSceneModels(); void rewriteAndBakeSceneTextures(); void exportScene(); void removeEmbeddedMediaFolder(); @@ -73,7 +74,7 @@ private: QUrl _fbxURL; FBXNode _rootNode; - FBXGeometry _geometry; + FBXGeometry* _geometry; QHash _textureContent; QString _bakedFBXFilePath; diff --git a/libraries/fbx/src/FBX.h b/libraries/fbx/src/FBX.h index fc45e9a792..9f5fad7d66 100644 --- a/libraries/fbx/src/FBX.h +++ b/libraries/fbx/src/FBX.h @@ -32,6 +32,13 @@ static const QByteArray FBX_BINARY_PROLOG = "Kaydara FBX Binary "; static const int FBX_HEADER_BYTES_BEFORE_VERSION = 23; static const quint32 FBX_VERSION_2016 = 7500; + +// TODO Convert to GeometryAttribute type +static const int DRACO_BEGIN_CUSTOM_HIFI_ATTRIBUTES = 1000; +static const int DRACO_ATTRIBUTE_MATERIAL_ID = DRACO_BEGIN_CUSTOM_HIFI_ATTRIBUTES; +static const int DRACO_ATTRIBUTE_TEX_COORD_1 = DRACO_BEGIN_CUSTOM_HIFI_ATTRIBUTES + 1; + + class FBXNode; using FBXNodeList = QList; diff --git a/libraries/fbx/src/FBXReader.h b/libraries/fbx/src/FBXReader.h index a600ac6bab..8a5e394da9 100644 --- a/libraries/fbx/src/FBXReader.h +++ b/libraries/fbx/src/FBXReader.h @@ -110,7 +110,7 @@ public: FBXGeometry* extractFBXGeometry(const QVariantHash& mapping, const QString& url); - ExtractedMesh extractMesh(const FBXNode& object, unsigned int& meshIndex); + static ExtractedMesh extractMesh(const FBXNode& object, unsigned int& meshIndex); QHash meshes; static void buildModelMesh(FBXMesh& extractedMesh, const QString& url); diff --git a/libraries/fbx/src/FBXReader_Mesh.cpp b/libraries/fbx/src/FBXReader_Mesh.cpp index 4e153dfe3a..14f1c09b75 100644 --- a/libraries/fbx/src/FBXReader_Mesh.cpp +++ b/libraries/fbx/src/FBXReader_Mesh.cpp @@ -304,8 +304,9 @@ ExtractedMesh FBXReader::extractMesh(const FBXNode& object, unsigned int& meshIn if (subdata.name == "Materials") { materials = getIntVector(subdata); } else if (subdata.name == "MappingInformationType") { - if (subdata.properties.at(0) == BY_POLYGON) + if (subdata.properties.at(0) == BY_POLYGON) { isMaterialPerPolygon = true; + } } else { isMaterialPerPolygon = false; } diff --git a/tools/oven/CMakeLists.txt b/tools/oven/CMakeLists.txt index 9d5d6c1aad..1022c204c5 100644 --- a/tools/oven/CMakeLists.txt +++ b/tools/oven/CMakeLists.txt @@ -6,11 +6,6 @@ link_hifi_libraries(networking shared image gpu ktx fbx baking model) setup_memory_debugger() -add_dependency_external_projects(draco) -find_package(Draco REQUIRED) -target_include_directories(${TARGET_NAME} SYSTEM PRIVATE ${DRACO_INCLUDE_DIRS}) -target_link_libraries(${TARGET_NAME} ${DRACO_LIBRARY} ${DRACO_ENCODER_LIBRARY}) - if (WIN32) package_libraries_for_deployment() endif () From 3d6d383a150e3eb92287e0b0e120c2b384677337 Mon Sep 17 00:00:00 2001 From: Ryan Huffman Date: Fri, 8 Sep 2017 16:58:57 -0700 Subject: [PATCH 073/129] Clean up duplicate list code in encodeFBXProperty --- libraries/fbx/src/FBXWriter.cpp | 57 ++++++++++----------------------- 1 file changed, 17 insertions(+), 40 deletions(-) diff --git a/libraries/fbx/src/FBXWriter.cpp b/libraries/fbx/src/FBXWriter.cpp index f6bb92a8b5..9f389fca88 100644 --- a/libraries/fbx/src/FBXWriter.cpp +++ b/libraries/fbx/src/FBXWriter.cpp @@ -13,6 +13,17 @@ #include +template +void writeVector(QDataStream& out, char ch, QVector list) { + out.device()->write(&ch, 1); + out << (int32_t)list.length(); + out << (int32_t)0; + out << (int32_t)0; + for (auto& value : list) { + out << value; + } +} + QByteArray FBXWriter::encodeFBX(const FBXNode& root) { QByteArray data; @@ -93,6 +104,7 @@ void FBXWriter::encodeFBXProperty(QDataStream& out, const QVariant& prop) { case QVariant::Type::Bool: out.device()->write("C", 1); + //out.device()->write(prop.toBool() ? 1 : 0, 1); out << prop.toBool(); break; @@ -200,50 +212,15 @@ void FBXWriter::encodeFBXProperty(QDataStream& out, const QVariant& prop) { default: { if (prop.canConvert>()) { - auto list = prop.value>(); - out.device()->write("f", 1); - out << (int32_t)list.length(); - out << (int32_t)0; - out << (int32_t)0; - for (auto& value : list) { - out << value; - } + writeVector(out, 'f', prop.value>()); } else if (prop.canConvert>()) { - auto list = prop.value>(); - out.device()->write("d", 1); - out << (int32_t)list.length(); - out << (int32_t)0; - out << (int32_t)0; - for (auto& value : list) { - out << value; - } + writeVector(out, 'd', prop.value>()); } else if (prop.canConvert>()) { - auto list = prop.value>(); - out.device()->write("l", 1); - out << (int32_t)list.length(); - out << (int32_t)0; - out << (int32_t)0; - for (auto& value : list) { - out << value; - } + writeVector(out, 'l', prop.value>()); } else if (prop.canConvert>()) { - auto list = prop.value>(); - out.device()->write("i", 1); - out << (int32_t)list.length(); - out << (int32_t)0; - out << (int32_t)0; - for (auto& value : list) { - out << value; - } + writeVector(out, 'i', prop.value>()); } else if (prop.canConvert>()) { - auto list = prop.value>(); - out.device()->write("b", 1); - out << (int32_t)list.length(); - out << (int32_t)0; - out << (int32_t)0; - for (auto& value : list) { - out << value; - } + writeVector(out, 'b', prop.value>()); } else { qDebug() << "Unsupported property type in FBXWriter::encodeNode: " << type << prop; } From ebd925b966ecdb56d34401f7604651c9263d69e0 Mon Sep 17 00:00:00 2001 From: Ryan Huffman Date: Mon, 11 Sep 2017 10:52:59 -0700 Subject: [PATCH 074/129] Add proper per face materials and texCoords1 to fbx baking --- libraries/baking/src/FBXBaker.cpp | 35 ++++++++++++++++++++++++------- 1 file changed, 27 insertions(+), 8 deletions(-) diff --git a/libraries/baking/src/FBXBaker.cpp b/libraries/baking/src/FBXBaker.cpp index f504b3f03c..b5cd94cec7 100644 --- a/libraries/baking/src/FBXBaker.cpp +++ b/libraries/baking/src/FBXBaker.cpp @@ -311,20 +311,18 @@ void FBXBaker::rewriteAndBakeSceneModels() { bool hasNormals { mesh.normals.size() > 0 }; bool hasColors { mesh.colors.size() > 0 }; bool hasTexCoords { mesh.texCoords.size() > 0 }; - //bool hasTexCoords1 { mesh.texCoords1.size() > 0 }; + bool hasTexCoords1 { mesh.texCoords1.size() > 0 }; + bool hasPerFaceMaterials { mesh.parts.size() > 0 }; int normalsAttributeID { -1 }; int colorsAttributeID { -1 }; int texCoordsAttributeID { -1 }; - //int texCoords1AttributeID { -1 }; + int texCoords1AttributeID { -1 }; + int faceMaterialAttributeID { -1 }; const int positionAttributeID = meshBuilder.AddAttribute(draco::GeometryAttribute::POSITION, 3, draco::DT_FLOAT32); - const int faceMaterialAttributeID = meshBuilder.AddAttribute( - (draco::GeometryAttribute::Type)DRACO_ATTRIBUTE_MATERIAL_ID, - 1, draco::DT_INT64); - if (hasNormals) { normalsAttributeID = meshBuilder.AddAttribute(draco::GeometryAttribute::NORMAL, 3, draco::DT_FLOAT32); @@ -337,13 +335,25 @@ void FBXBaker::rewriteAndBakeSceneModels() { texCoordsAttributeID = meshBuilder.AddAttribute(draco::GeometryAttribute::TEX_COORD, 2, draco::DT_FLOAT32); } + if (hasTexCoords1) { + texCoords1AttributeID = meshBuilder.AddAttribute( + (draco::GeometryAttribute::Type)DRACO_ATTRIBUTE_TEX_COORD_1, + 2, draco::DT_FLOAT32); + } + if (hasPerFaceMaterials) { + faceMaterialAttributeID = meshBuilder.AddAttribute( + (draco::GeometryAttribute::Type)DRACO_ATTRIBUTE_MATERIAL_ID, + 2, draco::DT_INT64); + } + auto partIndex = 0; for (auto& part : mesh.parts) { //Q_ASSERT(part.quadTrianglesIndices % 3 == 0); //Q_ASSERT(part.triangleIndices % 3 == 0); - int64_t materialID = 0; + const auto matTex = extractedMesh.partMaterialTextures[partIndex]; + const int64_t matTexData[2] = { matTex.first, matTex.second }; for (int i = 0; (i + 2) < part.quadTrianglesIndices.size(); i += 3) { auto idx0 = part.quadTrianglesIndices[i]; @@ -352,7 +362,9 @@ void FBXBaker::rewriteAndBakeSceneModels() { auto face = draco::FaceIndex(i / 3); - meshBuilder.SetPerFaceAttributeValueForFace(faceMaterialAttributeID, face, &materialID); + if (hasPerFaceMaterials) { + meshBuilder.SetPerFaceAttributeValueForFace(faceMaterialAttributeID, face, &matTexData); + } meshBuilder.SetAttributeValuesForFace(positionAttributeID, face, &mesh.vertices[idx0], &mesh.vertices[idx1], @@ -373,7 +385,14 @@ void FBXBaker::rewriteAndBakeSceneModels() { &mesh.texCoords[idx0], &mesh.texCoords[idx1], &mesh.texCoords[idx2]); } + if (hasTexCoords1) { + meshBuilder.SetAttributeValuesForFace(texCoords1AttributeID, face, + &mesh.texCoords1[idx0], &mesh.texCoords1[idx1], + &mesh.texCoords1[idx2]); + } } + + partIndex++; } auto dracoMesh = meshBuilder.Finalize(); From b1d0df06d98b4348d37271e11f353a4641b2de39 Mon Sep 17 00:00:00 2001 From: Ryan Huffman Date: Mon, 11 Sep 2017 12:07:41 -0700 Subject: [PATCH 075/129] Add error handling in FBXBaker for empty nodes and failed draco compression --- libraries/baking/src/FBXBaker.cpp | 10 ++++++++++ 1 file changed, 10 insertions(+) diff --git a/libraries/baking/src/FBXBaker.cpp b/libraries/baking/src/FBXBaker.cpp index b5cd94cec7..30c9510a52 100644 --- a/libraries/baking/src/FBXBaker.cpp +++ b/libraries/baking/src/FBXBaker.cpp @@ -304,6 +304,11 @@ void FBXBaker::rewriteAndBakeSceneModels() { numTriangles += part.triangleIndices.size() / 3; } + if (numTriangles == 0) { + qDebug() << "Skipping compression of mesh because no triangles were found"; + continue; + } + draco::TriangleSoupMeshBuilder meshBuilder; meshBuilder.Start(numTriangles); @@ -397,6 +402,11 @@ void FBXBaker::rewriteAndBakeSceneModels() { auto dracoMesh = meshBuilder.Finalize(); + if (!dracoMesh) { + qWarning() << "Failed to finalize the baking of a draco Geometry node"; + continue; + } + draco::Encoder encoder; draco::EncoderBuffer buffer; encoder.EncodeMeshToBuffer(*dracoMesh, &buffer); From 1b2ba3acb686792a2a34a07a1579c98f80eaed4b Mon Sep 17 00:00:00 2001 From: Stephen Birarda Date: Mon, 11 Sep 2017 12:07:12 -0700 Subject: [PATCH 076/129] force libc++ for OS X draco build --- cmake/externals/draco/CMakeLists.txt | 10 ++++++---- 1 file changed, 6 insertions(+), 4 deletions(-) diff --git a/cmake/externals/draco/CMakeLists.txt b/cmake/externals/draco/CMakeLists.txt index 03a6622a1b..41af6646db 100644 --- a/cmake/externals/draco/CMakeLists.txt +++ b/cmake/externals/draco/CMakeLists.txt @@ -4,12 +4,14 @@ if (ANDROID) set(ANDROID_CMAKE_ARGS "-DCMAKE_TOOLCHAIN_FILE=${CMAKE_TOOLCHAIN_FILE}" "-DANDROID_NATIVE_API_LEVEL=19") endif () +set(EXTRA_CMAKE_FLAGS -DCMAKE_CXX_FLAGS=-stdlib=libc++ -DCMAKE_EXE_LINKER_FLAGS=-stdlib=libc++) + include(ExternalProject) ExternalProject_Add( ${EXTERNAL_NAME} - URL http://hifi-public.s3.amazonaws.com/dependencies/draco-master-c8b6219.zip - URL_MD5 d60ed5fb2e1792445078f931875e3ee1 - CONFIGURE_COMMAND CMAKE_ARGS ${ANDROID_CMAKE_ARGS} -DCMAKE_BUILD_TYPE=${CMAKE_BUILD_TYPE} -DCMAKE_INSTALL_PREFIX:PATH= + URL http://hifi-public.s3.amazonaws.com/dependencies/draco-1.0.1.zip + URL_MD5 f1826d5ba1ffd413311d78346f4c114b + CONFIGURE_COMMAND CMAKE_ARGS ${ANDROID_CMAKE_ARGS} -DCMAKE_BUILD_TYPE=${CMAKE_BUILD_TYPE} -DCMAKE_INSTALL_PREFIX:PATH= ${EXTRA_CMAKE_FLAGS} LOG_DOWNLOAD 1 LOG_CONFIGURE 1 LOG_BUILD 1 @@ -22,7 +24,7 @@ ExternalProject_Get_Property(${EXTERNAL_NAME} INSTALL_DIR) string(TOUPPER ${EXTERNAL_NAME} EXTERNAL_NAME_UPPER) -if (UNIX) +if (UNIX AND NOT APPLE) set(${EXTERNAL_NAME_UPPER}_INCLUDE_DIRS ${INSTALL_DIR}/include/draco/src CACHE PATH "List of Draco include directories") else () set(${EXTERNAL_NAME_UPPER}_INCLUDE_DIRS ${INSTALL_DIR}/include CACHE PATH "List of Draco include directories") From 8fc8b8100d31bff229560ee7efbbef6b3c494c5c Mon Sep 17 00:00:00 2001 From: Ryan Huffman Date: Tue, 12 Sep 2017 09:44:44 -0700 Subject: [PATCH 077/129] Fix FBXBaker failing on bad mesh data and draco meshes that can't be encoded --- libraries/baking/src/FBXBaker.cpp | 13 +++++++------ 1 file changed, 7 insertions(+), 6 deletions(-) diff --git a/libraries/baking/src/FBXBaker.cpp b/libraries/baking/src/FBXBaker.cpp index 30c9510a52..45f3836dfc 100644 --- a/libraries/baking/src/FBXBaker.cpp +++ b/libraries/baking/src/FBXBaker.cpp @@ -297,15 +297,16 @@ void FBXBaker::rewriteAndBakeSceneModels() { int64_t numTriangles { 0 }; for (auto& part : mesh.parts) { - Q_ASSERT(part.quadTrianglesIndices.size() % 3 == 0); - Q_ASSERT(part.triangleIndices.size() % 3 == 0); - + if ((part.quadTrianglesIndices.size() % 3) != 0 || (part.triangleIndices.size() % 3) != 0) { + handleWarning("Found a mesh part with invalid index data, skipping"); + continue; + } numTriangles += part.quadTrianglesIndices.size() / 3; numTriangles += part.triangleIndices.size() / 3; } if (numTriangles == 0) { - qDebug() << "Skipping compression of mesh because no triangles were found"; + handleWarning("Skipping compression of mesh because no triangles were found"); continue; } @@ -317,7 +318,7 @@ void FBXBaker::rewriteAndBakeSceneModels() { bool hasColors { mesh.colors.size() > 0 }; bool hasTexCoords { mesh.texCoords.size() > 0 }; bool hasTexCoords1 { mesh.texCoords1.size() > 0 }; - bool hasPerFaceMaterials { mesh.parts.size() > 0 }; + bool hasPerFaceMaterials { mesh.parts.size() > 1 }; int normalsAttributeID { -1 }; int colorsAttributeID { -1 }; @@ -403,7 +404,7 @@ void FBXBaker::rewriteAndBakeSceneModels() { auto dracoMesh = meshBuilder.Finalize(); if (!dracoMesh) { - qWarning() << "Failed to finalize the baking of a draco Geometry node"; + handleWarning("Failed to finalize the baking of a draco Geometry node"); continue; } From 2d9d4322215e3953fa8c66296699a0cf5b3018f4 Mon Sep 17 00:00:00 2001 From: Ryan Huffman Date: Tue, 12 Sep 2017 09:45:24 -0700 Subject: [PATCH 078/129] Update FBXBaker to remove unused nodes from the Geometry node --- libraries/baking/src/FBXBaker.cpp | 28 ++++++++++++++++++++++++++++ 1 file changed, 28 insertions(+) diff --git a/libraries/baking/src/FBXBaker.cpp b/libraries/baking/src/FBXBaker.cpp index 45f3836dfc..760141f029 100644 --- a/libraries/baking/src/FBXBaker.cpp +++ b/libraries/baking/src/FBXBaker.cpp @@ -428,6 +428,34 @@ void FBXBaker::rewriteAndBakeSceneModels() { } objectChild.children.push_back(dracoMeshNode); + + static const std::vector nodeNamesToDelete { + // Node data that is packed into the draco mesh + "Vertices", + "PolygonVertexIndex", + "LayerElementNormal", + "LayerElementColor", + "LayerElementUV", + "LayerElementMaterial", + "LayerElementTexture", + + // Node data that we don't support + "Edges", + "LayerElementTangent", + "LayerElementBinormal", + "LayerElementSmoothing" + }; + auto& children = objectChild.children; + auto it = children.begin(); + while (it != children.end()) { + auto begin = nodeNamesToDelete.begin(); + auto end = nodeNamesToDelete.end(); + if (find(begin, end, it->name) != end) { + it = children.erase(it); + } else { + ++it; + } + } } } } From 9243cf75907677f7a92b39c8f4c1aea4e5d659c0 Mon Sep 17 00:00:00 2001 From: Ryan Huffman Date: Tue, 12 Sep 2017 09:45:46 -0700 Subject: [PATCH 079/129] Remove extraneous logging in FBXWriter --- libraries/fbx/src/FBXWriter.cpp | 6 ------ 1 file changed, 6 deletions(-) diff --git a/libraries/fbx/src/FBXWriter.cpp b/libraries/fbx/src/FBXWriter.cpp index 9f389fca88..c14e383402 100644 --- a/libraries/fbx/src/FBXWriter.cpp +++ b/libraries/fbx/src/FBXWriter.cpp @@ -63,12 +63,6 @@ void FBXWriter::encodeNode(QDataStream& out, const FBXNode& node) { out << (quint8)node.name.size(); out.writeRawData(node.name, node.name.size()); - if (node.name == "Vertices") { - for (auto& prop : node.properties) { - qDebug() << "Properties: " << prop; - } - } - auto nodePropertiesStartPos = device->pos(); for (const auto& prop : node.properties) { From f9cc82c992a3703d8e51ff2ebbb56154a741b2e1 Mon Sep 17 00:00:00 2001 From: Stephen Birarda Date: Mon, 11 Sep 2017 18:09:51 -0700 Subject: [PATCH 080/129] use draco 1.1.0 in external --- cmake/externals/draco/CMakeLists.txt | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/cmake/externals/draco/CMakeLists.txt b/cmake/externals/draco/CMakeLists.txt index 41af6646db..07ce861820 100644 --- a/cmake/externals/draco/CMakeLists.txt +++ b/cmake/externals/draco/CMakeLists.txt @@ -9,8 +9,8 @@ set(EXTRA_CMAKE_FLAGS -DCMAKE_CXX_FLAGS=-stdlib=libc++ -DCMAKE_EXE_LINKER_FLAGS= include(ExternalProject) ExternalProject_Add( ${EXTERNAL_NAME} - URL http://hifi-public.s3.amazonaws.com/dependencies/draco-1.0.1.zip - URL_MD5 f1826d5ba1ffd413311d78346f4c114b + URL http://hifi-public.s3.amazonaws.com/dependencies/draco-1.1.0.zip + URL_MD5 208f8b04c91d5f1c73d731a3ea37c5bb CONFIGURE_COMMAND CMAKE_ARGS ${ANDROID_CMAKE_ARGS} -DCMAKE_BUILD_TYPE=${CMAKE_BUILD_TYPE} -DCMAKE_INSTALL_PREFIX:PATH= ${EXTRA_CMAKE_FLAGS} LOG_DOWNLOAD 1 LOG_CONFIGURE 1 From 2105f2da9210bb4206c95c235e38184bd95e364d Mon Sep 17 00:00:00 2001 From: Stephen Birarda Date: Mon, 11 Sep 2017 18:21:44 -0700 Subject: [PATCH 081/129] change unique ID for material and second tex coord --- libraries/baking/src/FBXBaker.cpp | 10 ++++++++++ 1 file changed, 10 insertions(+) diff --git a/libraries/baking/src/FBXBaker.cpp b/libraries/baking/src/FBXBaker.cpp index 760141f029..18e7dbbd76 100644 --- a/libraries/baking/src/FBXBaker.cpp +++ b/libraries/baking/src/FBXBaker.cpp @@ -408,6 +408,16 @@ void FBXBaker::rewriteAndBakeSceneModels() { continue; } + // we need to modify unique attribute IDs for custom attributes + // so the attributes are easily retrievable on the other side + if (hasPerFaceMaterials) { + dracoMesh->attribute(faceMaterialAttributeID)->set_unique_id(DRACO_ATTRIBUTE_MATERIAL_ID); + } + + if (hasTexCoords1) { + dracoMesh->attribute(texCoords1AttributeID)->set_unique_id(DRACO_ATTRIBUTE_TEX_COORD_1); + } + draco::Encoder encoder; draco::EncoderBuffer buffer; encoder.EncodeMeshToBuffer(*dracoMesh, &buffer); From 61314949ec1fa45dc5868f102a1445a71d21facb Mon Sep 17 00:00:00 2001 From: Stephen Birarda Date: Tue, 12 Sep 2017 12:54:45 -0700 Subject: [PATCH 082/129] add draco as external to FBX library --- libraries/fbx/CMakeLists.txt | 7 +++++-- 1 file changed, 5 insertions(+), 2 deletions(-) diff --git a/libraries/fbx/CMakeLists.txt b/libraries/fbx/CMakeLists.txt index 3355ccd06e..7cead5aa4f 100644 --- a/libraries/fbx/CMakeLists.txt +++ b/libraries/fbx/CMakeLists.txt @@ -1,7 +1,10 @@ set(TARGET_NAME fbx) setup_hifi_library() - - link_hifi_libraries(shared model networking image) include_hifi_library_headers(gpu image) + +add_dependency_external_projects(draco) +find_package(Draco REQUIRED) +target_include_directories(${TARGET_NAME} SYSTEM PRIVATE ${DRACO_INCLUDE_DIRS}) +target_link_libraries(${TARGET_NAME} ${DRACO_LIBRARY} ${DRACO_ENCODER_LIBRARY}) From cf282dd3fc992d5f311c98294b6249ef634131a3 Mon Sep 17 00:00:00 2001 From: Stephen Birarda Date: Tue, 12 Sep 2017 13:42:45 -0700 Subject: [PATCH 083/129] add code to read draco mesh from FBX --- libraries/fbx/src/FBXReader_Mesh.cpp | 236 ++++++++++++++++++++------- 1 file changed, 178 insertions(+), 58 deletions(-) diff --git a/libraries/fbx/src/FBXReader_Mesh.cpp b/libraries/fbx/src/FBXReader_Mesh.cpp index 14f1c09b75..4af68b8878 100644 --- a/libraries/fbx/src/FBXReader_Mesh.cpp +++ b/libraries/fbx/src/FBXReader_Mesh.cpp @@ -9,6 +9,8 @@ // See the accompanying file LICENSE or http://www.apache.org/licenses/LICENSE-2.0.html // +#include + #include #include #include @@ -168,11 +170,17 @@ void appendIndex(MeshData& data, QVector& indices, int index) { ExtractedMesh FBXReader::extractMesh(const FBXNode& object, unsigned int& meshIndex) { MeshData data; data.extracted.mesh.meshIndex = meshIndex++; + QVector materials; QVector textures; + bool isMaterialPerPolygon = false; + static const QVariant BY_VERTICE = QByteArray("ByVertice"); static const QVariant INDEX_TO_DIRECT = QByteArray("IndexToDirect"); + + bool isDracoMesh = false; + foreach (const FBXNode& child, object.children) { if (child.name == "Vertices") { data.vertices = createVec3Vector(getDoubleVector(child)); @@ -319,70 +327,182 @@ ExtractedMesh FBXReader::extractMesh(const FBXNode& object, unsigned int& meshIn textures = getIntVector(subdata); } } - } - } + } else if (child.name == "DracoMesh") { + isDracoMesh = true; - bool isMultiMaterial = false; - if (isMaterialPerPolygon) { - isMultiMaterial = true; - } - // TODO: make excellent use of isMultiMaterial - Q_UNUSED(isMultiMaterial); + // load the draco mesh from the FBX and create a draco::Mesh + draco::Decoder decoder; + draco::DecoderBuffer decodedBuffer; + QByteArray dracoArray = child.properties.at(0).value(); + decodedBuffer.Init(dracoArray.data(), dracoArray.size()); - // convert the polygons to quads and triangles - int polygonIndex = 0; - QHash, int> materialTextureParts; - for (int beginIndex = 0; beginIndex < data.polygonIndices.size(); polygonIndex++) { - int endIndex = beginIndex; - while (endIndex < data.polygonIndices.size() && data.polygonIndices.at(endIndex++) >= 0); + std::unique_ptr dracoMesh(new draco::Mesh()); + decoder.DecodeBufferToGeometry(&decodedBuffer, dracoMesh.get()); - QPair materialTexture((polygonIndex < materials.size()) ? materials.at(polygonIndex) : 0, - (polygonIndex < textures.size()) ? textures.at(polygonIndex) : 0); - int& partIndex = materialTextureParts[materialTexture]; - if (partIndex == 0) { - data.extracted.partMaterialTextures.append(materialTexture); - data.extracted.mesh.parts.resize(data.extracted.mesh.parts.size() + 1); - partIndex = data.extracted.mesh.parts.size(); - } - FBXMeshPart& part = data.extracted.mesh.parts[partIndex - 1]; - - if (endIndex - beginIndex == 4) { - appendIndex(data, part.quadIndices, beginIndex++); - appendIndex(data, part.quadIndices, beginIndex++); - appendIndex(data, part.quadIndices, beginIndex++); - appendIndex(data, part.quadIndices, beginIndex++); + // read positions from draco mesh to extracted mesh + auto positionAttribute = dracoMesh->GetNamedAttribute(draco::GeometryAttribute::POSITION); + if (positionAttribute) { + std::array positionValue; - int quadStartIndex = part.quadIndices.size() - 4; - int i0 = part.quadIndices[quadStartIndex + 0]; - int i1 = part.quadIndices[quadStartIndex + 1]; - int i2 = part.quadIndices[quadStartIndex + 2]; - int i3 = part.quadIndices[quadStartIndex + 3]; - - // Sam's recommended triangle slices - // Triangle tri1 = { v0, v1, v3 }; - // Triangle tri2 = { v1, v2, v3 }; - // NOTE: Random guy on the internet's recommended triangle slices - // Triangle tri1 = { v0, v1, v2 }; - // Triangle tri2 = { v2, v3, v0 }; - - part.quadTrianglesIndices.append(i0); - part.quadTrianglesIndices.append(i1); - part.quadTrianglesIndices.append(i3); - - part.quadTrianglesIndices.append(i1); - part.quadTrianglesIndices.append(i2); - part.quadTrianglesIndices.append(i3); - - } else { - for (int nextIndex = beginIndex + 1;; ) { - appendIndex(data, part.triangleIndices, beginIndex); - appendIndex(data, part.triangleIndices, nextIndex++); - appendIndex(data, part.triangleIndices, nextIndex); - if (nextIndex >= data.polygonIndices.size() || data.polygonIndices.at(nextIndex) < 0) { - break; + for (draco::AttributeValueIndex i (0); i < positionAttribute->size(); ++i) { + positionAttribute->ConvertValue(i, &positionValue[0]); + data.extracted.mesh.vertices.append({ positionValue[0], positionValue[1], positionValue[2] }); } } - beginIndex = endIndex; + + // enumerate the faces from draco mesh to collect vertex indices + for (int i = 0; i < dracoMesh->num_faces() * 3; ++i) { + auto vertexIndex = dracoMesh->face(draco::FaceIndex(i / 3))[i % 3]; + auto mappedIndex = positionAttribute->mapped_index(vertexIndex).value(); + data.extracted.newIndices.insert(mappedIndex, mappedIndex); + } + + // read normals from draco mesh to extracted mesh + auto normalAttribute = dracoMesh->GetNamedAttribute(draco::GeometryAttribute::NORMAL); + if (normalAttribute) { + std::array normalValue; + + for (draco::AttributeValueIndex i (0); i < normalAttribute->size(); ++i) { + normalAttribute->ConvertValue(i, &normalValue[0]); + data.extracted.mesh.normals.append({ normalValue[0], normalValue[1], normalValue[2] }); + } + } + + // read UVs from draco mesh to extracted mesh + auto texCoordAttribute = dracoMesh->GetNamedAttribute(draco::GeometryAttribute::TEX_COORD); + if (texCoordAttribute) { + std::array texCoordValue; + + for (draco::AttributeValueIndex i (0); i < texCoordAttribute->size(); ++i) { + texCoordAttribute->ConvertValue(i, &texCoordValue[0]); + data.extracted.mesh.texCoords.append({ texCoordValue[0], texCoordValue[1] }); + } + } + + // some meshes have a second set of UVs, read those to extracted mesh + auto extraTexCoordAttribute = dracoMesh->GetAttributeByUniqueId(DRACO_ATTRIBUTE_TEX_COORD_1); + if (extraTexCoordAttribute) { + std::array texCoordValue; + + for (draco::AttributeValueIndex i (0); i < extraTexCoordAttribute->size(); ++i) { + extraTexCoordAttribute->ConvertValue(i, &texCoordValue[0]); + data.extracted.mesh.texCoords1.append({ texCoordValue[0], texCoordValue[1] }); + } + } + + // read vertex colors from draco mesh to extracted mesh + auto colorAttribute = dracoMesh->GetNamedAttribute(draco::GeometryAttribute::COLOR); + if (colorAttribute) { + std::array colorValue; + + for (draco::AttributeValueIndex i (0); i < colorAttribute->size(); ++i) { + colorAttribute->ConvertValue(i, &colorValue[0]); + data.extracted.mesh.colors.append({ colorValue[0], colorValue[1], colorValue[2] }); + } + } + + // read material ID and texture ID mappings into materials and texture vectors + auto matTexAttribute = dracoMesh->GetAttributeByUniqueId(DRACO_ATTRIBUTE_MATERIAL_ID); + if (matTexAttribute) { + std::array matTexValue; + + for (draco::AttributeValueIndex i (0); i < matTexAttribute->size(); ++i) { + matTexAttribute->ConvertValue(i, &matTexValue[0]); + materials.append(matTexValue[0]); + textures.append(matTexValue[1]); + } + } + + // enumerate the faces and construct the extracted mesh + auto vertexIndices = data.extracted.newIndices.keys(); + QHash, int> materialTextureParts; + + for (auto i = 0; i < vertexIndices.size(); i += 3) { + // grab the material ID and texture ID for this face, if we have it + QPair materialTexture(materials.at(i), textures.at(i)); + + // grab or setup the FBXMeshPart for the part this face belongs to + int& partIndex = materialTextureParts[materialTexture]; + if (partIndex == 0) { + data.extracted.partMaterialTextures.append(materialTexture); + data.extracted.mesh.parts.resize(data.extracted.mesh.parts.size() + 1); + partIndex = data.extracted.mesh.parts.size() - 1; + } + + FBXMeshPart& part = data.extracted.mesh.parts[partIndex]; + + // give the mesh part its indices + part.triangleIndices.append({ vertexIndices[i], vertexIndices[i + 1], vertexIndices[i + 2]}); + } + + } + } + + // when we have a draco mesh, we've already built the extracted mesh, so we don't need to do the + // processing we do for normal meshes below + if (!isDracoMesh) { + bool isMultiMaterial = false; + if (isMaterialPerPolygon) { + isMultiMaterial = true; + } + // TODO: make excellent use of isMultiMaterial + Q_UNUSED(isMultiMaterial); + + // convert the polygons to quads and triangles + int polygonIndex = 0; + QHash, int> materialTextureParts; + for (int beginIndex = 0; beginIndex < data.polygonIndices.size(); polygonIndex++) { + int endIndex = beginIndex; + while (endIndex < data.polygonIndices.size() && data.polygonIndices.at(endIndex++) >= 0); + + QPair materialTexture((polygonIndex < materials.size()) ? materials.at(polygonIndex) : 0, + (polygonIndex < textures.size()) ? textures.at(polygonIndex) : 0); + int& partIndex = materialTextureParts[materialTexture]; + if (partIndex == 0) { + data.extracted.partMaterialTextures.append(materialTexture); + data.extracted.mesh.parts.resize(data.extracted.mesh.parts.size() + 1); + partIndex = data.extracted.mesh.parts.size(); + } + FBXMeshPart& part = data.extracted.mesh.parts[partIndex - 1]; + + if (endIndex - beginIndex == 4) { + appendIndex(data, part.quadIndices, beginIndex++); + appendIndex(data, part.quadIndices, beginIndex++); + appendIndex(data, part.quadIndices, beginIndex++); + appendIndex(data, part.quadIndices, beginIndex++); + + int quadStartIndex = part.quadIndices.size() - 4; + int i0 = part.quadIndices[quadStartIndex + 0]; + int i1 = part.quadIndices[quadStartIndex + 1]; + int i2 = part.quadIndices[quadStartIndex + 2]; + int i3 = part.quadIndices[quadStartIndex + 3]; + + // Sam's recommended triangle slices + // Triangle tri1 = { v0, v1, v3 }; + // Triangle tri2 = { v1, v2, v3 }; + // NOTE: Random guy on the internet's recommended triangle slices + // Triangle tri1 = { v0, v1, v2 }; + // Triangle tri2 = { v2, v3, v0 }; + + part.quadTrianglesIndices.append(i0); + part.quadTrianglesIndices.append(i1); + part.quadTrianglesIndices.append(i3); + + part.quadTrianglesIndices.append(i1); + part.quadTrianglesIndices.append(i2); + part.quadTrianglesIndices.append(i3); + + } else { + for (int nextIndex = beginIndex + 1;; ) { + appendIndex(data, part.triangleIndices, beginIndex); + appendIndex(data, part.triangleIndices, nextIndex++); + appendIndex(data, part.triangleIndices, nextIndex); + if (nextIndex >= data.polygonIndices.size() || data.polygonIndices.at(nextIndex) < 0) { + break; + } + } + beginIndex = endIndex; + } } } From 5d9ed7836915b240e289ad6bccd9f249c830819c Mon Sep 17 00:00:00 2001 From: Stephen Birarda Date: Tue, 12 Sep 2017 15:57:00 -0700 Subject: [PATCH 084/129] fixes for extracted mesh construction --- libraries/fbx/src/FBXReader_Mesh.cpp | 145 +++++++++++++-------------- 1 file changed, 72 insertions(+), 73 deletions(-) diff --git a/libraries/fbx/src/FBXReader_Mesh.cpp b/libraries/fbx/src/FBXReader_Mesh.cpp index 4af68b8878..2d4e722227 100644 --- a/libraries/fbx/src/FBXReader_Mesh.cpp +++ b/libraries/fbx/src/FBXReader_Mesh.cpp @@ -339,87 +339,86 @@ ExtractedMesh FBXReader::extractMesh(const FBXNode& object, unsigned int& meshIn std::unique_ptr dracoMesh(new draco::Mesh()); decoder.DecodeBufferToGeometry(&decodedBuffer, dracoMesh.get()); - // read positions from draco mesh to extracted mesh + // prepare attributes for this mesh auto positionAttribute = dracoMesh->GetNamedAttribute(draco::GeometryAttribute::POSITION); - if (positionAttribute) { - std::array positionValue; - - for (draco::AttributeValueIndex i (0); i < positionAttribute->size(); ++i) { - positionAttribute->ConvertValue(i, &positionValue[0]); - data.extracted.mesh.vertices.append({ positionValue[0], positionValue[1], positionValue[2] }); - } - } - - // enumerate the faces from draco mesh to collect vertex indices - for (int i = 0; i < dracoMesh->num_faces() * 3; ++i) { - auto vertexIndex = dracoMesh->face(draco::FaceIndex(i / 3))[i % 3]; - auto mappedIndex = positionAttribute->mapped_index(vertexIndex).value(); - data.extracted.newIndices.insert(mappedIndex, mappedIndex); - } - - // read normals from draco mesh to extracted mesh auto normalAttribute = dracoMesh->GetNamedAttribute(draco::GeometryAttribute::NORMAL); - if (normalAttribute) { - std::array normalValue; - - for (draco::AttributeValueIndex i (0); i < normalAttribute->size(); ++i) { - normalAttribute->ConvertValue(i, &normalValue[0]); - data.extracted.mesh.normals.append({ normalValue[0], normalValue[1], normalValue[2] }); - } - } - - // read UVs from draco mesh to extracted mesh auto texCoordAttribute = dracoMesh->GetNamedAttribute(draco::GeometryAttribute::TEX_COORD); - if (texCoordAttribute) { - std::array texCoordValue; - - for (draco::AttributeValueIndex i (0); i < texCoordAttribute->size(); ++i) { - texCoordAttribute->ConvertValue(i, &texCoordValue[0]); - data.extracted.mesh.texCoords.append({ texCoordValue[0], texCoordValue[1] }); - } - } - - // some meshes have a second set of UVs, read those to extracted mesh auto extraTexCoordAttribute = dracoMesh->GetAttributeByUniqueId(DRACO_ATTRIBUTE_TEX_COORD_1); - if (extraTexCoordAttribute) { - std::array texCoordValue; - - for (draco::AttributeValueIndex i (0); i < extraTexCoordAttribute->size(); ++i) { - extraTexCoordAttribute->ConvertValue(i, &texCoordValue[0]); - data.extracted.mesh.texCoords1.append({ texCoordValue[0], texCoordValue[1] }); - } - } - - // read vertex colors from draco mesh to extracted mesh auto colorAttribute = dracoMesh->GetNamedAttribute(draco::GeometryAttribute::COLOR); - if (colorAttribute) { - std::array colorValue; - - for (draco::AttributeValueIndex i (0); i < colorAttribute->size(); ++i) { - colorAttribute->ConvertValue(i, &colorValue[0]); - data.extracted.mesh.colors.append({ colorValue[0], colorValue[1], colorValue[2] }); - } - } - - // read material ID and texture ID mappings into materials and texture vectors auto matTexAttribute = dracoMesh->GetAttributeByUniqueId(DRACO_ATTRIBUTE_MATERIAL_ID); - if (matTexAttribute) { - std::array matTexValue; - for (draco::AttributeValueIndex i (0); i < matTexAttribute->size(); ++i) { - matTexAttribute->ConvertValue(i, &matTexValue[0]); - materials.append(matTexValue[0]); - textures.append(matTexValue[1]); - } - } + // setup extracted mesh data structures given number of points + auto numVertices = dracoMesh->num_points(); - // enumerate the faces and construct the extracted mesh - auto vertexIndices = data.extracted.newIndices.keys(); QHash, int> materialTextureParts; - for (auto i = 0; i < vertexIndices.size(); i += 3) { + data.extracted.mesh.vertices.reserve(numVertices); + data.extracted.mesh.normals.reserve(numVertices); + data.extracted.mesh.texCoords.reserve(numVertices); + data.extracted.mesh.texCoords1.reserve(numVertices); + data.extracted.mesh.colors.reserve(numVertices); + + // enumerate the vertices and construct the extracted mesh + for (int i = 0; i < numVertices; ++i) { + draco::PointIndex vertexIndex(i); + + if (positionAttribute) { + // read position from draco mesh to extracted mesh + auto mappedIndex = positionAttribute->mapped_index(vertexIndex); + + std::array positionValue; + positionAttribute->ConvertValue(mappedIndex, &positionValue[0]); + data.extracted.mesh.vertices.append({ positionValue[0], positionValue[1], positionValue[2] }); + } + + if (normalAttribute) { + // read normals from draco mesh to extracted mesh + auto mappedIndex = normalAttribute->mapped_index(vertexIndex); + + std::array normalValue; + normalAttribute->ConvertValue(mappedIndex, &normalValue[0]); + data.extracted.mesh.normals.append({ normalValue[0], normalValue[1], normalValue[2] }); + } + + if (texCoordAttribute) { + // read UVs from draco mesh to extracted mesh + auto mappedIndex = texCoordAttribute->mapped_index(vertexIndex); + + std::array texCoordValue; + texCoordAttribute->ConvertValue(mappedIndex, &texCoordValue[0]); + data.extracted.mesh.texCoords.append({ texCoordValue[0], texCoordValue[1] }); + } + + if (extraTexCoordAttribute) { + // some meshes have a second set of UVs, read those to extracted mesh + auto mappedIndex = extraTexCoordAttribute->mapped_index(vertexIndex); + + std::array texCoordValue; + extraTexCoordAttribute->ConvertValue(mappedIndex, &texCoordValue[0]); + data.extracted.mesh.texCoords1.append({ texCoordValue[0], texCoordValue[1] }); + } + + if (colorAttribute) { + // read vertex colors from draco mesh to extracted mesh + auto mappedIndex = colorAttribute->mapped_index(vertexIndex); + + std::array colorValue; + + colorAttribute->ConvertValue(mappedIndex, &colorValue[0]); + data.extracted.mesh.colors.append({ colorValue[0], colorValue[1], colorValue[2] }); + } + + int64_t matTexValue[2] = { 0, 0 }; + + if (matTexAttribute) { + // read material ID and texture ID mappings into materials and texture vectors + auto mappedIndex = matTexAttribute->mapped_index(vertexIndex); + + matTexAttribute->ConvertValue(mappedIndex, &matTexValue[0]); + } + // grab the material ID and texture ID for this face, if we have it - QPair materialTexture(materials.at(i), textures.at(i)); + QPair materialTexture(matTexValue[0], matTexValue[1]); // grab or setup the FBXMeshPart for the part this face belongs to int& partIndex = materialTextureParts[materialTexture]; @@ -429,12 +428,12 @@ ExtractedMesh FBXReader::extractMesh(const FBXNode& object, unsigned int& meshIn partIndex = data.extracted.mesh.parts.size() - 1; } + // give the mesh part this index FBXMeshPart& part = data.extracted.mesh.parts[partIndex]; + part.triangleIndices.append(i); - // give the mesh part its indices - part.triangleIndices.append({ vertexIndices[i], vertexIndices[i + 1], vertexIndices[i + 2]}); + data.extracted.newIndices.insert(i, i); } - } } From bb7cd58e9a12dd3a04975b8decfd627a6f987c47 Mon Sep 17 00:00:00 2001 From: Stephen Birarda Date: Tue, 12 Sep 2017 16:28:12 -0700 Subject: [PATCH 085/129] fix reference to part index --- libraries/fbx/src/FBXReader_Mesh.cpp | 8 ++++---- 1 file changed, 4 insertions(+), 4 deletions(-) diff --git a/libraries/fbx/src/FBXReader_Mesh.cpp b/libraries/fbx/src/FBXReader_Mesh.cpp index 2d4e722227..30167d9eb5 100644 --- a/libraries/fbx/src/FBXReader_Mesh.cpp +++ b/libraries/fbx/src/FBXReader_Mesh.cpp @@ -421,15 +421,15 @@ ExtractedMesh FBXReader::extractMesh(const FBXNode& object, unsigned int& meshIn QPair materialTexture(matTexValue[0], matTexValue[1]); // grab or setup the FBXMeshPart for the part this face belongs to - int& partIndex = materialTextureParts[materialTexture]; - if (partIndex == 0) { + int& partIndexPlusOne = materialTextureParts[materialTexture]; + if (partIndexPlusOne == 0) { data.extracted.partMaterialTextures.append(materialTexture); data.extracted.mesh.parts.resize(data.extracted.mesh.parts.size() + 1); - partIndex = data.extracted.mesh.parts.size() - 1; + partIndexPlusOne = data.extracted.mesh.parts.size(); } // give the mesh part this index - FBXMeshPart& part = data.extracted.mesh.parts[partIndex]; + FBXMeshPart& part = data.extracted.mesh.parts[partIndexPlusOne - 1]; part.triangleIndices.append(i); data.extracted.newIndices.insert(i, i); From a25e5796dc2d40fbf6850dc51ae125154e7f39bc Mon Sep 17 00:00:00 2001 From: Stephen Birarda Date: Tue, 12 Sep 2017 16:35:21 -0700 Subject: [PATCH 086/129] use push_back instead of append for extracted mesh vectors --- libraries/fbx/src/FBXReader_Mesh.cpp | 10 +++++----- 1 file changed, 5 insertions(+), 5 deletions(-) diff --git a/libraries/fbx/src/FBXReader_Mesh.cpp b/libraries/fbx/src/FBXReader_Mesh.cpp index 30167d9eb5..2ff93545ff 100644 --- a/libraries/fbx/src/FBXReader_Mesh.cpp +++ b/libraries/fbx/src/FBXReader_Mesh.cpp @@ -368,7 +368,7 @@ ExtractedMesh FBXReader::extractMesh(const FBXNode& object, unsigned int& meshIn std::array positionValue; positionAttribute->ConvertValue(mappedIndex, &positionValue[0]); - data.extracted.mesh.vertices.append({ positionValue[0], positionValue[1], positionValue[2] }); + data.extracted.mesh.vertices.push_back({ positionValue[0], positionValue[1], positionValue[2] }); } if (normalAttribute) { @@ -377,7 +377,7 @@ ExtractedMesh FBXReader::extractMesh(const FBXNode& object, unsigned int& meshIn std::array normalValue; normalAttribute->ConvertValue(mappedIndex, &normalValue[0]); - data.extracted.mesh.normals.append({ normalValue[0], normalValue[1], normalValue[2] }); + data.extracted.mesh.normals.push_back({ normalValue[0], normalValue[1], normalValue[2] }); } if (texCoordAttribute) { @@ -386,7 +386,7 @@ ExtractedMesh FBXReader::extractMesh(const FBXNode& object, unsigned int& meshIn std::array texCoordValue; texCoordAttribute->ConvertValue(mappedIndex, &texCoordValue[0]); - data.extracted.mesh.texCoords.append({ texCoordValue[0], texCoordValue[1] }); + data.extracted.mesh.texCoords.push_back({ texCoordValue[0], texCoordValue[1] }); } if (extraTexCoordAttribute) { @@ -395,7 +395,7 @@ ExtractedMesh FBXReader::extractMesh(const FBXNode& object, unsigned int& meshIn std::array texCoordValue; extraTexCoordAttribute->ConvertValue(mappedIndex, &texCoordValue[0]); - data.extracted.mesh.texCoords1.append({ texCoordValue[0], texCoordValue[1] }); + data.extracted.mesh.texCoords1.push_back({ texCoordValue[0], texCoordValue[1] }); } if (colorAttribute) { @@ -405,7 +405,7 @@ ExtractedMesh FBXReader::extractMesh(const FBXNode& object, unsigned int& meshIn std::array colorValue; colorAttribute->ConvertValue(mappedIndex, &colorValue[0]); - data.extracted.mesh.colors.append({ colorValue[0], colorValue[1], colorValue[2] }); + data.extracted.mesh.colors.push_back({ colorValue[0], colorValue[1], colorValue[2] }); } int64_t matTexValue[2] = { 0, 0 }; From b253d3b57c8f59a39729aa3e14934368f86ccb8d Mon Sep 17 00:00:00 2001 From: Stephen Birarda Date: Tue, 12 Sep 2017 17:14:48 -0700 Subject: [PATCH 087/129] fix material triangle index insertion --- libraries/fbx/src/FBXReader_Mesh.cpp | 16 +++++++++++----- 1 file changed, 11 insertions(+), 5 deletions(-) diff --git a/libraries/fbx/src/FBXReader_Mesh.cpp b/libraries/fbx/src/FBXReader_Mesh.cpp index 2ff93545ff..f1524b1a81 100644 --- a/libraries/fbx/src/FBXReader_Mesh.cpp +++ b/libraries/fbx/src/FBXReader_Mesh.cpp @@ -408,16 +408,22 @@ ExtractedMesh FBXReader::extractMesh(const FBXNode& object, unsigned int& meshIn data.extracted.mesh.colors.push_back({ colorValue[0], colorValue[1], colorValue[2] }); } + data.extracted.newIndices.insert(i, i); + } + + for (int i = 0; i < dracoMesh->num_faces(); ++i) { + // grab the material ID and texture ID for this face, if we have it + auto firstCorner = dracoMesh->face(draco::FaceIndex(i))[0]; + int64_t matTexValue[2] = { 0, 0 }; if (matTexAttribute) { // read material ID and texture ID mappings into materials and texture vectors - auto mappedIndex = matTexAttribute->mapped_index(vertexIndex); + auto mappedIndex = matTexAttribute->mapped_index(firstCorner); matTexAttribute->ConvertValue(mappedIndex, &matTexValue[0]); } - // grab the material ID and texture ID for this face, if we have it QPair materialTexture(matTexValue[0], matTexValue[1]); // grab or setup the FBXMeshPart for the part this face belongs to @@ -430,9 +436,9 @@ ExtractedMesh FBXReader::extractMesh(const FBXNode& object, unsigned int& meshIn // give the mesh part this index FBXMeshPart& part = data.extracted.mesh.parts[partIndexPlusOne - 1]; - part.triangleIndices.append(i); - - data.extracted.newIndices.insert(i, i); + part.triangleIndices.append(firstCorner.value()); + part.triangleIndices.append(dracoMesh->face(draco::FaceIndex(i))[1].value()); + part.triangleIndices.append(dracoMesh->face(draco::FaceIndex(i))[2].value()); } } } From c9024f5e8754c0e22bef7ab2c527f7857815d946 Mon Sep 17 00:00:00 2001 From: Ryan Huffman Date: Tue, 12 Sep 2017 19:51:49 -0700 Subject: [PATCH 088/129] Update FBXBaker to bake triangleIndices and fix faces being lost --- libraries/baking/src/FBXBaker.cpp | 23 +++++++++++++---------- 1 file changed, 13 insertions(+), 10 deletions(-) diff --git a/libraries/baking/src/FBXBaker.cpp b/libraries/baking/src/FBXBaker.cpp index 18e7dbbd76..3407579e9e 100644 --- a/libraries/baking/src/FBXBaker.cpp +++ b/libraries/baking/src/FBXBaker.cpp @@ -354,19 +354,15 @@ void FBXBaker::rewriteAndBakeSceneModels() { auto partIndex = 0; + draco::FaceIndex face; for (auto& part : mesh.parts) { - //Q_ASSERT(part.quadTrianglesIndices % 3 == 0); - //Q_ASSERT(part.triangleIndices % 3 == 0); - const auto matTex = extractedMesh.partMaterialTextures[partIndex]; const int64_t matTexData[2] = { matTex.first, matTex.second }; - for (int i = 0; (i + 2) < part.quadTrianglesIndices.size(); i += 3) { - auto idx0 = part.quadTrianglesIndices[i]; - auto idx1 = part.quadTrianglesIndices[i + 1]; - auto idx2 = part.quadTrianglesIndices[i + 2]; - - auto face = draco::FaceIndex(i / 3); + auto addFace = [&](QVector& indices, int index, draco::FaceIndex face) { + auto idx0 = indices[index]; + auto idx1 = indices[index + 1]; + auto idx2 = indices[index + 2]; if (hasPerFaceMaterials) { meshBuilder.SetPerFaceAttributeValueForFace(faceMaterialAttributeID, face, &matTexData); @@ -378,7 +374,7 @@ void FBXBaker::rewriteAndBakeSceneModels() { if (hasNormals) { meshBuilder.SetAttributeValuesForFace(normalsAttributeID, face, - &mesh.normals[idx0],&mesh.normals[idx1], + &mesh.normals[idx0], &mesh.normals[idx1], &mesh.normals[idx2]); } if (hasColors) { @@ -396,6 +392,13 @@ void FBXBaker::rewriteAndBakeSceneModels() { &mesh.texCoords1[idx0], &mesh.texCoords1[idx1], &mesh.texCoords1[idx2]); } + }; + + for (int i = 0; (i + 2) < part.quadTrianglesIndices.size(); i += 3) { + addFace(part.quadTrianglesIndices, i, face++); + } + for (int i = 0; (i + 2) < part.triangleIndices.size(); i += 3) { + addFace(part.triangleIndices, i, face++); } partIndex++; From 88b8fb4c5ecce95f298737ff731013e75667870e Mon Sep 17 00:00:00 2001 From: Stephen Birarda Date: Tue, 12 Sep 2017 20:02:27 -0700 Subject: [PATCH 089/129] use general worker threads for fbx baking --- libraries/baking/src/FBXBaker.cpp | 1 + tools/oven/src/BakerCLI.cpp | 4 ++-- tools/oven/src/DomainBaker.cpp | 2 +- tools/oven/src/Oven.cpp | 26 +------------------------- tools/oven/src/Oven.h | 2 -- tools/oven/src/ui/ModelBakeWidget.cpp | 2 +- 6 files changed, 6 insertions(+), 31 deletions(-) diff --git a/libraries/baking/src/FBXBaker.cpp b/libraries/baking/src/FBXBaker.cpp index 3407579e9e..52c63c26d0 100644 --- a/libraries/baking/src/FBXBaker.cpp +++ b/libraries/baking/src/FBXBaker.cpp @@ -397,6 +397,7 @@ void FBXBaker::rewriteAndBakeSceneModels() { for (int i = 0; (i + 2) < part.quadTrianglesIndices.size(); i += 3) { addFace(part.quadTrianglesIndices, i, face++); } + for (int i = 0; (i + 2) < part.triangleIndices.size(); i += 3) { addFace(part.triangleIndices, i, face++); } diff --git a/tools/oven/src/BakerCLI.cpp b/tools/oven/src/BakerCLI.cpp index 7b65f8bbf0..5ab995be95 100644 --- a/tools/oven/src/BakerCLI.cpp +++ b/tools/oven/src/BakerCLI.cpp @@ -42,7 +42,7 @@ void BakerCLI::bakeFile(QUrl inputUrl, const QString outputPath) { // create our appropiate baker if (isFBX) { _baker = std::unique_ptr { new FBXBaker(inputUrl, []() -> QThread* { return qApp->getNextWorkerThread(); }, outputPath) }; - _baker->moveToThread(qApp->getFBXBakerThread()); + _baker->moveToThread(qApp->getNextWorkerThread()); } else if (isSupportedImage) { _baker = std::unique_ptr { new TextureBaker(inputUrl, image::TextureUsage::CUBE_TEXTURE, outputPath) }; _baker->moveToThread(qApp->getNextWorkerThread()); @@ -61,4 +61,4 @@ void BakerCLI::bakeFile(QUrl inputUrl, const QString outputPath) { void BakerCLI::handleFinishedBaker() { qCDebug(model_baking) << "Finished baking file."; QApplication::exit(_baker.get()->hasErrors()); -} \ No newline at end of file +} diff --git a/tools/oven/src/DomainBaker.cpp b/tools/oven/src/DomainBaker.cpp index 5dd7c20b2e..535d9a49a9 100644 --- a/tools/oven/src/DomainBaker.cpp +++ b/tools/oven/src/DomainBaker.cpp @@ -214,7 +214,7 @@ void DomainBaker::enumerateEntities() { // move the baker to the baker thread // and kickoff the bake - baker->moveToThread(qApp->getFBXBakerThread()); + baker->moveToThread(qApp->getNextWorkerThread()); QMetaObject::invokeMethod(baker.data(), "bake"); // keep track of the total number of baking entities diff --git a/tools/oven/src/Oven.cpp b/tools/oven/src/Oven.cpp index c9d3b4e5f0..d91206a592 100644 --- a/tools/oven/src/Oven.cpp +++ b/tools/oven/src/Oven.cpp @@ -51,11 +51,7 @@ Oven::Oven(int argc, char* argv[]) : image::setCubeTexturesCompressionEnabled(true); // setup our worker threads - setupWorkerThreads(QThread::idealThreadCount() - 1); - - // Autodesk's SDK means that we need a single thread for all FBX importing/exporting in the same process - // setup the FBX Baker thread - setupFBXBakerThread(); + setupWorkerThreads(QThread::idealThreadCount()); // check if we were passed any command line arguments that would tell us just to run without the GUI if (parser.isSet(CLI_INPUT_PARAMETER) || parser.isSet(CLI_OUTPUT_PARAMETER)) { @@ -81,10 +77,6 @@ Oven::~Oven() { _workerThreads[i]->quit(); _workerThreads[i]->wait(); } - - // cleanup the FBX Baker thread - _fbxBakerThread->quit(); - _fbxBakerThread->wait(); } void Oven::setupWorkerThreads(int numWorkerThreads) { @@ -97,22 +89,6 @@ void Oven::setupWorkerThreads(int numWorkerThreads) { } } -void Oven::setupFBXBakerThread() { - // we're being asked for the FBX baker thread, but we don't have one yet - // so set that up now - _fbxBakerThread = new QThread(this); - _fbxBakerThread->setObjectName("Oven FBX Baker Thread"); -} - -QThread* Oven::getFBXBakerThread() { - if (!_fbxBakerThread->isRunning()) { - // start the FBX baker thread if it isn't running yet - _fbxBakerThread->start(); - } - - return _fbxBakerThread; -} - QThread* Oven::getNextWorkerThread() { // Here we replicate some of the functionality of QThreadPool by giving callers an available worker thread to use. // We can't use QThreadPool because we want to put QObjects with signals/slots on these threads. diff --git a/tools/oven/src/Oven.h b/tools/oven/src/Oven.h index 569b73a3e2..928ba4eb11 100644 --- a/tools/oven/src/Oven.h +++ b/tools/oven/src/Oven.h @@ -34,7 +34,6 @@ public: OvenMainWindow* getMainWindow() const { return _mainWindow; } - QThread* getFBXBakerThread(); QThread* getNextWorkerThread(); private: @@ -42,7 +41,6 @@ private: void setupFBXBakerThread(); OvenMainWindow* _mainWindow; - QThread* _fbxBakerThread; QList _workerThreads; std::atomic _nextWorkerThreadIndex; diff --git a/tools/oven/src/ui/ModelBakeWidget.cpp b/tools/oven/src/ui/ModelBakeWidget.cpp index 926de8bae0..7829993740 100644 --- a/tools/oven/src/ui/ModelBakeWidget.cpp +++ b/tools/oven/src/ui/ModelBakeWidget.cpp @@ -201,7 +201,7 @@ void ModelBakeWidget::bakeButtonClicked() { }; // move the baker to the FBX baker thread - baker->moveToThread(qApp->getFBXBakerThread()); + baker->moveToThread(qApp->getNextWorkerThread()); // invoke the bake method on the baker thread QMetaObject::invokeMethod(baker.get(), "bake"); From c445d914d7d4704edd7bcbad2716b7144bdbbfce Mon Sep 17 00:00:00 2001 From: Stephen Birarda Date: Tue, 12 Sep 2017 21:59:46 -0700 Subject: [PATCH 090/129] make sure material IDs are de-duplicated --- libraries/baking/src/FBXBaker.cpp | 6 +++--- libraries/fbx/src/FBXReader_Mesh.cpp | 6 +++--- 2 files changed, 6 insertions(+), 6 deletions(-) diff --git a/libraries/baking/src/FBXBaker.cpp b/libraries/baking/src/FBXBaker.cpp index 52c63c26d0..271a25e9dd 100644 --- a/libraries/baking/src/FBXBaker.cpp +++ b/libraries/baking/src/FBXBaker.cpp @@ -349,7 +349,7 @@ void FBXBaker::rewriteAndBakeSceneModels() { if (hasPerFaceMaterials) { faceMaterialAttributeID = meshBuilder.AddAttribute( (draco::GeometryAttribute::Type)DRACO_ATTRIBUTE_MATERIAL_ID, - 2, draco::DT_INT64); + 1, draco::DT_UINT16); } @@ -357,7 +357,6 @@ void FBXBaker::rewriteAndBakeSceneModels() { draco::FaceIndex face; for (auto& part : mesh.parts) { const auto matTex = extractedMesh.partMaterialTextures[partIndex]; - const int64_t matTexData[2] = { matTex.first, matTex.second }; auto addFace = [&](QVector& indices, int index, draco::FaceIndex face) { auto idx0 = indices[index]; @@ -365,7 +364,8 @@ void FBXBaker::rewriteAndBakeSceneModels() { auto idx2 = indices[index + 2]; if (hasPerFaceMaterials) { - meshBuilder.SetPerFaceAttributeValueForFace(faceMaterialAttributeID, face, &matTexData); + uint16_t materialID = matTex.first; + meshBuilder.SetPerFaceAttributeValueForFace(faceMaterialAttributeID, face, &materialID); } meshBuilder.SetAttributeValuesForFace(positionAttributeID, face, diff --git a/libraries/fbx/src/FBXReader_Mesh.cpp b/libraries/fbx/src/FBXReader_Mesh.cpp index f1524b1a81..323ecac37b 100644 --- a/libraries/fbx/src/FBXReader_Mesh.cpp +++ b/libraries/fbx/src/FBXReader_Mesh.cpp @@ -415,16 +415,16 @@ ExtractedMesh FBXReader::extractMesh(const FBXNode& object, unsigned int& meshIn // grab the material ID and texture ID for this face, if we have it auto firstCorner = dracoMesh->face(draco::FaceIndex(i))[0]; - int64_t matTexValue[2] = { 0, 0 }; + uint16_t materialID { 0 }; if (matTexAttribute) { // read material ID and texture ID mappings into materials and texture vectors auto mappedIndex = matTexAttribute->mapped_index(firstCorner); - matTexAttribute->ConvertValue(mappedIndex, &matTexValue[0]); + matTexAttribute->ConvertValue(mappedIndex, &materialID); } - QPair materialTexture(matTexValue[0], matTexValue[1]); + QPair materialTexture(materialID, 0); // grab or setup the FBXMeshPart for the part this face belongs to int& partIndexPlusOne = materialTextureParts[materialTexture]; From 087a95a6259c122dc0a8c543c26e0f13a54db3af Mon Sep 17 00:00:00 2001 From: Ryan Huffman Date: Tue, 12 Sep 2017 21:32:37 -0700 Subject: [PATCH 091/129] Add explicit draco encoder options to FBXBaker --- libraries/baking/src/FBXBaker.cpp | 9 +++++++++ 1 file changed, 9 insertions(+) diff --git a/libraries/baking/src/FBXBaker.cpp b/libraries/baking/src/FBXBaker.cpp index 271a25e9dd..1abd411e29 100644 --- a/libraries/baking/src/FBXBaker.cpp +++ b/libraries/baking/src/FBXBaker.cpp @@ -423,6 +423,15 @@ void FBXBaker::rewriteAndBakeSceneModels() { } draco::Encoder encoder; + + encoder.SetAttributeQuantization(draco::GeometryAttribute::POSITION, + 14); + encoder.SetAttributeQuantization(draco::GeometryAttribute::TEX_COORD, + 12); + encoder.SetAttributeQuantization(draco::GeometryAttribute::NORMAL, + 10); + encoder.SetSpeedOptions(0, 0); + draco::EncoderBuffer buffer; encoder.EncodeMeshToBuffer(*dracoMesh, &buffer); From 4efd9b845379f2a1cf19a4eebb45505a40cbfa69 Mon Sep 17 00:00:00 2001 From: Stephen Birarda Date: Tue, 12 Sep 2017 23:45:32 -0700 Subject: [PATCH 092/129] set explicit draco encoding options --- libraries/baking/src/FBXBaker.cpp | 11 ++++------- 1 file changed, 4 insertions(+), 7 deletions(-) diff --git a/libraries/baking/src/FBXBaker.cpp b/libraries/baking/src/FBXBaker.cpp index 1abd411e29..29e5dc6a81 100644 --- a/libraries/baking/src/FBXBaker.cpp +++ b/libraries/baking/src/FBXBaker.cpp @@ -424,13 +424,10 @@ void FBXBaker::rewriteAndBakeSceneModels() { draco::Encoder encoder; - encoder.SetAttributeQuantization(draco::GeometryAttribute::POSITION, - 14); - encoder.SetAttributeQuantization(draco::GeometryAttribute::TEX_COORD, - 12); - encoder.SetAttributeQuantization(draco::GeometryAttribute::NORMAL, - 10); - encoder.SetSpeedOptions(0, 0); + encoder.SetAttributeQuantization(draco::GeometryAttribute::POSITION, 14); + encoder.SetAttributeQuantization(draco::GeometryAttribute::TEX_COORD, 12); + encoder.SetAttributeQuantization(draco::GeometryAttribute::NORMAL, 10); + encoder.SetSpeedOptions(0, 5); draco::EncoderBuffer buffer; encoder.EncodeMeshToBuffer(*dracoMesh, &buffer); From 8a9e4029ae935e08cc8c6ce50400c51cf7ff3fee Mon Sep 17 00:00:00 2001 From: Stephen Birarda Date: Wed, 13 Sep 2017 00:23:52 -0700 Subject: [PATCH 093/129] use explicit conversion from size_t to int --- libraries/baking/src/FBXBaker.cpp | 8 ++++---- 1 file changed, 4 insertions(+), 4 deletions(-) diff --git a/libraries/baking/src/FBXBaker.cpp b/libraries/baking/src/FBXBaker.cpp index 29e5dc6a81..79a5b9157b 100644 --- a/libraries/baking/src/FBXBaker.cpp +++ b/libraries/baking/src/FBXBaker.cpp @@ -434,7 +434,7 @@ void FBXBaker::rewriteAndBakeSceneModels() { FBXNode dracoMeshNode; dracoMeshNode.name = "DracoMesh"; - auto value = QVariant::fromValue(QByteArray(buffer.data(), buffer.size())); + auto value = QVariant::fromValue(QByteArray(buffer.data(), (int) buffer.size())); dracoMeshNode.properties.append(value); @@ -653,13 +653,13 @@ void FBXBaker::handleBakedTexture() { } else { // there was an error baking this texture - add it to our list of errors _errorList.append(bakedTexture->getErrors()); - + // we don't emit finished yet so that the other textures can finish baking first _pendingErrorEmission = true; - + // now that this texture has been baked, even though it failed, we can remove that TextureBaker from our list _bakingTextures.remove(bakedTexture->getTextureURL()); - + checkIfTexturesFinished(); } } else { From dc86c6fe73584fd7165194f37c619d1773d593a4 Mon Sep 17 00:00:00 2001 From: Stephen Birarda Date: Wed, 13 Sep 2017 08:37:36 -0700 Subject: [PATCH 094/129] fixes for warnings in FBXReader/FBXWriter --- libraries/baking/CMakeLists.txt | 2 +- libraries/fbx/CMakeLists.txt | 2 +- libraries/fbx/src/FBXReader.cpp | 6 +++--- libraries/fbx/src/FBXWriter.cpp | 12 ++++++------ 4 files changed, 11 insertions(+), 11 deletions(-) diff --git a/libraries/baking/CMakeLists.txt b/libraries/baking/CMakeLists.txt index 66cf791776..2304a5e0c0 100644 --- a/libraries/baking/CMakeLists.txt +++ b/libraries/baking/CMakeLists.txt @@ -6,5 +6,5 @@ include_hifi_library_headers(gpu) add_dependency_external_projects(draco) find_package(Draco REQUIRED) -target_include_directories(${TARGET_NAME} SYSTEM PRIVATE ${DRACO_INCLUDE_DIRS}) +include_directories(SYSTEM ${DRACO_INCLUDE_DIRS}) target_link_libraries(${TARGET_NAME} ${DRACO_LIBRARY} ${DRACO_ENCODER_LIBRARY}) diff --git a/libraries/fbx/CMakeLists.txt b/libraries/fbx/CMakeLists.txt index 7cead5aa4f..4c81f10302 100644 --- a/libraries/fbx/CMakeLists.txt +++ b/libraries/fbx/CMakeLists.txt @@ -6,5 +6,5 @@ include_hifi_library_headers(gpu image) add_dependency_external_projects(draco) find_package(Draco REQUIRED) -target_include_directories(${TARGET_NAME} SYSTEM PRIVATE ${DRACO_INCLUDE_DIRS}) +include_directories(SYSTEM ${DRACO_INCLUDE_DIRS}) target_link_libraries(${TARGET_NAME} ${DRACO_LIBRARY} ${DRACO_ENCODER_LIBRARY}) diff --git a/libraries/fbx/src/FBXReader.cpp b/libraries/fbx/src/FBXReader.cpp index d4a58a1126..e4fea00a34 100644 --- a/libraries/fbx/src/FBXReader.cpp +++ b/libraries/fbx/src/FBXReader.cpp @@ -169,7 +169,7 @@ QString getID(const QVariantList& properties, int index = 0) { } /// The names of the joints in the Maya HumanIK rig -static const std::array HUMANIK_JOINTS = { +static const std::array HUMANIK_JOINTS = {{ "RightHand", "RightForeArm", "RightArm", @@ -186,7 +186,7 @@ static const std::array HUMANIK_JOINTS = { "LeftLeg", "RightFoot", "LeftFoot" -}; +}}; class FBXModel { public: @@ -512,7 +512,7 @@ FBXGeometry* FBXReader::extractFBXGeometry(const QVariantHash& mapping, const QS QVector humanIKJointNames; - for (int i = 0; i < HUMANIK_JOINTS.size(); i++) { + for (int i = 0; i < (int) HUMANIK_JOINTS.size(); i++) { QByteArray jointName = HUMANIK_JOINTS[i]; humanIKJointNames.append(processID(getString(joints.value(jointName, jointName)))); } diff --git a/libraries/fbx/src/FBXWriter.cpp b/libraries/fbx/src/FBXWriter.cpp index c14e383402..cc34696f92 100644 --- a/libraries/fbx/src/FBXWriter.cpp +++ b/libraries/fbx/src/FBXWriter.cpp @@ -143,7 +143,7 @@ void FBXWriter::encodeFBXProperty(QDataStream& out, const QVariant& prop) { break; } - // TODO Delete? Do we ever use QList instead of QVector? + // TODO Delete? Do we ever use QList instead of QVector? case QVariant::Type::List: { auto list = prop.toList(); @@ -156,7 +156,7 @@ void FBXWriter::encodeFBXProperty(QDataStream& out, const QVariant& prop) { out << (int32_t)0; out << (int32_t)0; for (auto& innerProp : list) { - out << prop.toFloat(); + out << innerProp.toFloat(); } break; @@ -166,7 +166,7 @@ void FBXWriter::encodeFBXProperty(QDataStream& out, const QVariant& prop) { out << (int32_t)0; out << (int32_t)0; for (auto& innerProp : list) { - out << prop.toDouble(); + out << innerProp.toDouble(); } break; @@ -176,7 +176,7 @@ void FBXWriter::encodeFBXProperty(QDataStream& out, const QVariant& prop) { out << (int32_t)0; out << (int32_t)0; for (auto& innerProp : list) { - out << prop.toLongLong(); + out << innerProp.toLongLong(); } break; @@ -186,7 +186,7 @@ void FBXWriter::encodeFBXProperty(QDataStream& out, const QVariant& prop) { out << (int32_t)0; out << (int32_t)0; for (auto& innerProp : list) { - out << prop.toInt(); + out << innerProp.toInt(); } break; @@ -196,7 +196,7 @@ void FBXWriter::encodeFBXProperty(QDataStream& out, const QVariant& prop) { out << (int32_t)0; out << (int32_t)0; for (auto& innerProp : list) { - out << prop.toBool(); + out << innerProp.toBool(); } break; } From 48e58febc6402651f0bcc36b03cb9fae684aac49 Mon Sep 17 00:00:00 2001 From: Atlante45 Date: Tue, 12 Sep 2017 12:56:23 -0700 Subject: [PATCH 095/129] Remove deprecated AutoBaker --- assignment-client/src/assets/AssetServer.h | 1 - assignment-client/src/assets/AutoBaker.cpp | 37 ---------------------- assignment-client/src/assets/AutoBaker.h | 32 ------------------- 3 files changed, 70 deletions(-) delete mode 100644 assignment-client/src/assets/AutoBaker.cpp delete mode 100644 assignment-client/src/assets/AutoBaker.h diff --git a/assignment-client/src/assets/AssetServer.h b/assignment-client/src/assets/AssetServer.h index 907726e1ab..4a36825764 100644 --- a/assignment-client/src/assets/AssetServer.h +++ b/assignment-client/src/assets/AssetServer.h @@ -19,7 +19,6 @@ #include #include "AssetUtils.h" -#include "AutoBaker.h" #include "ReceivedMessage.h" diff --git a/assignment-client/src/assets/AutoBaker.cpp b/assignment-client/src/assets/AutoBaker.cpp deleted file mode 100644 index 38c564425b..0000000000 --- a/assignment-client/src/assets/AutoBaker.cpp +++ /dev/null @@ -1,37 +0,0 @@ -// -// AutoBaker.cpp -// assignment-client/src/assets -// -// Created by Clement Brisset on 8/9/17 -// Copyright 2015 High Fidelity, Inc. -// -// Distributed under the Apache License, Version 2.0. -// See the accompanying file LICENSE or http://www.apache.org/licenses/LICENSE-2.0.html -// - -#include "AutoBaker.h" -#include - -using namespace alg; - -void AutoBaker::addPendingBake(AssetHash hash) { - _pendingBakes.push_back(hash); - - // Maybe start baking it right away -} - -bool AutoBaker::assetNeedsBaking(AssetPath path, AssetHash hash) { - return path.endsWith(".fbx"); -} - -BakingStatus AutoBaker::getAssetStatus(AssetHash hash) { - if (find(_pendingBakes, hash) != std::end(_pendingBakes)) { - return Pending; - } - - if (find(_currentlyBaking, hash) != std::end(_currentlyBaking)) { - return Baking; - } - - return NotBaked; -} diff --git a/assignment-client/src/assets/AutoBaker.h b/assignment-client/src/assets/AutoBaker.h deleted file mode 100644 index 258fb15931..0000000000 --- a/assignment-client/src/assets/AutoBaker.h +++ /dev/null @@ -1,32 +0,0 @@ -// -// AutoBaker.h -// assignment-client/src/assets -// -// Created by Clement Brisset on 8/9/17 -// Copyright 2015 High Fidelity, Inc. -// -// Distributed under the Apache License, Version 2.0. -// See the accompanying file LICENSE or http://www.apache.org/licenses/LICENSE-2.0.html -// - -#ifndef hifi_AutoBaker_h -#define hifi_AutoBaker_h - -#include - -#include "AssetUtils.h" - -class AutoBaker { -public: - void addPendingBake(AssetHash hash); - - bool assetNeedsBaking(AssetPath path, AssetHash hash); - - BakingStatus getAssetStatus(AssetHash hash); - -private: - std::vector _pendingBakes; - std::vector _currentlyBaking; -}; - -#endif /* hifi_AutoBaker_h */ From 7d9daea0f6346a6274fb3654312aea1249b59228 Mon Sep 17 00:00:00 2001 From: Atlante45 Date: Tue, 12 Sep 2017 13:43:00 -0700 Subject: [PATCH 096/129] Transmit baking errors to interface for display. --- assignment-client/src/assets/AssetServer.cpp | 56 ++++++++++++------- assignment-client/src/assets/AssetServer.h | 7 ++- .../AssetMappingsScriptingInterface.cpp | 1 + libraries/networking/src/AssetUtils.h | 1 + libraries/networking/src/MappingRequest.cpp | 6 +- 5 files changed, 46 insertions(+), 25 deletions(-) diff --git a/assignment-client/src/assets/AssetServer.cpp b/assignment-client/src/assets/AssetServer.cpp index cee4c77170..6da299ea87 100644 --- a/assignment-client/src/assets/AssetServer.cpp +++ b/assignment-client/src/assets/AssetServer.cpp @@ -83,7 +83,8 @@ void BakeAssetTask::run() { if (baker->hasErrors()) { qDebug() << "Failed to bake: " << _assetHash << _assetPath << baker->getErrors(); - emit bakeFailed(_assetHash, _assetPath); + auto errors = baker->getErrors().join('\n'); // Join error list into a single string for convenience + emit bakeFailed(_assetHash, _assetPath, errors); } else { auto vectorOutputFiles = QVector::fromStdVector(baker->getOutputFiles()); qDebug() << "Finished baking: " << _assetHash << _assetPath << vectorOutputFiles; @@ -112,19 +113,19 @@ QString AssetServer::getPathToAssetHash(const AssetHash& assetHash) { return _filesDirectory.absoluteFilePath(assetHash); } -BakingStatus AssetServer::getAssetStatus(const AssetPath& path, const AssetHash& hash) { +std::pair AssetServer::getAssetStatus(const AssetPath& path, const AssetHash& hash) { auto it = _pendingBakes.find(hash); if (it != _pendingBakes.end()) { - return (*it)->isBaking() ? Baking : Pending; + return { (*it)->isBaking() ? Baking : Pending, "" }; } if (path.startsWith(HIDDEN_BAKED_CONTENT_FOLDER)) { - return Baked; + return { Baked, "" }; } auto dotIndex = path.lastIndexOf("."); if (dotIndex == -1) { - return Irrelevant; + return { Irrelevant, "" }; } auto extension = path.mid(dotIndex + 1); @@ -136,28 +137,28 @@ BakingStatus AssetServer::getAssetStatus(const AssetPath& path, const AssetHash& } else if (BAKEABLE_TEXTURE_EXTENSIONS.contains(extension.toLocal8Bit()) && hasMetaFile(hash)) { bakedFilename = BAKED_TEXTURE_SIMPLE_NAME; } else { - return Irrelevant; + return { Irrelevant, "" }; } auto bakedPath = HIDDEN_BAKED_CONTENT_FOLDER + hash + "/" + bakedFilename; auto jt = _fileMappings.find(bakedPath); if (jt != _fileMappings.end()) { if (jt->second == hash) { - bool loaded; - AssetMeta meta; - - std::tie(loaded, meta) = readMetaFile(hash); - if (loaded && meta.failedLastBake) { - return Error; - } - - return NotBaked; + return { NotBaked, "" }; } else { - return Baked; + return { Baked, "" }; + } + } else { + bool loaded; + AssetMeta meta; + + std::tie(loaded, meta) = readMetaFile(hash); + if (loaded && meta.failedLastBake) { + return { Error, meta.lastBakeErrors }; } } - return Pending; + return { Pending, "" }; } void AssetServer::bakeAssets() { @@ -564,7 +565,14 @@ void AssetServer::handleGetAllMappingOperation(ReceivedMessage& message, SharedN auto hash = it->second; replyPacket.writeString(mapping); replyPacket.write(QByteArray::fromHex(hash.toUtf8())); - replyPacket.writePrimitive(getAssetStatus(mapping, hash)); + + BakingStatus status; + QString lastBakeErrors; + std::tie(status, lastBakeErrors) = getAssetStatus(mapping, hash); + replyPacket.writePrimitive(status); + if (status == Error) { + replyPacket.writeString(lastBakeErrors); + } } } @@ -1146,8 +1154,8 @@ QString getBakeMapping(const AssetHash& hash, const QString& relativeFilePath) { return HIDDEN_BAKED_CONTENT_FOLDER + hash + "/" + relativeFilePath; } -void AssetServer::handleFailedBake(QString originalAssetHash, QString assetPath) { - qDebug() << "Failed: " << originalAssetHash << assetPath; +void AssetServer::handleFailedBake(QString originalAssetHash, QString assetPath, QString errors) { + qDebug() << "Failed: " << originalAssetHash << assetPath << errors; bool loaded; AssetMeta meta; @@ -1155,6 +1163,7 @@ void AssetServer::handleFailedBake(QString originalAssetHash, QString assetPath) std::tie(loaded, meta) = readMetaFile(originalAssetHash); meta.failedLastBake = true; + meta.lastBakeErrors = errors; writeMetaFile(originalAssetHash, meta); } @@ -1239,6 +1248,7 @@ void AssetServer::handleCompletedBake(QString originalAssetHash, QString origina static const QString BAKE_VERSION_KEY = "bake_version"; static const QString APP_VERSION_KEY = "app_version"; static const QString FAILED_LAST_BAKE_KEY = "failed_last_bake"; +static const QString LAST_BAKE_ERRORS_KEY = "last_bake_errors"; std::pair AssetServer::readMetaFile(AssetHash hash) { auto metaFilePath = HIDDEN_BAKED_CONTENT_FOLDER + hash + "/" + "meta.json"; @@ -1265,15 +1275,18 @@ std::pair AssetServer::readMetaFile(AssetHash hash) { auto bakeVersion = root[BAKE_VERSION_KEY].toInt(-1); auto appVersion = root[APP_VERSION_KEY].toInt(-1); auto failedLastBake = root[FAILED_LAST_BAKE_KEY]; + auto lastBakeErrors = root[LAST_BAKE_ERRORS_KEY]; if (bakeVersion != -1 && appVersion != -1 - && failedLastBake.isBool()) { + && failedLastBake.isBool() + && lastBakeErrors.isString()) { AssetMeta meta; meta.bakeVersion = bakeVersion; meta.applicationVersion = appVersion; meta.failedLastBake = failedLastBake.toBool(); + meta.lastBakeErrors = failedLastBake.toString(); return { true, meta }; } else { @@ -1292,6 +1305,7 @@ bool AssetServer::writeMetaFile(AssetHash originalAssetHash, const AssetMeta& me metaFileObject[BAKE_VERSION_KEY] = meta.bakeVersion; metaFileObject[APP_VERSION_KEY] = meta.applicationVersion; metaFileObject[FAILED_LAST_BAKE_KEY] = meta.failedLastBake; + metaFileObject[LAST_BAKE_ERRORS_KEY] = meta.lastBakeErrors; QJsonDocument metaFileDoc; metaFileDoc.setObject(metaFileObject); diff --git a/assignment-client/src/assets/AssetServer.h b/assignment-client/src/assets/AssetServer.h index 4a36825764..8641f5498f 100644 --- a/assignment-client/src/assets/AssetServer.h +++ b/assignment-client/src/assets/AssetServer.h @@ -40,7 +40,7 @@ public: signals: void bakeComplete(QString assetHash, QString assetPath, QVector outputFiles); - void bakeFailed(QString assetHash, QString assetPath); + void bakeFailed(QString assetHash, QString assetPath, QString errors); private: std::atomic _isBaking { false }; @@ -56,6 +56,7 @@ struct AssetMeta { int bakeVersion { 0 }; int applicationVersion { 0 }; bool failedLastBake { false }; + QString lastBakeErrors; }; class AssetServer : public ThreadedAssignment { @@ -108,7 +109,7 @@ private: QString getPathToAssetHash(const AssetHash& assetHash); - BakingStatus getAssetStatus(const AssetPath& path, const AssetHash& hash); + std::pair getAssetStatus(const AssetPath& path, const AssetHash& hash); void bakeAssets(); void maybeBake(const AssetPath& path, const AssetHash& hash); @@ -119,7 +120,7 @@ private: /// Move baked content for asset to baked directory and update baked status void handleCompletedBake(QString originalAssetHash, QString assetPath, QVector bakedFilePaths); - void handleFailedBake(QString originalAssetHash, QString assetPath); + void handleFailedBake(QString originalAssetHash, QString assetPath, QString errors); /// Create meta file to describe baked content for original asset std::pair readMetaFile(AssetHash hash); diff --git a/interface/src/scripting/AssetMappingsScriptingInterface.cpp b/interface/src/scripting/AssetMappingsScriptingInterface.cpp index 70deabd412..14e0ef5b28 100644 --- a/interface/src/scripting/AssetMappingsScriptingInterface.cpp +++ b/interface/src/scripting/AssetMappingsScriptingInterface.cpp @@ -261,6 +261,7 @@ void AssetMappingModel::refresh() { // update status auto statusString = isFolder ? "--" : bakingStatusToString(mapping.second.status); lastItem->setData(statusString, Qt::UserRole + 5); + lastItem->setData(mapping.second.bakingErrors, Qt::UserRole + 6); } Q_ASSERT(fullPath == path); diff --git a/libraries/networking/src/AssetUtils.h b/libraries/networking/src/AssetUtils.h index c09c725f22..a7c053c3d6 100644 --- a/libraries/networking/src/AssetUtils.h +++ b/libraries/networking/src/AssetUtils.h @@ -66,6 +66,7 @@ enum BakingStatus { struct MappingInfo { AssetHash hash; BakingStatus status; + QString bakingErrors; }; using AssetMapping = std::map; diff --git a/libraries/networking/src/MappingRequest.cpp b/libraries/networking/src/MappingRequest.cpp index 588d8ecc88..a79105e3ab 100644 --- a/libraries/networking/src/MappingRequest.cpp +++ b/libraries/networking/src/MappingRequest.cpp @@ -137,8 +137,12 @@ void GetAllMappingsRequest::doStart() { auto path = message->readString(); auto hash = message->read(SHA256_HASH_LENGTH).toHex(); BakingStatus status; + QString lastBakeErrors; message->readPrimitive(&status); - _mappings[path] = { hash, status }; + if (status == BakingStatus::Error) { + lastBakeErrors = message->readString(); + } + _mappings[path] = { hash, status, lastBakeErrors }; } } emit finished(this); From 944a4fd49d6737b2140046c0c1bac49f8a1d79b4 Mon Sep 17 00:00:00 2001 From: Atlante45 Date: Tue, 12 Sep 2017 13:47:52 -0700 Subject: [PATCH 097/129] Update hifi glyphs + font reference --- interface/resources/fonts/hifi-glyphs.ttf | Bin 29160 -> 29268 bytes .../fonts/hifi-glyphs/icons-reference.html | 1056 +++++++++++++++++ .../resources/fonts/hifi-glyphs/styles.css | 421 +++++++ 3 files changed, 1477 insertions(+) create mode 100644 interface/resources/fonts/hifi-glyphs/icons-reference.html create mode 100644 interface/resources/fonts/hifi-glyphs/styles.css diff --git a/interface/resources/fonts/hifi-glyphs.ttf b/interface/resources/fonts/hifi-glyphs.ttf index 8733349227646b207ec15315e9eab9ee7ae9e252..558562cec52168a9025db4b7c4dd20280a5a5d74 100644 GIT binary patch delta 641 zcmYL{Ur1AN6vsd3?zg+u)^xe1g4P6aQOw)|b&v*O3+&H?LL#blnakm}Tu#tl{Pz&_ zCfzsbAqslz5BJ3fjp!{9>LFjENUDdDf`odIh{e4QI)}sgoR{=pXKcrnMHs- z%;91vmAY_tYWwyeO!NXRrv^@@c!CZSO+X}l{zxLdawZ3I5g5*vjPjdnm0zH(fHpL0 zRLa#VE|{nXyraci*Zcpzyaaj(MwatAW5oZieF}IFbm#NASwnTn21p2K&zB~rTYJM# zLG1Du$Fs)Z{iBb-?FZ3I#&nsFauviD7`b7TazA@s48zzOh*ci1OfEh={0WM4;9hjh z3HUbm_hssi{l=$Z$NGn@)t#w6!`#{efga+ca0Q&F0Oqj%buSJ4#UacvWLJK7Q0#$0tGa z`B+=@97$WP+Jz#_vl_bzvtDDJFxzIP&}g<6GM+dIVkFJ^!XXt;#FFOYLdA>}+hoBy zQPf0}8Ec@FQR&^;d@fsvC1PE1K&n+V=j{O4?!cK{%p)jgvctYF*7_d2w)_8&-&*|# DhsuO) delta 504 zcmX|+OK1~u5XOJAFQeNRSQa0tiJ&d0@fENYv!F+w;@W$_t;HANQ)Cnh0N?bQ*`p^8t z)`;puBj4^$E#H1DXQipe<(8zvCz6sA(MFFu*|9(_pm!*VAxENYB + + + + + + Font Reference - HiFi Glyphs + + + + + +
+

HiFi Glyphs

+

This font was created inHigh Fidelity

+

CSS mapping

+
    +
  • +
    + +
    • +
  • +
    + +
    • +
  • +
    + +
    • +
  • +
    + +
    • +
  • +
    + +
    • +
  • +
    + +
    • +
  • +
    + +
    • +
  • +
    + +
    • +
  • +
    + +
    • +
  • +
    + +
    • +
  • +
    + +
    • +
  • +
    + +
    • +
  • +
    + +
    • +
  • +
    + +
    • +
  • +
    + +
    • +
  • +
    + +
    • +
  • +
    + +
    • +
  • +
    + +
    • +
  • +
    + +
    • +
  • +
    + +
    • +
  • +
    + +
    • +
  • +
    + +
    • +
  • +
    + +
    • +
  • +
    + +
    • +
  • +
    + +
    • +
  • +
    + +
    • +
  • +
    + +
    • +
  • +
    + +
    • +
  • +
    + +
    • +
  • +
    + +
    • +
  • +
    + +
    • +
  • +
    + +
    • +
  • +
    + +
    • +
  • +
    + +
    • +
  • +
    + +
    • +
  • +
    + +
    • +
  • +
    + +
    • +
  • +
    + +
    • +
  • +
    + +
    • +
  • +
    + +
    • +
  • +
    + +
    • +
  • +
    + +
    • +
  • +
    + +
    • +
  • +
    + +
    • +
  • +
    + +
    • +
  • +
    + +
    • +
  • +
    + +
    • +
  • +
    + +
    • +
  • +
    + +
    • +
  • +
    + +
    • +
  • +
    + +
    • +
  • +
    + +
    • +
  • +
    + +
    • +
  • +
    + +
    • +
  • +
    + +
    • +
  • +
    + +
    • +
  • +
    + +
    • +
  • +
    + +
    • +
  • +
    + +
    • +
  • +
    + +
    • +
  • +
    + +
    • +
  • +
    + +
    • +
  • +
    + +
    • +
  • +
    + +
    • +
  • +
    + +
    • +
  • +
    + +
    • +
  • +
    + +
    • +
  • +
    + +
    • +
  • +
    + +
    • +
  • +
    + +
    • +
  • +
    + +
    • +
  • +
    + +
    • +
  • +
    + +
    • +
  • +
    + +
    • +
  • +
    + +
    • +
  • +
    + +
    • +
  • +
    + +
    • +
  • +
    + +
    • +
  • +
    + +
    • +
  • +
    + +
    • +
  • +
    + +
    • +
  • +
    + +
    • +
  • +
    + +
    • +
  • +
    + +
    • +
  • +
    + +
    • +
  • +
    + +
    • +
  • +
    + +
    • +
  • +
    + +
    • +
  • +
    + +
    • +
  • +
    + +
    • +
  • +
    + +
    • +
  • +
    + +
    • +
  • +
    + +
    • +
  • +
    + +
    • +
  • +
    + +
    • +
  • +
    + +
    • +
  • +
    + +
    • +
  • +
    + +
    • +
  • +
    + +
    • +
  • +
    + +
    • +
  • +
    + +
    • +
  • +
    + +
    • +
  • +
    + +
    • +
  • +
    + +
    • +
  • +
    + +
    • +
  • +
    + +
    • +
  • +
    + +
    • +
  • +
    + +
    • +
  • +
    + +
    • +
  • +
    + +
    • +
  • +
    + +
    • +
  • +
    + +
    • +
  • +
    + +
    • +
  • +
    + +
    • +
  • +
    + +
    • +
  • +
    + +
    • +
  • +
    + +
    • +
  • +
    + +
    • +
  • +
    + +
    • +
  • +
    + +
    • +
  • +
    + +
    • +
  • +
    + +
    • +
  • +
    + +
    • +
  • +
    + +
    • +
  • +
    + +
    • +
  • +
    + +
    • +
  • +
    + +
    • +
+

Character mapping

+
    +
  • +
    + +
    • +
  • +
    + +
    • +
  • +
    + +
    • +
  • +
    + +
    • +
  • +
    + +
    • +
  • +
    + +
    • +
  • +
    + +
    • +
  • +
    + +
    • +
  • +
    + +
    • +
  • +
    + +
    • +
  • +
    + +
    • +
  • +
    + +
    • +
  • +
    + +
    • +
  • +
    + +
    • +
  • +
    + +
    • +
  • +
    + +
    • +
  • +
    + +
    • +
  • +
    + +
    • +
  • +
    + +
    • +
  • +
    + +
    • +
  • +
    + +
    • +
  • +
    + +
    • +
  • +
    + +
    • +
  • +
    + +
    • +
  • +
    + +
    • +
  • +
    + +
    • +
  • +
    + +
    • +
  • +
    + +
    • +
  • +
    + +
    • +
  • +
    + +
    • +
  • +
    + +
    • +
  • +
    + +
    • +
  • +
    + +
    • +
  • +
    + +
    • +
  • +
    + +
    • +
  • +
    + +
    • +
  • +
    + +
    • +
  • +
    + +
    • +
  • +
    + +
    • +
  • +
    + +
    • +
  • +
    + +
    • +
  • +
    + +
    • +
  • +
    + +
    • +
  • +
    + +
    • +
  • +
    + +
    • +
  • +
    + +
    • +
  • +
    + +
    • +
  • +
    + +
    • +
  • +
    + +
    • +
  • +
    + +
    • +
  • +
    + +
    • +
  • +
    + +
    • +
  • +
    + +
    • +
  • +
    + +
    • +
  • +
    + +
    • +
  • +
    + +
    • +
  • +
    + +
    • +
  • +
    + +
    • +
  • +
    + +
    • +
  • +
    + +
    • +
  • +
    + +
    • +
  • +
    + +
    • +
  • +
    + +
    • +
  • +
    + +
    • +
  • +
    + +
    • +
  • +
    + +
    • +
  • +
    + +
    • +
  • +
    + +
    • +
  • +
    + +
    • +
  • +
    + +
    • +
  • +
    + +
    • +
  • +
    + +
    • +
  • +
    + +
    • +
  • +
    + +
    • +
  • +
    + +
    • +
  • +
    + +
    • +
  • +
    + +
    • +
  • +
    + +
    • +
  • +
    + +
    • +
  • +
    + +
    • +
  • +
    + +
    • +
  • +
    + +
    • +
  • +
    + +
    • +
  • +
    + +
    • +
  • +
    + +
    • +
  • +
    + +
    • +
  • +
    + +
    • +
  • +
    + +
    • +
  • +
    + +
    • +
  • +
    + +
    • +
  • +
    + +
    • +
  • +
    + +
    • +
  • +
    + +
    • +
  • +
    + +
    • +
  • +
    + +
    • +
  • +
    + +
    • +
  • +
    + +
    • +
  • +
    + +
    • +
  • +
    + +
    • +
  • +
    + +
    • +
  • +
    + +
    • +
  • +
    + +
    • +
  • +
    + +
    • +
  • +
    + +
    • +
  • +
    + +
    • +
  • +
    + +
    • +
  • +
    + +
    • +
  • +
    + +
    • +
  • +
    + +
    • +
  • +
    + +
    • +
  • +
    + +
    • +
  • +
    + +
    • +
  • +
    + +
    • +
  • +
    + +
    • +
  • +
    + +
    • +
  • +
    + +
    • +
  • +
    + +
    • +
  • +
    + +
    • +
  • +
    + +
    • +
  • +
    + +
    • +
  • +
    + +
    • +
  • +
    + +
    • +
  • +
    + +
    • +
  • +
    + +
    • +
  • +
    + +
    • +
  • +
    + +
    • +
  • +
    + +
    • +
+
+ + + diff --git a/interface/resources/fonts/hifi-glyphs/styles.css b/interface/resources/fonts/hifi-glyphs/styles.css new file mode 100644 index 0000000000..66277740fc --- /dev/null +++ b/interface/resources/fonts/hifi-glyphs/styles.css @@ -0,0 +1,421 @@ +@charset "UTF-8"; + +@font-face { + font-family: "hifi-glyphs"; + src:url("fonts/hifi-glyphs.eot"); + src:url("fonts/hifi-glyphs.eot?#iefix") format("embedded-opentype"), + url("fonts/hifi-glyphs.woff") format("woff"), + url("fonts/hifi-glyphs.ttf") format("truetype"), + url("fonts/hifi-glyphs.svg#hifi-glyphs") format("svg"); + font-weight: normal; + font-style: normal; + +} + +[data-icon]:before { + font-family: "hifi-glyphs" !important; + content: attr(data-icon); + font-style: normal !important; + font-weight: normal !important; + font-variant: normal !important; + text-transform: none !important; + speak: none; + line-height: 1; + -webkit-font-smoothing: antialiased; + -moz-osx-font-smoothing: grayscale; +} + +[class^="icon-"]:before, +[class*=" icon-"]:before { + font-family: "hifi-glyphs" !important; + font-style: normal !important; + font-weight: normal !important; + font-variant: normal !important; + text-transform: none !important; + speak: none; + line-height: 1; + -webkit-font-smoothing: antialiased; + -moz-osx-font-smoothing: grayscale; +} + +.icon-hmd:before { + content: "\62"; +} +.icon-2d-screen:before { + content: "\63"; +} +.icon-keyboard:before { + content: "\64"; +} +.icon-hand-controllers:before { + content: "\65"; +} +.icon-headphones-mic:before { + content: "\66"; +} +.icon-gamepad:before { + content: "\67"; +} +.icon-headphones:before { + content: "\68"; +} +.icon-mic:before { + content: "\69"; +} +.icon-upload:before { + content: "\6a"; +} +.icon-script:before { + content: "\6b"; +} +.icon-text:before { + content: "\6c"; +} +.icon-cube:before { + content: "\6d"; +} +.icon-sphere:before { + content: "\6e"; +} +.icon-zone:before { + content: "\6f"; +} +.icon-light:before { + content: "\70"; +} +.icon-web:before { + content: "\71"; +} +.icon-web-2:before { + content: "\72"; +} +.icon-edit:before { + content: "\73"; +} +.icon-market:before { + content: "\74"; +} +.icon-directory:before { + content: "\75"; +} +.icon-menu:before { + content: "\76"; +} +.icon-close:before { + content: "\77"; +} +.icon-close-inverted:before { + content: "\78"; +} +.icon-pin:before { + content: "\79"; +} +.icon-pin-inverted:before { + content: "\7a"; +} +.icon-resize-handle:before { + content: "\41"; +} +.icon-diclosure-expand:before { + content: "\42"; +} +.icon-reload-small:before { + content: "\61"; +} +.icon-close-small:before { + content: "\43"; +} +.icon-backward:before { + content: "\45"; +} +.icon-reload:before { + content: "\46"; +} +.icon-minimize:before { + content: "\49"; +} +.icon-maximize:before { + content: "\4a"; +} +.icon-maximize-inverted:before { + content: "\4b"; +} +.icon-disclosure-button-expand:before { + content: "\4c"; +} +.icon-disclosure-button-collapse:before { + content: "\4d"; +} +.icon-script-stop:before { + content: "\4e"; +} +.icon-script-reload:before { + content: "\4f"; +} +.icon-script-run:before { + content: "\50"; +} +.icon-script-new:before { + content: "\51"; +} +.icon-hifi-forum:before { + content: "\32"; +} +.icon-hifi-logo-small:before { + content: "\53"; +} +.icon-avatar-1:before { + content: "\54"; +} +.icon-placemark:before { + content: "\55"; +} +.icon-box:before { + content: "\56"; +} +.icon-community:before { + content: "\30"; +} +.icon-grab-handle:before { + content: "\58"; +} +.icon-search:before { + content: "\59"; +} +.icon-disclosure-collapse:before { + content: "\5a"; +} +.icon-script-upload:before { + content: "\52"; +} +.icon-code:before { + content: "\57"; +} +.icon-avatar:before { + content: "\3c"; +} +.icon-arrows-h:before { + content: "\3a"; +} +.icon-arrows-v:before { + content: "\3b"; +} +.icon-arrows:before { + content: "\60"; +} +.icon-compress:before { + content: "\21"; +} +.icon-expand:before { + content: "\22"; +} +.icon-placemark-1:before { + content: "\23"; +} +.icon-circle:before { + content: "\24"; +} +.icon-hand-pointer:before { + content: "\39"; +} +.icon-plus-square-o:before { + content: "\25"; +} +.icon-square:before { + content: "\27"; +} +.icon-align-center:before { + content: "\38"; +} +.icon-align-justify:before { + content: "\29"; +} +.icon-align-left:before { + content: "\2a"; +} +.icon-align-right:before { + content: "\5e"; +} +.icon-bars:before { + content: "\37"; +} +.icon-circle-slash:before { + content: "\2c"; +} +.icon-sync:before { + content: "\28"; +} +.icon-key:before { + content: "\2d"; +} +.icon-link:before { + content: "\2e"; +} +.icon-location:before { + content: "\2f"; +} +.icon-carat-r:before { + content: "\33"; +} +.icon-carat-l:before { + content: "\34"; +} +.icon-folder-lg:before { + content: "\3e"; +} +.icon-folder-sm:before { + content: "\3f"; +} +.icon-level-up:before { + content: "\31"; +} +.icon-info:before { + content: "\5b"; +} +.icon-question:before { + content: "\5d"; +} +.icon-alert:before { + content: "\2b"; +} +.icon-home:before { + content: "\5f"; +} +.icon-error:before { + content: "\3d"; +} +.icon-settings:before { + content: "\40"; +} +.icon-trash:before { + content: "\7b"; +} +.icon-object-group:before { + content: "\e000"; +} +.icon-cm:before { + content: "\7d"; +} +.icon-msvg:before { + content: "\7e"; +} +.icon-deg:before { + content: "\5c"; +} +.icon-px:before { + content: "\7c"; +} +.icon-m-sq:before { + content: "\e001"; +} +.icon-m-cubed:before { + content: "\e002"; +} +.icon-acceleration:before { + content: "\e003"; +} +.icon-particles:before { + content: "\e004"; +} +.icon-voxels:before { + content: "\e005"; +} +.icon-lock:before { + content: "\e006"; +} +.icon-visible:before { + content: "\e007"; +} +.icon-model:before { + content: "\e008"; +} +.icon-forward:before { + content: "\44"; +} +.icon-avatar-2:before { + content: "\e009"; +} +.icon-arrow-dn:before { + content: "\35"; +} +.icon-arrow-up:before { + content: "\36"; +} +.icon-time:before { + content: "\e00a"; +} +.icon-transparency:before { + content: "\e00b"; +} +.icon-unmuted:before { + content: "\47"; +} +.icon-user:before { + content: "\e00c"; +} +.icon-edit-pencil:before { + content: "\e00d"; +} +.icon-muted:before { + content: "\48"; +} +.icon-vol-0:before { + content: "\e00e"; +} +.icon-vol-1:before { + content: "\e00f"; +} +.icon-vol-2:before { + content: "\e010"; +} +.icon-vol-3:before { + content: "\e011"; +} +.icon-vol-4:before { + content: "\e012"; +} +.icon-vol-x-0:before { + content: "\e013"; +} +.icon-vol-x-1:before { + content: "\e014"; +} +.icon-vol-x-2:before { + content: "\e015"; +} +.icon-vol-x-3:before { + content: "\e016"; +} +.icon-vol-x-4:before { + content: "\e017"; +} +.icon-share-ext:before { + content: "\e018"; +} +.icon-ellipsis:before { + content: "\e019"; +} +.icon-check:before { + content: "\e01a"; +} +.icon-sliders:before { + content: "\26"; +} +.icon-polyline:before { + content: "\e01b"; +} +.icon-source:before { + content: "\e01c"; +} +.icon-playback-play:before { + content: "\e01d"; +} +.icon-stop-square:before { + content: "\e01e"; +} +.icon-avatar-t-pose:before { + content: "\e01f"; +} +.icon-check-2-01:before { + content: "\e020"; +} From 9f2b0258e95829ad7a70a3127197fe95f6379339 Mon Sep 17 00:00:00 2001 From: Atlante45 Date: Tue, 12 Sep 2017 14:10:17 -0700 Subject: [PATCH 098/129] Update baking status with the correct checkmark glyph --- interface/resources/qml/AssetServer.qml | 2 +- interface/resources/qml/styles-uit/HifiConstants.qml | 1 + 2 files changed, 2 insertions(+), 1 deletion(-) diff --git a/interface/resources/qml/AssetServer.qml b/interface/resources/qml/AssetServer.qml index 5e564f8626..44b85b0898 100644 --- a/interface/resources/qml/AssetServer.qml +++ b/interface/resources/qml/AssetServer.qml @@ -561,7 +561,7 @@ ScrollingWindow { case "Not Baked": return hifi.glyphs.circleSlash; case "Baked": - return hifi.glyphs.placemark; + return hifi.glyphs.check_2_01; case "Error": return hifi.glyphs.alert; default: diff --git a/interface/resources/qml/styles-uit/HifiConstants.qml b/interface/resources/qml/styles-uit/HifiConstants.qml index 4a26d11128..a091c6d5cf 100644 --- a/interface/resources/qml/styles-uit/HifiConstants.qml +++ b/interface/resources/qml/styles-uit/HifiConstants.qml @@ -338,5 +338,6 @@ Item { readonly property string stop_square: "\ue01e" readonly property string avatarTPose: "\ue01f" readonly property string lock: "\ue006" + readonly property string check_2_01: "\ue020" } } From 2e84f48539255d6cc515f9767e5e53df1b25ecd2 Mon Sep 17 00:00:00 2001 From: Atlante45 Date: Tue, 12 Sep 2017 15:49:49 -0700 Subject: [PATCH 099/129] Fix QML errors --- interface/resources/qml/AssetServer.qml | 2 +- interface/resources/qml/controls-uit/Tree.qml | 7 ------- interface/resources/qml/hifi/dialogs/RunningScripts.qml | 7 +++++++ .../resources/qml/hifi/dialogs/TabletRunningScripts.qml | 7 +++++++ 4 files changed, 15 insertions(+), 8 deletions(-) diff --git a/interface/resources/qml/AssetServer.qml b/interface/resources/qml/AssetServer.qml index 44b85b0898..5d6426e3a7 100644 --- a/interface/resources/qml/AssetServer.qml +++ b/interface/resources/qml/AssetServer.qml @@ -651,7 +651,7 @@ ScrollingWindow { } onAccepted: { if (acceptableInput && styleData.selected) { - if (!modifyEl(selection.currentIndex, text)) { + if (!treeView.modifyEl(treeView.selection.currentIndex, text)) { text = styleData.value; } unfocusHelper.forceActiveFocus(); diff --git a/interface/resources/qml/controls-uit/Tree.qml b/interface/resources/qml/controls-uit/Tree.qml index d94a8cdceb..79d3b958ea 100644 --- a/interface/resources/qml/controls-uit/Tree.qml +++ b/interface/resources/qml/controls-uit/Tree.qml @@ -225,11 +225,4 @@ TreeView { onClicked: { selectionModel.setCurrentIndex(index, ItemSelectionModel.ClearAndSelect); } - - onActivated: { - var path = scriptsModel.data(index, 0x100) - if (path) { - loadScript(path) - } - } } diff --git a/interface/resources/qml/hifi/dialogs/RunningScripts.qml b/interface/resources/qml/hifi/dialogs/RunningScripts.qml index a649c2c4d6..5263f0fa8c 100644 --- a/interface/resources/qml/hifi/dialogs/RunningScripts.qml +++ b/interface/resources/qml/hifi/dialogs/RunningScripts.qml @@ -375,6 +375,13 @@ ScrollingWindow { TableViewColumn { role: "display"; } + + onActivated: { + var path = scriptsModel.data(index, 0x100) + if (path) { + loadScript(path) + } + } } HifiControls.VerticalSpacer { diff --git a/interface/resources/qml/hifi/dialogs/TabletRunningScripts.qml b/interface/resources/qml/hifi/dialogs/TabletRunningScripts.qml index 5677cc92a1..c5be38eed5 100644 --- a/interface/resources/qml/hifi/dialogs/TabletRunningScripts.qml +++ b/interface/resources/qml/hifi/dialogs/TabletRunningScripts.qml @@ -426,6 +426,13 @@ Rectangle { } } } + + onActivated: { + var path = scriptsModel.data(index, 0x100) + if (path) { + loadScript(path) + } + } } Item { From 9e11ae311e9c01b537df38ddf6a3ec38976dea63 Mon Sep 17 00:00:00 2001 From: Atlante45 Date: Tue, 12 Sep 2017 18:07:26 -0700 Subject: [PATCH 100/129] Fix error reporting --- assignment-client/src/assets/AssetServer.cpp | 4 +++- 1 file changed, 3 insertions(+), 1 deletion(-) diff --git a/assignment-client/src/assets/AssetServer.cpp b/assignment-client/src/assets/AssetServer.cpp index 6da299ea87..f1e921c768 100644 --- a/assignment-client/src/assets/AssetServer.cpp +++ b/assignment-client/src/assets/AssetServer.cpp @@ -1166,6 +1166,8 @@ void AssetServer::handleFailedBake(QString originalAssetHash, QString assetPath, meta.lastBakeErrors = errors; writeMetaFile(originalAssetHash, meta); + + _pendingBakes.remove(originalAssetHash); } void AssetServer::handleCompletedBake(QString originalAssetHash, QString originalAssetPath, QVector bakedFilePaths) { @@ -1286,7 +1288,7 @@ std::pair AssetServer::readMetaFile(AssetHash hash) { meta.bakeVersion = bakeVersion; meta.applicationVersion = appVersion; meta.failedLastBake = failedLastBake.toBool(); - meta.lastBakeErrors = failedLastBake.toString(); + meta.lastBakeErrors = lastBakeErrors.toString(); return { true, meta }; } else { From c04a6b5b88714bee948117e3f2d789cab9a7e4db Mon Sep 17 00:00:00 2001 From: Atlante45 Date: Tue, 12 Sep 2017 18:20:04 -0700 Subject: [PATCH 101/129] Add error tooltip --- interface/resources/qml/AssetServer.qml | 8 +++ .../resources/qml/controls-uit/ToolTip.qml | 51 +++++++++++++++++++ 2 files changed, 59 insertions(+) create mode 100644 interface/resources/qml/controls-uit/ToolTip.qml diff --git a/interface/resources/qml/AssetServer.qml b/interface/resources/qml/AssetServer.qml index 5d6426e3a7..0c4b952331 100644 --- a/interface/resources/qml/AssetServer.qml +++ b/interface/resources/qml/AssetServer.qml @@ -606,6 +606,14 @@ ScrollingWindow { elide: Text.ElideRight horizontalAlignment: TextInput.AlignHCenter + + HifiControls.ToolTip { + anchors.fill: parent + + visible: styleData.value === "Error" + + toolTip: assetProxyModel.data(styleData.index, 0x106) + } } } Component { diff --git a/interface/resources/qml/controls-uit/ToolTip.qml b/interface/resources/qml/controls-uit/ToolTip.qml new file mode 100644 index 0000000000..4fe36adcd5 --- /dev/null +++ b/interface/resources/qml/controls-uit/ToolTip.qml @@ -0,0 +1,51 @@ +// +// ToolTip.qml +// +// Created by Clement on 9/12/17 +// Copyright 2017 High Fidelity, Inc. +// +// Distributed under the Apache License, Version 2.0. +// See the accompanying file LICENSE or http://www.apache.org/licenses/LICENSE-2.0.html +// + +import QtQuick 2.5 + +Item { + property string toolTip + property bool showToolTip: false + + Rectangle { + id: toolTipRectangle + anchors.right: parent.right + + width: toolTipText.width + 4 + height: toolTipText.height + 4 + opacity: (toolTip != "" && showToolTip) ? 1 : 0 + color: "#ffffaa" + border.color: "#0a0a0a" + Text { + id: toolTipText + text: toolTip + color: "black" + anchors.centerIn: parent + } + Behavior on opacity { + PropertyAnimation { + easing.type: Easing.InOutQuad + duration: 250 + } + } + } + MouseArea { + id: mouseArea + anchors.fill: parent + onEntered: showTimer.start() + onExited: { showToolTip = false; showTimer.stop(); } + hoverEnabled: true + } + Timer { + id: showTimer + interval: 250 + onTriggered: { showToolTip = true; } + } +} \ No newline at end of file From 4066c9392d309b0532f4e4bb7fe62acbd76a6e77 Mon Sep 17 00:00:00 2001 From: Stephen Birarda Date: Wed, 13 Sep 2017 13:34:15 -0700 Subject: [PATCH 102/129] output from oven into folder with model name --- tools/oven/src/ui/ModelBakeWidget.cpp | 44 +++++++++++++++++---------- 1 file changed, 28 insertions(+), 16 deletions(-) diff --git a/tools/oven/src/ui/ModelBakeWidget.cpp b/tools/oven/src/ui/ModelBakeWidget.cpp index 7829993740..7963b3f3c4 100644 --- a/tools/oven/src/ui/ModelBakeWidget.cpp +++ b/tools/oven/src/ui/ModelBakeWidget.cpp @@ -156,22 +156,6 @@ void ModelBakeWidget::outputDirectoryChanged(const QString& newDirectory) { } void ModelBakeWidget::bakeButtonClicked() { - // make sure we have a valid output directory - QDir outputDirectory(_outputDirLineEdit->text()); - - outputDirectory.mkdir("."); - if (!outputDirectory.exists()) { - QMessageBox::warning(this, "Unable to create directory", "Unable to create output directory. Please create it manually or choose a different directory."); - return; - } - - QDir bakedOutputDirectory = outputDirectory.absoluteFilePath("baked"); - QDir originalOutputDirectory = outputDirectory.absoluteFilePath("original"); - - bakedOutputDirectory.mkdir("."); - originalOutputDirectory.mkdir("."); - - // make sure we have a non empty URL to a model to bake if (_modelLineEdit->text().isEmpty()) { @@ -193,6 +177,34 @@ void ModelBakeWidget::bakeButtonClicked() { qDebug() << "New url: " << modelToBakeURL; } + auto modelName = modelToBakeURL.fileName().left(modelToBakeURL.fileName().lastIndexOf('.')); + + // make sure we have a valid output directory + QDir outputDirectory(_outputDirLineEdit->text()); + QString subFolderName = modelName + "/"; + + // output in a sub-folder with the name of the fbx, potentially suffixed by a number to make it unique + int iteration = 0; + + while (outputDirectory.exists(subFolderName)) { + subFolderName = modelName + "-" + QString::number(++iteration) + "/"; + } + + outputDirectory.mkdir(subFolderName); + + if (!outputDirectory.exists()) { + QMessageBox::warning(this, "Unable to create directory", "Unable to create output directory. Please create it manually or choose a different directory."); + return; + } + + outputDirectory.cd(subFolderName); + + QDir bakedOutputDirectory = outputDirectory.absoluteFilePath("baked"); + QDir originalOutputDirectory = outputDirectory.absoluteFilePath("original"); + + bakedOutputDirectory.mkdir("."); + originalOutputDirectory.mkdir("."); + // everything seems to be in place, kick off a bake for this model now auto baker = std::unique_ptr { new FBXBaker(modelToBakeURL, []() -> QThread* { From 1e6154fbc8a500278da067bd18594aee0d291c56 Mon Sep 17 00:00:00 2001 From: Stephen Birarda Date: Wed, 13 Sep 2017 13:45:26 -0700 Subject: [PATCH 103/129] make draco compile flags OS X only --- cmake/externals/draco/CMakeLists.txt | 4 +++- libraries/baking/CMakeLists.txt | 2 +- libraries/fbx/CMakeLists.txt | 2 +- 3 files changed, 5 insertions(+), 3 deletions(-) diff --git a/cmake/externals/draco/CMakeLists.txt b/cmake/externals/draco/CMakeLists.txt index 07ce861820..3534461443 100644 --- a/cmake/externals/draco/CMakeLists.txt +++ b/cmake/externals/draco/CMakeLists.txt @@ -4,7 +4,9 @@ if (ANDROID) set(ANDROID_CMAKE_ARGS "-DCMAKE_TOOLCHAIN_FILE=${CMAKE_TOOLCHAIN_FILE}" "-DANDROID_NATIVE_API_LEVEL=19") endif () -set(EXTRA_CMAKE_FLAGS -DCMAKE_CXX_FLAGS=-stdlib=libc++ -DCMAKE_EXE_LINKER_FLAGS=-stdlib=libc++) +if (APPLE) + set(EXTRA_CMAKE_FLAGS -DCMAKE_CXX_FLAGS=-stdlib=libc++ -DCMAKE_EXE_LINKER_FLAGS=-stdlib=libc++) +endif () include(ExternalProject) ExternalProject_Add( diff --git a/libraries/baking/CMakeLists.txt b/libraries/baking/CMakeLists.txt index 2304a5e0c0..66cf791776 100644 --- a/libraries/baking/CMakeLists.txt +++ b/libraries/baking/CMakeLists.txt @@ -6,5 +6,5 @@ include_hifi_library_headers(gpu) add_dependency_external_projects(draco) find_package(Draco REQUIRED) -include_directories(SYSTEM ${DRACO_INCLUDE_DIRS}) +target_include_directories(${TARGET_NAME} SYSTEM PRIVATE ${DRACO_INCLUDE_DIRS}) target_link_libraries(${TARGET_NAME} ${DRACO_LIBRARY} ${DRACO_ENCODER_LIBRARY}) diff --git a/libraries/fbx/CMakeLists.txt b/libraries/fbx/CMakeLists.txt index 4c81f10302..7cead5aa4f 100644 --- a/libraries/fbx/CMakeLists.txt +++ b/libraries/fbx/CMakeLists.txt @@ -6,5 +6,5 @@ include_hifi_library_headers(gpu image) add_dependency_external_projects(draco) find_package(Draco REQUIRED) -include_directories(SYSTEM ${DRACO_INCLUDE_DIRS}) +target_include_directories(${TARGET_NAME} SYSTEM PRIVATE ${DRACO_INCLUDE_DIRS}) target_link_libraries(${TARGET_NAME} ${DRACO_LIBRARY} ${DRACO_ENCODER_LIBRARY}) From c365e1718ee3d999a146db41d2860d1f70a2f3a5 Mon Sep 17 00:00:00 2001 From: Stephen Birarda Date: Wed, 13 Sep 2017 13:51:26 -0700 Subject: [PATCH 104/129] set include dir for draco the same across the board --- cmake/externals/draco/CMakeLists.txt | 6 +----- 1 file changed, 1 insertion(+), 5 deletions(-) diff --git a/cmake/externals/draco/CMakeLists.txt b/cmake/externals/draco/CMakeLists.txt index 3534461443..44ddd6d3de 100644 --- a/cmake/externals/draco/CMakeLists.txt +++ b/cmake/externals/draco/CMakeLists.txt @@ -26,11 +26,7 @@ ExternalProject_Get_Property(${EXTERNAL_NAME} INSTALL_DIR) string(TOUPPER ${EXTERNAL_NAME} EXTERNAL_NAME_UPPER) -if (UNIX AND NOT APPLE) - set(${EXTERNAL_NAME_UPPER}_INCLUDE_DIRS ${INSTALL_DIR}/include/draco/src CACHE PATH "List of Draco include directories") -else () - set(${EXTERNAL_NAME_UPPER}_INCLUDE_DIRS ${INSTALL_DIR}/include CACHE PATH "List of Draco include directories") -endif () +set(${EXTERNAL_NAME_UPPER}_INCLUDE_DIRS ${INSTALL_DIR}/include CACHE PATH "List of Draco include directories") if (UNIX) set(LIB_PREFIX "lib") From bfb4dd0cdb00b23133fa7be12f4cba16ea32a75d Mon Sep 17 00:00:00 2001 From: Stephen Birarda Date: Wed, 13 Sep 2017 14:00:41 -0700 Subject: [PATCH 105/129] disable warning c4267 when including draco --- libraries/baking/src/FBXBaker.cpp | 10 ++++++++++ libraries/fbx/src/FBXReader_Mesh.cpp | 9 +++++++++ 2 files changed, 19 insertions(+) diff --git a/libraries/baking/src/FBXBaker.cpp b/libraries/baking/src/FBXBaker.cpp index 79a5b9157b..5abb3feb0d 100644 --- a/libraries/baking/src/FBXBaker.cpp +++ b/libraries/baking/src/FBXBaker.cpp @@ -33,9 +33,19 @@ #include "FBXBaker.h" +#ifdef _WIN32 +#pragma warning( push ) +#pragma warning( disable : 4267 ) +#endif + #include #include +#ifdef _WIN32 +#pragma warning( pop ) +#endif + + FBXBaker::FBXBaker(const QUrl& fbxURL, TextureBakerThreadGetter textureThreadGetter, const QString& bakedOutputDir, const QString& originalOutputDir) : _fbxURL(fbxURL), diff --git a/libraries/fbx/src/FBXReader_Mesh.cpp b/libraries/fbx/src/FBXReader_Mesh.cpp index 323ecac37b..fd9756b4c0 100644 --- a/libraries/fbx/src/FBXReader_Mesh.cpp +++ b/libraries/fbx/src/FBXReader_Mesh.cpp @@ -9,8 +9,17 @@ // See the accompanying file LICENSE or http://www.apache.org/licenses/LICENSE-2.0.html // +#ifdef _WIN32 +#pragma warning( push ) +#pragma warning( disable : 4267 ) +#endif + #include +#ifdef _WIN32 +#pragma warning( pop ) +#endif + #include #include #include From 9cb4e2c5f2ba61b409f5ae7efa2e32dd8daab4b5 Mon Sep 17 00:00:00 2001 From: Stephen Birarda Date: Thu, 14 Sep 2017 13:54:04 -0700 Subject: [PATCH 106/129] address code review comments --- libraries/baking/src/FBXBaker.cpp | 14 ++-------- libraries/fbx/src/FBXReader_Mesh.cpp | 38 ++++++++++++---------------- libraries/fbx/src/FBXWriter.cpp | 6 ++--- 3 files changed, 20 insertions(+), 38 deletions(-) diff --git a/libraries/baking/src/FBXBaker.cpp b/libraries/baking/src/FBXBaker.cpp index 5abb3feb0d..791d89e503 100644 --- a/libraries/baking/src/FBXBaker.cpp +++ b/libraries/baking/src/FBXBaker.cpp @@ -299,7 +299,7 @@ void FBXBaker::rewriteAndBakeSceneModels() { // TODO Pull this out of _geometry instead so we don't have to reprocess it auto extractedMesh = FBXReader::extractMesh(objectChild, meshIndex); - auto mesh = extractedMesh.mesh; + auto& mesh = extractedMesh.mesh; Q_ASSERT(mesh.normals.size() == 0 || mesh.normals.size() == mesh.vertices.size()); Q_ASSERT(mesh.colors.size() == 0 || mesh.colors.size() == mesh.vertices.size()); @@ -366,7 +366,7 @@ void FBXBaker::rewriteAndBakeSceneModels() { auto partIndex = 0; draco::FaceIndex face; for (auto& part : mesh.parts) { - const auto matTex = extractedMesh.partMaterialTextures[partIndex]; + const auto& matTex = extractedMesh.partMaterialTextures[partIndex]; auto addFace = [&](QVector& indices, int index, draco::FaceIndex face) { auto idx0 = indices[index]; @@ -447,16 +447,6 @@ void FBXBaker::rewriteAndBakeSceneModels() { auto value = QVariant::fromValue(QByteArray(buffer.data(), (int) buffer.size())); dracoMeshNode.properties.append(value); - - QFile file("C:/Users/huffm/encodedFBX/" + this->_fbxURL.fileName() + "-" + QString::number(meshIndex) + ".drc"); - if (file.open(QIODevice::WriteOnly)) { - file.write(buffer.data(), buffer.size()); - file.close(); - } else { - qWarning() << "Failed to write to: " << file.fileName(); - - } - objectChild.children.push_back(dracoMeshNode); static const std::vector nodeNamesToDelete { diff --git a/libraries/fbx/src/FBXReader_Mesh.cpp b/libraries/fbx/src/FBXReader_Mesh.cpp index fd9756b4c0..bb35447adb 100644 --- a/libraries/fbx/src/FBXReader_Mesh.cpp +++ b/libraries/fbx/src/FBXReader_Mesh.cpp @@ -361,11 +361,11 @@ ExtractedMesh FBXReader::extractMesh(const FBXNode& object, unsigned int& meshIn QHash, int> materialTextureParts; - data.extracted.mesh.vertices.reserve(numVertices); - data.extracted.mesh.normals.reserve(numVertices); - data.extracted.mesh.texCoords.reserve(numVertices); - data.extracted.mesh.texCoords1.reserve(numVertices); - data.extracted.mesh.colors.reserve(numVertices); + data.extracted.mesh.vertices.resize(numVertices); + data.extracted.mesh.normals.resize(numVertices); + data.extracted.mesh.texCoords.resize(numVertices); + data.extracted.mesh.texCoords1.resize(numVertices); + data.extracted.mesh.colors.resize(numVertices); // enumerate the vertices and construct the extracted mesh for (int i = 0; i < numVertices; ++i) { @@ -375,46 +375,40 @@ ExtractedMesh FBXReader::extractMesh(const FBXNode& object, unsigned int& meshIn // read position from draco mesh to extracted mesh auto mappedIndex = positionAttribute->mapped_index(vertexIndex); - std::array positionValue; - positionAttribute->ConvertValue(mappedIndex, &positionValue[0]); - data.extracted.mesh.vertices.push_back({ positionValue[0], positionValue[1], positionValue[2] }); + positionAttribute->ConvertValue(mappedIndex, + reinterpret_cast(&data.extracted.mesh.vertices[i])); } if (normalAttribute) { // read normals from draco mesh to extracted mesh auto mappedIndex = normalAttribute->mapped_index(vertexIndex); - std::array normalValue; - normalAttribute->ConvertValue(mappedIndex, &normalValue[0]); - data.extracted.mesh.normals.push_back({ normalValue[0], normalValue[1], normalValue[2] }); + normalAttribute->ConvertValue(mappedIndex, + reinterpret_cast(&data.extracted.mesh.normals[i])); } if (texCoordAttribute) { // read UVs from draco mesh to extracted mesh auto mappedIndex = texCoordAttribute->mapped_index(vertexIndex); - std::array texCoordValue; - texCoordAttribute->ConvertValue(mappedIndex, &texCoordValue[0]); - data.extracted.mesh.texCoords.push_back({ texCoordValue[0], texCoordValue[1] }); + texCoordAttribute->ConvertValue(mappedIndex, + reinterpret_cast(&data.extracted.mesh.texCoords[i])); } if (extraTexCoordAttribute) { // some meshes have a second set of UVs, read those to extracted mesh auto mappedIndex = extraTexCoordAttribute->mapped_index(vertexIndex); - std::array texCoordValue; - extraTexCoordAttribute->ConvertValue(mappedIndex, &texCoordValue[0]); - data.extracted.mesh.texCoords1.push_back({ texCoordValue[0], texCoordValue[1] }); + extraTexCoordAttribute->ConvertValue(mappedIndex, + reinterpret_cast(&data.extracted.mesh.texCoords1[i])); } if (colorAttribute) { // read vertex colors from draco mesh to extracted mesh auto mappedIndex = colorAttribute->mapped_index(vertexIndex); - std::array colorValue; - - colorAttribute->ConvertValue(mappedIndex, &colorValue[0]); - data.extracted.mesh.colors.push_back({ colorValue[0], colorValue[1], colorValue[2] }); + colorAttribute->ConvertValue(mappedIndex, + reinterpret_cast(&data.extracted.mesh.colors[i])); } data.extracted.newIndices.insert(i, i); @@ -422,7 +416,7 @@ ExtractedMesh FBXReader::extractMesh(const FBXNode& object, unsigned int& meshIn for (int i = 0; i < dracoMesh->num_faces(); ++i) { // grab the material ID and texture ID for this face, if we have it - auto firstCorner = dracoMesh->face(draco::FaceIndex(i))[0]; + auto& firstCorner = dracoMesh->face(draco::FaceIndex(i))[0]; uint16_t materialID { 0 }; diff --git a/libraries/fbx/src/FBXWriter.cpp b/libraries/fbx/src/FBXWriter.cpp index cc34696f92..7195d2abb4 100644 --- a/libraries/fbx/src/FBXWriter.cpp +++ b/libraries/fbx/src/FBXWriter.cpp @@ -19,9 +19,8 @@ void writeVector(QDataStream& out, char ch, QVector list) { out << (int32_t)list.length(); out << (int32_t)0; out << (int32_t)0; - for (auto& value : list) { - out << value; - } + + out.writeBytes(reinterpret_cast(list.constData()), list.length() * sizeof(T)); } @@ -98,7 +97,6 @@ void FBXWriter::encodeFBXProperty(QDataStream& out, const QVariant& prop) { case QVariant::Type::Bool: out.device()->write("C", 1); - //out.device()->write(prop.toBool() ? 1 : 0, 1); out << prop.toBool(); break; From bb98df38d4c9f4add6b320e2bbbaaa1295b0b96b Mon Sep 17 00:00:00 2001 From: Stephen Birarda Date: Thu, 14 Sep 2017 14:42:40 -0700 Subject: [PATCH 107/129] remove re-grabbing of draco faces --- libraries/fbx/src/FBXReader_Mesh.cpp | 7 ++++--- 1 file changed, 4 insertions(+), 3 deletions(-) diff --git a/libraries/fbx/src/FBXReader_Mesh.cpp b/libraries/fbx/src/FBXReader_Mesh.cpp index bb35447adb..163a92e3bd 100644 --- a/libraries/fbx/src/FBXReader_Mesh.cpp +++ b/libraries/fbx/src/FBXReader_Mesh.cpp @@ -416,7 +416,8 @@ ExtractedMesh FBXReader::extractMesh(const FBXNode& object, unsigned int& meshIn for (int i = 0; i < dracoMesh->num_faces(); ++i) { // grab the material ID and texture ID for this face, if we have it - auto& firstCorner = dracoMesh->face(draco::FaceIndex(i))[0]; + auto& dracoFace = dracoMesh->face(draco::FaceIndex(i)); + auto& firstCorner = dracoFace[0]; uint16_t materialID { 0 }; @@ -440,8 +441,8 @@ ExtractedMesh FBXReader::extractMesh(const FBXNode& object, unsigned int& meshIn // give the mesh part this index FBXMeshPart& part = data.extracted.mesh.parts[partIndexPlusOne - 1]; part.triangleIndices.append(firstCorner.value()); - part.triangleIndices.append(dracoMesh->face(draco::FaceIndex(i))[1].value()); - part.triangleIndices.append(dracoMesh->face(draco::FaceIndex(i))[2].value()); + part.triangleIndices.append(dracoFace[1].value()); + part.triangleIndices.append(dracoFace[2].value()); } } } From 43196cd0a66cb048d7198a225ebe396ed61b3bc4 Mon Sep 17 00:00:00 2001 From: Stephen Birarda Date: Thu, 14 Sep 2017 14:46:48 -0700 Subject: [PATCH 108/129] fix buffer resizing to only occur when attribute is present --- libraries/fbx/src/FBXReader_Mesh.cpp | 20 ++++++++++++++++---- 1 file changed, 16 insertions(+), 4 deletions(-) diff --git a/libraries/fbx/src/FBXReader_Mesh.cpp b/libraries/fbx/src/FBXReader_Mesh.cpp index 163a92e3bd..bef36770a0 100644 --- a/libraries/fbx/src/FBXReader_Mesh.cpp +++ b/libraries/fbx/src/FBXReader_Mesh.cpp @@ -362,10 +362,22 @@ ExtractedMesh FBXReader::extractMesh(const FBXNode& object, unsigned int& meshIn QHash, int> materialTextureParts; data.extracted.mesh.vertices.resize(numVertices); - data.extracted.mesh.normals.resize(numVertices); - data.extracted.mesh.texCoords.resize(numVertices); - data.extracted.mesh.texCoords1.resize(numVertices); - data.extracted.mesh.colors.resize(numVertices); + + if (normalAttribute) { + data.extracted.mesh.normals.resize(numVertices); + } + + if (texCoordAttribute) { + data.extracted.mesh.texCoords.resize(numVertices); + } + + if (extraTexCoordAttribute) { + data.extracted.mesh.texCoords1.resize(numVertices); + } + + if (colorAttribute) { + data.extracted.mesh.colors.resize(numVertices); + } // enumerate the vertices and construct the extracted mesh for (int i = 0; i < numVertices; ++i) { From 2a62bac4f92960b7b5e57622940cc041228d8160 Mon Sep 17 00:00:00 2001 From: Stephen Birarda Date: Thu, 14 Sep 2017 16:00:20 -0700 Subject: [PATCH 109/129] refuse to re-bake an already baked FBX --- libraries/baking/src/FBXBaker.cpp | 7 ++++++- libraries/fbx/src/FBX.h | 1 + libraries/fbx/src/FBXReader_Mesh.cpp | 1 + 3 files changed, 8 insertions(+), 1 deletion(-) diff --git a/libraries/baking/src/FBXBaker.cpp b/libraries/baking/src/FBXBaker.cpp index 791d89e503..2a9deb44f1 100644 --- a/libraries/baking/src/FBXBaker.cpp +++ b/libraries/baking/src/FBXBaker.cpp @@ -301,6 +301,11 @@ void FBXBaker::rewriteAndBakeSceneModels() { auto extractedMesh = FBXReader::extractMesh(objectChild, meshIndex); auto& mesh = extractedMesh.mesh; + if (mesh.wasCompressed) { + handleError("Cannot re-bake a file that contains compressed mesh"); + return; + } + Q_ASSERT(mesh.normals.size() == 0 || mesh.normals.size() == mesh.vertices.size()); Q_ASSERT(mesh.colors.size() == 0 || mesh.colors.size() == mesh.vertices.size()); Q_ASSERT(mesh.texCoords.size() == 0 || mesh.texCoords.size() == mesh.vertices.size()); @@ -529,7 +534,7 @@ void FBXBaker::rewriteAndBakeSceneTextures() { if (textureFileInfo.suffix() == BAKED_TEXTURE_EXT.mid(1)) { // re-baking an FBX that already references baked textures is a fail // so we add an error and return from here - handleError("Cannot re-bake a partially baked FBX file that references baked KTX textures"); + handleError("Cannot re-bake a file that references compressed textures"); return; } diff --git a/libraries/fbx/src/FBX.h b/libraries/fbx/src/FBX.h index 9f5fad7d66..89bc893df2 100644 --- a/libraries/fbx/src/FBX.h +++ b/libraries/fbx/src/FBX.h @@ -234,6 +234,7 @@ public: unsigned int meshIndex; // the order the meshes appeared in the object file model::MeshPointer _mesh; + bool wasCompressed { false }; }; class ExtractedMesh { diff --git a/libraries/fbx/src/FBXReader_Mesh.cpp b/libraries/fbx/src/FBXReader_Mesh.cpp index bef36770a0..93976afe90 100644 --- a/libraries/fbx/src/FBXReader_Mesh.cpp +++ b/libraries/fbx/src/FBXReader_Mesh.cpp @@ -338,6 +338,7 @@ ExtractedMesh FBXReader::extractMesh(const FBXNode& object, unsigned int& meshIn } } else if (child.name == "DracoMesh") { isDracoMesh = true; + data.extracted.mesh.wasCompressed = true; // load the draco mesh from the FBX and create a draco::Mesh draco::Decoder decoder; From 379306204530c1afa5e81649ee61ec329c973637 Mon Sep 17 00:00:00 2001 From: Ryan Huffman Date: Wed, 13 Sep 2017 09:51:06 -0700 Subject: [PATCH 110/129] Add optional FBX 2015 encoding to FBXWriter --- libraries/fbx/src/FBX.h | 1 + libraries/fbx/src/FBXWriter.cpp | 26 ++++++++++++++++++++------ libraries/fbx/src/FBXWriter.h | 2 ++ 3 files changed, 23 insertions(+), 6 deletions(-) diff --git a/libraries/fbx/src/FBX.h b/libraries/fbx/src/FBX.h index 9f5fad7d66..6af23a0151 100644 --- a/libraries/fbx/src/FBX.h +++ b/libraries/fbx/src/FBX.h @@ -30,6 +30,7 @@ static const QByteArray FBX_BINARY_PROLOG = "Kaydara FBX Binary "; static const int FBX_HEADER_BYTES_BEFORE_VERSION = 23; +static const quint32 FBX_VERSION_2015 = 7400; static const quint32 FBX_VERSION_2016 = 7500; diff --git a/libraries/fbx/src/FBXWriter.cpp b/libraries/fbx/src/FBXWriter.cpp index 7195d2abb4..764d8df6cf 100644 --- a/libraries/fbx/src/FBXWriter.cpp +++ b/libraries/fbx/src/FBXWriter.cpp @@ -13,6 +13,16 @@ #include +#ifdef USE_FBX_2016_FORMAT + using FBXEndOffset = int64_t; + using FBXPropertyCount = uint64_t; + using FBXListLength = uint64_t; +#else + using FBXEndOffset = int32_t; + using FBXPropertyCount = uint32_t; + using FBXListLength = uint32_t; +#endif + template void writeVector(QDataStream& out, char ch, QVector list) { out.device()->write(&ch, 1); @@ -34,7 +44,11 @@ QByteArray FBXWriter::encodeFBX(const FBXNode& root) { auto bytes = QByteArray(FBX_HEADER_BYTES_BEFORE_VERSION - FBX_BINARY_PROLOG.size(), '\0'); out.writeRawData(bytes, bytes.size()); +#ifdef USE_FBX_2016_FORMAT out << FBX_VERSION_2016; +#else + out << FBX_VERSION_2015; +#endif for (auto& child : root.children) { encodeNode(out, child); @@ -51,13 +65,13 @@ void FBXWriter::encodeNode(QDataStream& out, const FBXNode& node) { auto nodeStartPos = device->pos(); // endOffset (temporary, updated later) - out << (qint64)0; + out << (FBXEndOffset)0; // Property count - out << (quint64)node.properties.size(); + out << (FBXPropertyCount)node.properties.size(); // Property list length (temporary, updated later) - out << (quint64)0; + out << (FBXListLength)0; out << (quint8)node.name.size(); out.writeRawData(node.name, node.name.size()); @@ -70,8 +84,8 @@ void FBXWriter::encodeNode(QDataStream& out, const FBXNode& node) { // Go back and write property list length auto nodePropertiesEndPos = device->pos(); - device->seek(nodeStartPos + sizeof(qint64) + sizeof(quint64)); - out << (quint64)(nodePropertiesEndPos - nodePropertiesStartPos); + device->seek(nodeStartPos + sizeof(FBXEndOffset) + sizeof(FBXPropertyCount)); + out << (FBXListLength)(nodePropertiesEndPos - nodePropertiesStartPos); device->seek(nodePropertiesEndPos); @@ -86,7 +100,7 @@ void FBXWriter::encodeNode(QDataStream& out, const FBXNode& node) { // Go back and write actual endOffset auto nodeEndPos = device->pos(); device->seek(nodeStartPos); - out << (qint64)(nodeEndPos); + out << (FBXEndOffset)(nodeEndPos); device->seek(nodeEndPos); } diff --git a/libraries/fbx/src/FBXWriter.h b/libraries/fbx/src/FBXWriter.h index fa33983345..57c33e0e1c 100644 --- a/libraries/fbx/src/FBXWriter.h +++ b/libraries/fbx/src/FBXWriter.h @@ -17,6 +17,8 @@ #include #include +#define USE_FBX_2016_FORMAT + class FBXWriter { public: static QByteArray encodeFBX(const FBXNode& root); From 7db8d1dcbc3f66c1da75281cadd27fc121d0b7b7 Mon Sep 17 00:00:00 2001 From: Ryan Huffman Date: Thu, 14 Sep 2017 14:29:54 -0700 Subject: [PATCH 111/129] Add support for baked skinned meshes --- libraries/baking/src/FBXBaker.cpp | 21 +++++++++++++---- libraries/fbx/src/FBX.h | 2 ++ libraries/fbx/src/FBXReader.h | 2 +- libraries/fbx/src/FBXReader_Mesh.cpp | 34 +++++++++++++++++++--------- libraries/fbx/src/FBXWriter.cpp | 9 +++++--- 5 files changed, 48 insertions(+), 20 deletions(-) diff --git a/libraries/baking/src/FBXBaker.cpp b/libraries/baking/src/FBXBaker.cpp index 791d89e503..380e893c9b 100644 --- a/libraries/baking/src/FBXBaker.cpp +++ b/libraries/baking/src/FBXBaker.cpp @@ -298,7 +298,7 @@ void FBXBaker::rewriteAndBakeSceneModels() { if (objectChild.name == "Geometry") { // TODO Pull this out of _geometry instead so we don't have to reprocess it - auto extractedMesh = FBXReader::extractMesh(objectChild, meshIndex); + auto extractedMesh = FBXReader::extractMesh(objectChild, meshIndex, false); auto& mesh = extractedMesh.mesh; Q_ASSERT(mesh.normals.size() == 0 || mesh.normals.size() == mesh.vertices.size()); @@ -338,6 +338,9 @@ void FBXBaker::rewriteAndBakeSceneModels() { const int positionAttributeID = meshBuilder.AddAttribute(draco::GeometryAttribute::POSITION, 3, draco::DT_FLOAT32); + const int originalIndexAttributeID = meshBuilder.AddAttribute( + (draco::GeometryAttribute::Type)DRACO_ATTRIBUTE_ORIGINAL_INDEX, + 1, draco::DT_INT32); if (hasNormals) { normalsAttributeID = meshBuilder.AddAttribute(draco::GeometryAttribute::NORMAL, @@ -367,14 +370,14 @@ void FBXBaker::rewriteAndBakeSceneModels() { draco::FaceIndex face; for (auto& part : mesh.parts) { const auto& matTex = extractedMesh.partMaterialTextures[partIndex]; + uint16_t materialID = matTex.first; auto addFace = [&](QVector& indices, int index, draco::FaceIndex face) { - auto idx0 = indices[index]; - auto idx1 = indices[index + 1]; - auto idx2 = indices[index + 2]; + int32_t idx0 = indices[index]; + int32_t idx1 = indices[index + 1]; + int32_t idx2 = indices[index + 2]; if (hasPerFaceMaterials) { - uint16_t materialID = matTex.first; meshBuilder.SetPerFaceAttributeValueForFace(faceMaterialAttributeID, face, &materialID); } @@ -382,6 +385,12 @@ void FBXBaker::rewriteAndBakeSceneModels() { &mesh.vertices[idx0], &mesh.vertices[idx1], &mesh.vertices[idx2]); + if (originalIndexAttributeID) { + meshBuilder.SetAttributeValuesForFace(originalIndexAttributeID, face, + &mesh.originalIndex[idx0], + &mesh.originalIndex[idx1], + &mesh.originalIndex[idx2]); + } if (hasNormals) { meshBuilder.SetAttributeValuesForFace(normalsAttributeID, face, &mesh.normals[idx0], &mesh.normals[idx1], @@ -432,6 +441,8 @@ void FBXBaker::rewriteAndBakeSceneModels() { dracoMesh->attribute(texCoords1AttributeID)->set_unique_id(DRACO_ATTRIBUTE_TEX_COORD_1); } + dracoMesh->attribute(originalIndexAttributeID)->set_unique_id(DRACO_ATTRIBUTE_ORIGINAL_INDEX); + draco::Encoder encoder; encoder.SetAttributeQuantization(draco::GeometryAttribute::POSITION, 14); diff --git a/libraries/fbx/src/FBX.h b/libraries/fbx/src/FBX.h index 6af23a0151..78d35889cf 100644 --- a/libraries/fbx/src/FBX.h +++ b/libraries/fbx/src/FBX.h @@ -38,6 +38,7 @@ static const quint32 FBX_VERSION_2016 = 7500; static const int DRACO_BEGIN_CUSTOM_HIFI_ATTRIBUTES = 1000; static const int DRACO_ATTRIBUTE_MATERIAL_ID = DRACO_BEGIN_CUSTOM_HIFI_ATTRIBUTES; static const int DRACO_ATTRIBUTE_TEX_COORD_1 = DRACO_BEGIN_CUSTOM_HIFI_ATTRIBUTES + 1; +static const int DRACO_ATTRIBUTE_ORIGINAL_INDEX = DRACO_BEGIN_CUSTOM_HIFI_ATTRIBUTES + 2; class FBXNode; @@ -217,6 +218,7 @@ public: QVector parts; QVector vertices; + QVector originalIndex; QVector normals; QVector tangents; QVector colors; diff --git a/libraries/fbx/src/FBXReader.h b/libraries/fbx/src/FBXReader.h index 8a5e394da9..843a874a62 100644 --- a/libraries/fbx/src/FBXReader.h +++ b/libraries/fbx/src/FBXReader.h @@ -110,7 +110,7 @@ public: FBXGeometry* extractFBXGeometry(const QVariantHash& mapping, const QString& url); - static ExtractedMesh extractMesh(const FBXNode& object, unsigned int& meshIndex); + static ExtractedMesh extractMesh(const FBXNode& object, unsigned int& meshIndex, bool deduplicate = true); QHash meshes; static void buildModelMesh(FBXMesh& extractedMesh, const QString& url); diff --git a/libraries/fbx/src/FBXReader_Mesh.cpp b/libraries/fbx/src/FBXReader_Mesh.cpp index bef36770a0..3d70fa45ae 100644 --- a/libraries/fbx/src/FBXReader_Mesh.cpp +++ b/libraries/fbx/src/FBXReader_Mesh.cpp @@ -84,7 +84,7 @@ public: }; -void appendIndex(MeshData& data, QVector& indices, int index) { +void appendIndex(MeshData& data, QVector& indices, int index, bool deduplicate) { if (index >= data.polygonIndices.size()) { return; } @@ -156,12 +156,13 @@ void appendIndex(MeshData& data, QVector& indices, int index) { } QHash::const_iterator it = data.indices.find(vertex); - if (it == data.indices.constEnd()) { + if (!deduplicate || it == data.indices.constEnd()) { int newIndex = data.extracted.mesh.vertices.size(); indices.append(newIndex); data.indices.insert(vertex, newIndex); data.extracted.newIndices.insert(vertexIndex, newIndex); data.extracted.mesh.vertices.append(position); + data.extracted.mesh.originalIndices.append(vertexIndex); data.extracted.mesh.normals.append(normal); data.extracted.mesh.texCoords.append(vertex.texCoord); if (hasColors) { @@ -176,7 +177,7 @@ void appendIndex(MeshData& data, QVector& indices, int index) { } } -ExtractedMesh FBXReader::extractMesh(const FBXNode& object, unsigned int& meshIndex) { +ExtractedMesh FBXReader::extractMesh(const FBXNode& object, unsigned int& meshIndex, bool deduplicate) { MeshData data; data.extracted.mesh.meshIndex = meshIndex++; @@ -355,6 +356,7 @@ ExtractedMesh FBXReader::extractMesh(const FBXNode& object, unsigned int& meshIn auto extraTexCoordAttribute = dracoMesh->GetAttributeByUniqueId(DRACO_ATTRIBUTE_TEX_COORD_1); auto colorAttribute = dracoMesh->GetNamedAttribute(draco::GeometryAttribute::COLOR); auto matTexAttribute = dracoMesh->GetAttributeByUniqueId(DRACO_ATTRIBUTE_MATERIAL_ID); + auto originalIndexAttribute = dracoMesh->GetAttributeByUniqueId(DRACO_ATTRIBUTE_ORIGINAL_INDEX); // setup extracted mesh data structures given number of points auto numVertices = dracoMesh->num_points(); @@ -423,7 +425,17 @@ ExtractedMesh FBXReader::extractMesh(const FBXNode& object, unsigned int& meshIn reinterpret_cast(&data.extracted.mesh.colors[i])); } - data.extracted.newIndices.insert(i, i); + if (originalIndexAttribute) { + auto mappedIndex = originalIndexAttribute->mapped_index(vertexIndex); + + int32_t originalIndex; + + originalIndexAttribute->ConvertValue(mappedIndex, &originalIndex); + + data.extracted.newIndices.insert(originalIndex, i); + } else { + data.extracted.newIndices.insert(i, i); + } } for (int i = 0; i < dracoMesh->num_faces(); ++i) { @@ -487,10 +499,10 @@ ExtractedMesh FBXReader::extractMesh(const FBXNode& object, unsigned int& meshIn FBXMeshPart& part = data.extracted.mesh.parts[partIndex - 1]; if (endIndex - beginIndex == 4) { - appendIndex(data, part.quadIndices, beginIndex++); - appendIndex(data, part.quadIndices, beginIndex++); - appendIndex(data, part.quadIndices, beginIndex++); - appendIndex(data, part.quadIndices, beginIndex++); + appendIndex(data, part.quadIndices, beginIndex++, deduplicate); + appendIndex(data, part.quadIndices, beginIndex++, deduplicate); + appendIndex(data, part.quadIndices, beginIndex++, deduplicate); + appendIndex(data, part.quadIndices, beginIndex++, deduplicate); int quadStartIndex = part.quadIndices.size() - 4; int i0 = part.quadIndices[quadStartIndex + 0]; @@ -515,9 +527,9 @@ ExtractedMesh FBXReader::extractMesh(const FBXNode& object, unsigned int& meshIn } else { for (int nextIndex = beginIndex + 1;; ) { - appendIndex(data, part.triangleIndices, beginIndex); - appendIndex(data, part.triangleIndices, nextIndex++); - appendIndex(data, part.triangleIndices, nextIndex); + appendIndex(data, part.triangleIndices, beginIndex, deduplicate); + appendIndex(data, part.triangleIndices, nextIndex++, deduplicate); + appendIndex(data, part.triangleIndices, nextIndex, deduplicate); if (nextIndex >= data.polygonIndices.size() || data.polygonIndices.at(nextIndex) < 0) { break; } diff --git a/libraries/fbx/src/FBXWriter.cpp b/libraries/fbx/src/FBXWriter.cpp index 764d8df6cf..f7bf3517f6 100644 --- a/libraries/fbx/src/FBXWriter.cpp +++ b/libraries/fbx/src/FBXWriter.cpp @@ -59,8 +59,6 @@ QByteArray FBXWriter::encodeFBX(const FBXNode& root) { } void FBXWriter::encodeNode(QDataStream& out, const FBXNode& node) { - qDebug() << "Encoding " << node.name; - auto device = out.device(); auto nodeStartPos = device->pos(); @@ -108,8 +106,12 @@ void FBXWriter::encodeNode(QDataStream& out, const FBXNode& node) { void FBXWriter::encodeFBXProperty(QDataStream& out, const QVariant& prop) { auto type = prop.userType(); switch (type) { - case QVariant::Type::Bool: + case QMetaType::Short: + out.device()->write("Y", 1); + out << prop.value(); + break; + case QVariant::Type::Bool: out.device()->write("C", 1); out << prop.toBool(); break; @@ -229,6 +231,7 @@ void FBXWriter::encodeFBXProperty(QDataStream& out, const QVariant& prop) { writeVector(out, 'b', prop.value>()); } else { qDebug() << "Unsupported property type in FBXWriter::encodeNode: " << type << prop; + throw("Unsupported property type in FBXWriter::encodeNode: " + QString::number(type) + " " + prop.toString()); } } From fd1d4b9bd1e9a518e48e48c0971b3501c82cea66 Mon Sep 17 00:00:00 2001 From: Ryan Huffman Date: Thu, 14 Sep 2017 16:49:53 -0700 Subject: [PATCH 112/129] Update FBXBaker to only pack original indices if needed --- libraries/baking/src/FBXBaker.cpp | 30 +++++++++++++++++++++++++----- 1 file changed, 25 insertions(+), 5 deletions(-) diff --git a/libraries/baking/src/FBXBaker.cpp b/libraries/baking/src/FBXBaker.cpp index 380e893c9b..46f3df7493 100644 --- a/libraries/baking/src/FBXBaker.cpp +++ b/libraries/baking/src/FBXBaker.cpp @@ -292,6 +292,20 @@ QUrl FBXBaker::getTextureURL(const QFileInfo& textureFileInfo, QString relativeF void FBXBaker::rewriteAndBakeSceneModels() { unsigned int meshIndex = 0; + bool hasDeformers { false }; + for (FBXNode& rootChild : _rootNode.children) { + if (rootChild.name == "Objects") { + for (FBXNode& objectChild : rootChild.children) { + if (objectChild.name == "Deformer") { + hasDeformers = true; + break; + } + } + } + if (hasDeformers) { + break; + } + } for (FBXNode& rootChild : _rootNode.children) { if (rootChild.name == "Objects") { for (FBXNode& objectChild : rootChild.children) { @@ -329,18 +343,22 @@ void FBXBaker::rewriteAndBakeSceneModels() { bool hasTexCoords { mesh.texCoords.size() > 0 }; bool hasTexCoords1 { mesh.texCoords1.size() > 0 }; bool hasPerFaceMaterials { mesh.parts.size() > 1 }; + bool needsOriginalIndices { hasDeformers }; int normalsAttributeID { -1 }; int colorsAttributeID { -1 }; int texCoordsAttributeID { -1 }; int texCoords1AttributeID { -1 }; int faceMaterialAttributeID { -1 }; + int originalIndexAttributeID { -1 }; const int positionAttributeID = meshBuilder.AddAttribute(draco::GeometryAttribute::POSITION, 3, draco::DT_FLOAT32); - const int originalIndexAttributeID = meshBuilder.AddAttribute( - (draco::GeometryAttribute::Type)DRACO_ATTRIBUTE_ORIGINAL_INDEX, - 1, draco::DT_INT32); + if (needsOriginalIndices) { + originalIndexAttributeID = meshBuilder.AddAttribute( + (draco::GeometryAttribute::Type)DRACO_ATTRIBUTE_ORIGINAL_INDEX, + 1, draco::DT_INT32); + } if (hasNormals) { normalsAttributeID = meshBuilder.AddAttribute(draco::GeometryAttribute::NORMAL, @@ -385,7 +403,7 @@ void FBXBaker::rewriteAndBakeSceneModels() { &mesh.vertices[idx0], &mesh.vertices[idx1], &mesh.vertices[idx2]); - if (originalIndexAttributeID) { + if (needsOriginalIndices) { meshBuilder.SetAttributeValuesForFace(originalIndexAttributeID, face, &mesh.originalIndex[idx0], &mesh.originalIndex[idx1], @@ -441,7 +459,9 @@ void FBXBaker::rewriteAndBakeSceneModels() { dracoMesh->attribute(texCoords1AttributeID)->set_unique_id(DRACO_ATTRIBUTE_TEX_COORD_1); } - dracoMesh->attribute(originalIndexAttributeID)->set_unique_id(DRACO_ATTRIBUTE_ORIGINAL_INDEX); + if (needsOriginalIndices) { + dracoMesh->attribute(originalIndexAttributeID)->set_unique_id(DRACO_ATTRIBUTE_ORIGINAL_INDEX); + } draco::Encoder encoder; From 8711823a9b320915fcb0ba1ea92b25c700fc6d42 Mon Sep 17 00:00:00 2001 From: Atlante45 Date: Fri, 15 Sep 2017 11:35:59 -0700 Subject: [PATCH 113/129] Sync up AssetServer.qml and TabletAssetServer.qml --- interface/resources/qml/AssetServer.qml | 8 +- .../qml/hifi/dialogs/TabletAssetServer.qml | 190 +++++++++--------- 2 files changed, 99 insertions(+), 99 deletions(-) diff --git a/interface/resources/qml/AssetServer.qml b/interface/resources/qml/AssetServer.qml index 0c4b952331..23fec03ac2 100644 --- a/interface/resources/qml/AssetServer.qml +++ b/interface/resources/qml/AssetServer.qml @@ -72,11 +72,11 @@ ScrollingWindow { function doRenameFile(oldPath, newPath) { - if (newPath[0] != "/") { + if (newPath[0] !== "/") { newPath = "/" + newPath; } - if (oldPath[oldPath.length - 1] == '/' && newPath[newPath.length - 1] != '/') { + if (oldPath[oldPath.length - 1] === '/' && newPath[newPath.length - 1] != '/') { // this is a folder rename but the user neglected to add a trailing slash when providing a new path newPath = newPath + "/"; } @@ -302,7 +302,7 @@ ScrollingWindow { object.selected.connect(function(destinationPath) { destinationPath = destinationPath.trim(); - if (path == destinationPath) { + if (path === destinationPath) { return; } if (fileExists(destinationPath)) { @@ -363,7 +363,7 @@ ScrollingWindow { running: false } - property var uploadOpen: false; + property bool uploadOpen: false; Timer { id: timer } diff --git a/interface/resources/qml/hifi/dialogs/TabletAssetServer.qml b/interface/resources/qml/hifi/dialogs/TabletAssetServer.qml index a7b4ad7a53..250c13c405 100644 --- a/interface/resources/qml/hifi/dialogs/TabletAssetServer.qml +++ b/interface/resources/qml/hifi/dialogs/TabletAssetServer.qml @@ -105,12 +105,12 @@ Rectangle { function askForOverwrite(path, callback) { var object = tabletRoot.messageBox({ - icon: hifi.icons.question, - buttons: OriginalDialogs.StandardButton.Yes | OriginalDialogs.StandardButton.No, - defaultButton: OriginalDialogs.StandardButton.No, - title: "Overwrite File", - text: path + "\n" + "This file already exists. Do you want to overwrite it?" - }); + icon: hifi.icons.question, + buttons: OriginalDialogs.StandardButton.Yes | OriginalDialogs.StandardButton.No, + defaultButton: OriginalDialogs.StandardButton.No, + title: "Overwrite File", + text: path + "\n" + "This file already exists. Do you want to overwrite it?" + }); object.selected.connect(function(button) { if (button === OriginalDialogs.StandardButton.Yes) { callback(); @@ -149,9 +149,9 @@ Rectangle { function handleGetMappingsError(errorString) { errorMessageBox( - "There was a problem retreiving the list of assets from your Asset Server.\n" - + errorString - ); + "There was a problem retreiving the list of assets from your Asset Server.\n" + + errorString + ); } function addToWorld() { @@ -179,25 +179,25 @@ Rectangle { var SHAPE_TYPE_DEFAULT = SHAPE_TYPE_STATIC_MESH; var DYNAMIC_DEFAULT = false; var prompt = tabletRoot.customInputDialog({ - textInput: { - label: "Model URL", - text: defaultURL - }, - comboBox: { - label: "Automatic Collisions", - index: SHAPE_TYPE_DEFAULT, - items: SHAPE_TYPES - }, - checkBox: { - label: "Dynamic", - checked: DYNAMIC_DEFAULT, - disableForItems: [ - SHAPE_TYPE_STATIC_MESH - ], - checkStateOnDisable: false, - warningOnDisable: "Models with 'Exact' automatic collisions cannot be dynamic, and should not be used as floors" - } - }); + textInput: { + label: "Model URL", + text: defaultURL + }, + comboBox: { + label: "Automatic Collisions", + index: SHAPE_TYPE_DEFAULT, + items: SHAPE_TYPES + }, + checkBox: { + label: "Dynamic", + checked: DYNAMIC_DEFAULT, + disableForItems: [ + SHAPE_TYPE_STATIC_MESH + ], + checkStateOnDisable: false, + warningOnDisable: "Models with 'Exact' automatic collisions cannot be dynamic, and should not be used as floors" + } + }); prompt.selected.connect(function (jsonResult) { if (jsonResult) { @@ -205,23 +205,23 @@ Rectangle { var url = result.textInput.trim(); var shapeType; switch (result.comboBox) { - case SHAPE_TYPE_SIMPLE_HULL: - shapeType = "simple-hull"; - break; - case SHAPE_TYPE_SIMPLE_COMPOUND: - shapeType = "simple-compound"; - break; - case SHAPE_TYPE_STATIC_MESH: - shapeType = "static-mesh"; - break; - case SHAPE_TYPE_BOX: - shapeType = "box"; - break; - case SHAPE_TYPE_SPHERE: - shapeType = "sphere"; - break; - default: - shapeType = "none"; + case SHAPE_TYPE_SIMPLE_HULL: + shapeType = "simple-hull"; + break; + case SHAPE_TYPE_SIMPLE_COMPOUND: + shapeType = "simple-compound"; + break; + case SHAPE_TYPE_STATIC_MESH: + shapeType = "static-mesh"; + break; + case SHAPE_TYPE_BOX: + shapeType = "box"; + break; + case SHAPE_TYPE_SPHERE: + shapeType = "sphere"; + break; + default: + shapeType = "none"; } var dynamic = result.checkBox !== null ? result.checkBox : DYNAMIC_DEFAULT; @@ -230,7 +230,7 @@ Rectangle { print("Error: model cannot be both static mesh and dynamic. This should never happen."); } else if (url) { var name = assetProxyModel.data(treeView.selection.currentIndex); - var addPosition = Vec3.sum(MyAvatar.position, Vec3.multiply(2, Quat.getFront(MyAvatar.orientation))); + var addPosition = Vec3.sum(MyAvatar.position, Vec3.multiply(2, Quat.getForward(MyAvatar.orientation))); var gravity; if (dynamic) { // Create a vector <0, -10, 0>. { x: 0, y: -10, z: 0 } won't work because Qt is dumb and this is a @@ -293,10 +293,10 @@ Rectangle { } var object = tabletRoot.inputDialog({ - label: "Enter new path:", - current: path, - placeholderText: "Enter path here" - }); + label: "Enter new path:", + current: path, + placeholderText: "Enter path here" + }); object.selected.connect(function(destinationPath) { destinationPath = destinationPath.trim(); @@ -339,12 +339,12 @@ Rectangle { } var object = tabletRoot.messageBox({ - icon: hifi.icons.question, - buttons: OriginalDialogs.StandardButton.Yes + OriginalDialogs.StandardButton.No, - defaultButton: OriginalDialogs.StandardButton.Yes, - title: "Delete", - text: modalMessage - }); + icon: hifi.icons.question, + buttons: OriginalDialogs.StandardButton.Yes + OriginalDialogs.StandardButton.No, + defaultButton: OriginalDialogs.StandardButton.Yes, + title: "Delete", + text: modalMessage + }); object.selected.connect(function(button) { if (button === OriginalDialogs.StandardButton.Yes) { doDeleteFile(path); @@ -379,38 +379,38 @@ Rectangle { var filename = fileUrl.slice(fileUrl.lastIndexOf('/') + 1); Assets.uploadFile(fileUrl, directory + filename, - function() { - // Upload started - uploadSpinner.visible = true; - uploadButton.enabled = false; - uploadProgressLabel.text = "In progress..."; - }, - function(err, path) { - print(err, path); - if (err === "") { - uploadProgressLabel.text = "Upload Complete"; - timer.interval = 1000; - timer.repeat = false; - timer.triggered.connect(function() { - uploadSpinner.visible = false; - uploadButton.enabled = true; - uploadOpen = false; - }); - timer.start(); - console.log("Asset Browser - finished uploading: ", fileUrl); - reload(); - } else { - uploadSpinner.visible = false; - uploadButton.enabled = true; - uploadOpen = false; + function() { + // Upload started + uploadSpinner.visible = true; + uploadButton.enabled = false; + uploadProgressLabel.text = "In progress..."; + }, + function(err, path) { + print(err, path); + if (err === "") { + uploadProgressLabel.text = "Upload Complete"; + timer.interval = 1000; + timer.repeat = false; + timer.triggered.connect(function() { + uploadSpinner.visible = false; + uploadButton.enabled = true; + uploadOpen = false; + }); + timer.start(); + console.log("Asset Browser - finished uploading: ", fileUrl); + reload(); + } else { + uploadSpinner.visible = false; + uploadButton.enabled = true; + uploadOpen = false; - if (err !== -1) { - console.log("Asset Browser - error uploading: ", fileUrl, " - error ", err); - var box = errorMessageBox("There was an error uploading:\n" + fileUrl + "\n" + err); - box.selected.connect(reload); - } - } - }, dropping); + if (err !== -1) { + console.log("Asset Browser - error uploading: ", fileUrl, " - error ", err); + var box = errorMessageBox("There was an error uploading:\n" + fileUrl + "\n" + err); + box.selected.connect(reload); + } + } + }, dropping); } function initiateUpload(url) { @@ -421,9 +421,9 @@ Rectangle { doUpload(fileUrl, true); } else { var browser = tabletRoot.fileDialog({ - selectDirectory: false, - dir: currentDirectory - }); + selectDirectory: false, + dir: currentDirectory + }); browser.canceled.connect(function() { uploadOpen = false; @@ -445,11 +445,11 @@ Rectangle { function errorMessageBox(message) { return tabletRoot.messageBox({ - icon: hifi.icons.warning, - defaultButton: OriginalDialogs.StandardButton.Ok, - title: "Error", - text: message - }); + icon: hifi.icons.warning, + defaultButton: OriginalDialogs.StandardButton.Ok, + title: "Error", + text: message + }); } Column { From c4d018504bf28de8e37207da57ff0cc6ffca06da Mon Sep 17 00:00:00 2001 From: Atlante45 Date: Fri, 15 Sep 2017 11:46:28 -0700 Subject: [PATCH 114/129] Pull Auto-Baking changes to TabletAssetServer.qml --- .../qml/hifi/dialogs/TabletAssetServer.qml | 253 +++++++++++++++--- 1 file changed, 221 insertions(+), 32 deletions(-) diff --git a/interface/resources/qml/hifi/dialogs/TabletAssetServer.qml b/interface/resources/qml/hifi/dialogs/TabletAssetServer.qml index 250c13c405..cb2635a2ab 100644 --- a/interface/resources/qml/hifi/dialogs/TabletAssetServer.qml +++ b/interface/resources/qml/hifi/dialogs/TabletAssetServer.qml @@ -10,6 +10,7 @@ import QtQuick 2.5 import QtQuick.Controls 1.4 +import QtQuick.Controls.Styles 1.4 import QtQuick.Dialogs 1.2 as OriginalDialogs import Qt.labs.settings 1.0 @@ -469,15 +470,6 @@ Rectangle { height: 30 spacing: hifi.dimensions.contentSpacing.x - HifiControls.GlyphButton { - glyph: hifi.glyphs.reload - color: hifi.buttons.black - colorScheme: root.colorScheme - width: hifi.dimensions.controlLineHeight - - onClicked: root.reload() - } - HifiControls.Button { text: "Add To World" color: hifi.buttons.black @@ -510,8 +502,193 @@ Rectangle { onClicked: root.deleteFile() enabled: treeView.selection.hasSelection } + + HifiControls.GlyphButton { + + glyph: hifi.glyphs.reload + color: hifi.buttons.black + colorScheme: root.colorScheme + width: hifi.dimensions.controlLineHeight + + onClicked: root.reload() + } + } + } + + HifiControls.Tree { + id: treeView + anchors.top: assetDirectory.bottom + anchors.bottom: infoRow.top + anchors.margins: hifi.dimensions.contentMargin.x + 2 // Extra for border + anchors.left: parent.left + anchors.right: parent.right + + treeModel: assetProxyModel + selectionMode: SelectionMode.ExtendedSelection + headerVisible: true + sortIndicatorVisible: true + + colorScheme: root.colorScheme + + modifyEl: renameEl + + TableViewColumn { + id: nameColumn + title: "Name:" + role: "name" + width: treeView.width - bakedColumn.width; + } + TableViewColumn { + id: bakedColumn + title: "Use Baked?" + role: "baked" + width: 100 + } + + itemDelegate: Loader { + id: itemDelegateLoader + + anchors { + left: parent ? parent.left : undefined + leftMargin: (styleData.column === 0 ? (2 + styleData.depth) : 1) * hifi.dimensions.tablePadding + right: parent ? parent.right : undefined + rightMargin: hifi.dimensions.tablePadding + verticalCenter: parent ? parent.verticalCenter : undefined + } + + function convertToGlyph(text) { + switch (text) { + case "Not Baked": + return hifi.glyphs.circleSlash; + case "Baked": + return hifi.glyphs.check_2_01; + case "Error": + return hifi.glyphs.alert; + default: + return ""; + } + } + + function getComponent() { + if ((styleData.column === 0) && styleData.selected) { + return textFieldComponent; + } else if (convertToGlyph(styleData.value) != "") { + return glyphComponent; + } else { + return labelComponent; + } + + } + sourceComponent: getComponent() + + Component { + id: labelComponent + FiraSansSemiBold { + text: styleData.value + size: hifi.fontSizes.tableText + color: colorScheme == hifi.colorSchemes.light + ? (styleData.selected ? hifi.colors.black : hifi.colors.baseGrayHighlight) + : (styleData.selected ? hifi.colors.black : hifi.colors.lightGrayText) + + elide: Text.ElideRight + horizontalAlignment: styleData.column === 1 ? TextInput.AlignHCenter : TextInput.AlignLeft + } + } + Component { + id: glyphComponent + + HiFiGlyphs { + text: convertToGlyph(styleData.value) + size: hifi.dimensions.frameIconSize + color: colorScheme == hifi.colorSchemes.light + ? (styleData.selected ? hifi.colors.black : hifi.colors.baseGrayHighlight) + : (styleData.selected ? hifi.colors.black : hifi.colors.lightGrayText) + + elide: Text.ElideRight + horizontalAlignment: TextInput.AlignHCenter + + HifiControls.ToolTip { + anchors.fill: parent + + visible: styleData.value === "Error" + + toolTip: assetProxyModel.data(styleData.index, 0x106) + } + } + } + Component { + id: textFieldComponent + + TextField { + id: textField + readOnly: !activeFocus + + text: styleData.value + + FontLoader { id: firaSansSemiBold; source: "../../fonts/FiraSans-SemiBold.ttf"; } + font.family: firaSansSemiBold.name + font.pixelSize: hifi.fontSizes.textFieldInput + height: hifi.dimensions.tableRowHeight + + style: TextFieldStyle { + textColor: readOnly + ? hifi.colors.black + : (treeView.isLightColorScheme ? hifi.colors.black : hifi.colors.white) + background: Rectangle { + visible: !readOnly + color: treeView.isLightColorScheme ? hifi.colors.white : hifi.colors.black + border.color: hifi.colors.primaryHighlight + border.width: 1 + } + selectedTextColor: hifi.colors.black + selectionColor: hifi.colors.primaryHighlight + padding.left: readOnly ? 0 : hifi.dimensions.textPadding + padding.right: readOnly ? 0 : hifi.dimensions.textPadding + } + + validator: RegExpValidator { + regExp: /[^/]+/ + } + + Keys.onPressed: { + if (event.key == Qt.Key_Escape) { + text = styleData.value; + unfocusHelper.forceActiveFocus(); + event.accepted = true; + } + } + onAccepted: { + if (acceptableInput && styleData.selected) { + if (!treeView.modifyEl(treeView.selection.currentIndex, text)) { + text = styleData.value; + } + unfocusHelper.forceActiveFocus(); + } + } + + onReadOnlyChanged: { + // Have to explicily set keyboardRaised because automatic setting fails because readOnly is true at the time. + keyboardRaised = activeFocus; + } + } + } } + + MouseArea { + propagateComposedEvents: true + anchors.fill: parent + acceptedButtons: Qt.RightButton + onClicked: { + if (!HMD.active) { // Popup only displays properly on desktop + var index = treeView.indexAt(mouse.x, mouse.y); + treeView.selection.setCurrentIndex(index, 0x0002); + contextMenu.currentIndex = index; + contextMenu.popup(); + } + } + } + Menu { id: contextMenu title: "Edit" @@ -539,39 +716,51 @@ Rectangle { } } } - } - HifiControls.Tree { - id: treeView - height: 290 - anchors.leftMargin: hifi.dimensions.contentMargin.x + 2 // Extra for border - anchors.rightMargin: hifi.dimensions.contentMargin.x + 2 // Extra for border - anchors.left: parent.left - anchors.right: parent.right - treeModel: assetProxyModel - canEdit: true - colorScheme: root.colorScheme - selectionMode: SelectionMode.ExtendedSelection + Row { + id: infoRow + anchors.left: treeView.left + anchors.right: treeView.right + anchors.bottom: uploadSection.top + anchors.bottomMargin: hifi.dimensions.contentSpacing.y + spacing: hifi.dimensions.contentSpacing.x + + RalewayRegular { + size: hifi.fontSizes.sectionName + font.capitalization: Font.AllUppercase + text: selectedItems + " items selected" + color: hifi.colors.lightGrayText + } - modifyEl: renameEl + HifiControls.CheckBox { + function isChecked() { + var status = assetProxyModel.data(treeView.selection.currentIndex, 0x105); + var bakingDisabled = (status === "Not Baked" || status === "--"); + return selectedItems === 1 && !bakingDisabled; + } - MouseArea { - propagateComposedEvents: true - anchors.fill: parent - acceptedButtons: Qt.RightButton + text: "Use baked (optimized) versions" + colorScheme: root.colorScheme + enabled: selectedItems === 1 && assetProxyModel.data(treeView.selection.currentIndex, 0x105) !== "--" + checked: isChecked() onClicked: { - if (!HMD.active) { // Popup only displays properly on desktop - var index = treeView.indexAt(mouse.x, mouse.y); - treeView.selection.setCurrentIndex(index, 0x0002); - contextMenu.currentIndex = index; - contextMenu.popup(); + var mappings = []; + for (var i in treeView.selection.selectedIndexes) { + var index = treeView.selection.selectedIndexes[i]; + var path = assetProxyModel.data(index, 0x100); + mappings.push(path); } + print("Setting baking enabled:" + mappings + checked); + Assets.setBakingEnabled(mappings, checked, function() { + reload(); + }); + + checked = Qt.binding(isChecked); } } } - HifiControls.TabletContentSection { id: uploadSection name: "Upload A File" From c5852dfbe81ca620c0bdf6c2b8178e91241ba3f2 Mon Sep 17 00:00:00 2001 From: Ryan Huffman Date: Fri, 15 Sep 2017 11:43:06 -0700 Subject: [PATCH 115/129] Add list compression to FBXWriter --- libraries/baking/src/FBXBaker.cpp | 6 ++--- libraries/fbx/src/FBX.h | 6 ++--- libraries/fbx/src/FBXReader_Mesh.cpp | 8 +++---- libraries/fbx/src/FBXReader_Node.cpp | 6 ++--- libraries/fbx/src/FBXWriter.cpp | 33 +++++++++++++++++++++++----- libraries/fbx/src/FBXWriter.h | 2 +- 6 files changed, 40 insertions(+), 21 deletions(-) diff --git a/libraries/baking/src/FBXBaker.cpp b/libraries/baking/src/FBXBaker.cpp index 46f3df7493..aa05a135fd 100644 --- a/libraries/baking/src/FBXBaker.cpp +++ b/libraries/baking/src/FBXBaker.cpp @@ -405,9 +405,9 @@ void FBXBaker::rewriteAndBakeSceneModels() { if (needsOriginalIndices) { meshBuilder.SetAttributeValuesForFace(originalIndexAttributeID, face, - &mesh.originalIndex[idx0], - &mesh.originalIndex[idx1], - &mesh.originalIndex[idx2]); + &mesh.originalIndices[idx0], + &mesh.originalIndices[idx1], + &mesh.originalIndices[idx2]); } if (hasNormals) { meshBuilder.SetAttributeValuesForFace(normalsAttributeID, face, diff --git a/libraries/fbx/src/FBX.h b/libraries/fbx/src/FBX.h index 78d35889cf..836195f125 100644 --- a/libraries/fbx/src/FBX.h +++ b/libraries/fbx/src/FBX.h @@ -33,13 +33,13 @@ static const int FBX_HEADER_BYTES_BEFORE_VERSION = 23; static const quint32 FBX_VERSION_2015 = 7400; static const quint32 FBX_VERSION_2016 = 7500; - -// TODO Convert to GeometryAttribute type static const int DRACO_BEGIN_CUSTOM_HIFI_ATTRIBUTES = 1000; static const int DRACO_ATTRIBUTE_MATERIAL_ID = DRACO_BEGIN_CUSTOM_HIFI_ATTRIBUTES; static const int DRACO_ATTRIBUTE_TEX_COORD_1 = DRACO_BEGIN_CUSTOM_HIFI_ATTRIBUTES + 1; static const int DRACO_ATTRIBUTE_ORIGINAL_INDEX = DRACO_BEGIN_CUSTOM_HIFI_ATTRIBUTES + 2; +static const int32_t FBX_PROPERTY_UNCOMPRESSED_FLAG = 0; +static const int32_t FBX_PROPERTY_COMPRESSED_FLAG = 1; class FBXNode; using FBXNodeList = QList; @@ -218,7 +218,6 @@ public: QVector parts; QVector vertices; - QVector originalIndex; QVector normals; QVector tangents; QVector colors; @@ -226,6 +225,7 @@ public: QVector texCoords1; QVector clusterIndices; QVector clusterWeights; + QVector originalIndices; QVector clusters; diff --git a/libraries/fbx/src/FBXReader_Mesh.cpp b/libraries/fbx/src/FBXReader_Mesh.cpp index 3d70fa45ae..d4b6dde5fe 100644 --- a/libraries/fbx/src/FBXReader_Mesh.cpp +++ b/libraries/fbx/src/FBXReader_Mesh.cpp @@ -355,7 +355,7 @@ ExtractedMesh FBXReader::extractMesh(const FBXNode& object, unsigned int& meshIn auto texCoordAttribute = dracoMesh->GetNamedAttribute(draco::GeometryAttribute::TEX_COORD); auto extraTexCoordAttribute = dracoMesh->GetAttributeByUniqueId(DRACO_ATTRIBUTE_TEX_COORD_1); auto colorAttribute = dracoMesh->GetNamedAttribute(draco::GeometryAttribute::COLOR); - auto matTexAttribute = dracoMesh->GetAttributeByUniqueId(DRACO_ATTRIBUTE_MATERIAL_ID); + auto materialIDAttribute = dracoMesh->GetAttributeByUniqueId(DRACO_ATTRIBUTE_MATERIAL_ID); auto originalIndexAttribute = dracoMesh->GetAttributeByUniqueId(DRACO_ATTRIBUTE_ORIGINAL_INDEX); // setup extracted mesh data structures given number of points @@ -445,11 +445,11 @@ ExtractedMesh FBXReader::extractMesh(const FBXNode& object, unsigned int& meshIn uint16_t materialID { 0 }; - if (matTexAttribute) { + if (materialIDAttribute) { // read material ID and texture ID mappings into materials and texture vectors - auto mappedIndex = matTexAttribute->mapped_index(firstCorner); + auto mappedIndex = materialIDAttribute->mapped_index(firstCorner); - matTexAttribute->ConvertValue(mappedIndex, &materialID); + materialIDAttribute->ConvertValue(mappedIndex, &materialID); } QPair materialTexture(materialID, 0); diff --git a/libraries/fbx/src/FBXReader_Node.cpp b/libraries/fbx/src/FBXReader_Node.cpp index 111b4a295a..c4454421b6 100644 --- a/libraries/fbx/src/FBXReader_Node.cpp +++ b/libraries/fbx/src/FBXReader_Node.cpp @@ -48,9 +48,8 @@ QVariant readBinaryArray(QDataStream& in, int& position) { QVector values; if ((int)QSysInfo::ByteOrder == (int)in.byteOrder()) { values.resize(arrayLength); - const unsigned int DEFLATE_ENCODING = 1; QByteArray arrayData; - if (encoding == DEFLATE_ENCODING) { + if (encoding == FBX_PROPERTY_COMPRESSED_FLAG) { // preface encoded data with uncompressed length QByteArray compressed(sizeof(quint32) + compressedLength, 0); *((quint32*)compressed.data()) = qToBigEndian(arrayLength * sizeof(T)); @@ -72,8 +71,7 @@ QVariant readBinaryArray(QDataStream& in, int& position) { } } else { values.reserve(arrayLength); - const unsigned int DEFLATE_ENCODING = 1; - if (encoding == DEFLATE_ENCODING) { + if (encoding == FBX_PROPERTY_COMPRESSED_FLAG) { // preface encoded data with uncompressed length QByteArray compressed(sizeof(quint32) + compressedLength, 0); *((quint32*)compressed.data()) = qToBigEndian(arrayLength * sizeof(T)); diff --git a/libraries/fbx/src/FBXWriter.cpp b/libraries/fbx/src/FBXWriter.cpp index f7bf3517f6..bc0013b9ac 100644 --- a/libraries/fbx/src/FBXWriter.cpp +++ b/libraries/fbx/src/FBXWriter.cpp @@ -24,13 +24,34 @@ #endif template -void writeVector(QDataStream& out, char ch, QVector list) { - out.device()->write(&ch, 1); - out << (int32_t)list.length(); - out << (int32_t)0; - out << (int32_t)0; +void writeVector(QDataStream& out, char ch, QVector vec) { + // Minimum number of bytes to consider compressing + const int ATTEMPT_COMPRESSION_THRESHOLD_BYTES = 2000; - out.writeBytes(reinterpret_cast(list.constData()), list.length() * sizeof(T)); + out.device()->write(&ch, 1); + out << (int32_t)vec.length(); + + auto data { QByteArray::fromRawData((const char*)vec.constData(), vec.length() * sizeof(T)) }; + + if (data.size() >= ATTEMPT_COMPRESSION_THRESHOLD_BYTES) { + auto compressedDataWithLength { qCompress(data) }; + + // qCompress packs a length uint32 at the beginning of the buffer, but the FBX format + // does not expect it. This removes it. + auto compressedData = QByteArray::fromRawData( + compressedDataWithLength.constData() + sizeof(uint32_t), compressedDataWithLength.size() - sizeof(uint32_t)); + + if (compressedData.size() < data.size()) { + out << (int32_t)1; + out << (int32_t)compressedData.size(); + out.writeRawData(compressedData.constData(), compressedData.size()); + return; + } + } + + out << FBX_PROPERTY_UNCOMPRESSED_FLAG; + out << (int32_t)0; + out.writeRawData(data.constData(), data.size()); } diff --git a/libraries/fbx/src/FBXWriter.h b/libraries/fbx/src/FBXWriter.h index 57c33e0e1c..f20d208cb1 100644 --- a/libraries/fbx/src/FBXWriter.h +++ b/libraries/fbx/src/FBXWriter.h @@ -17,7 +17,7 @@ #include #include -#define USE_FBX_2016_FORMAT +//#define USE_FBX_2016_FORMAT class FBXWriter { public: From 2a4f7712c32988ada4b5862f442199fefd837452 Mon Sep 17 00:00:00 2001 From: Ryan Huffman Date: Fri, 15 Sep 2017 14:41:38 -0700 Subject: [PATCH 116/129] Replace magic number for compressed flag in FBXWriter --- libraries/fbx/src/FBXWriter.cpp | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/libraries/fbx/src/FBXWriter.cpp b/libraries/fbx/src/FBXWriter.cpp index bc0013b9ac..900fddc0a2 100644 --- a/libraries/fbx/src/FBXWriter.cpp +++ b/libraries/fbx/src/FBXWriter.cpp @@ -42,7 +42,7 @@ void writeVector(QDataStream& out, char ch, QVector vec) { compressedDataWithLength.constData() + sizeof(uint32_t), compressedDataWithLength.size() - sizeof(uint32_t)); if (compressedData.size() < data.size()) { - out << (int32_t)1; + out << FBX_PROPERTY_COMPRESSED_FLAG; out << (int32_t)compressedData.size(); out.writeRawData(compressedData.constData(), compressedData.size()); return; From a37642ba44b10701c404e39176f942a9362320c1 Mon Sep 17 00:00:00 2001 From: Atlante45 Date: Fri, 15 Sep 2017 11:46:28 -0700 Subject: [PATCH 117/129] Pull Auto-Baking changes to TabletAssetServer.qml --- interface/resources/qml/hifi/dialogs/TabletAssetServer.qml | 3 --- 1 file changed, 3 deletions(-) diff --git a/interface/resources/qml/hifi/dialogs/TabletAssetServer.qml b/interface/resources/qml/hifi/dialogs/TabletAssetServer.qml index cb2635a2ab..61f5a382ad 100644 --- a/interface/resources/qml/hifi/dialogs/TabletAssetServer.qml +++ b/interface/resources/qml/hifi/dialogs/TabletAssetServer.qml @@ -517,8 +517,6 @@ Rectangle { HifiControls.Tree { id: treeView - anchors.top: assetDirectory.bottom - anchors.bottom: infoRow.top anchors.margins: hifi.dimensions.contentMargin.x + 2 // Extra for border anchors.left: parent.left anchors.right: parent.right @@ -722,7 +720,6 @@ Rectangle { id: infoRow anchors.left: treeView.left anchors.right: treeView.right - anchors.bottom: uploadSection.top anchors.bottomMargin: hifi.dimensions.contentSpacing.y spacing: hifi.dimensions.contentSpacing.x From d80aa50e79332899e3cb7f3a8837eeee39322173 Mon Sep 17 00:00:00 2001 From: Ryan Huffman Date: Mon, 18 Sep 2017 09:03:57 -0700 Subject: [PATCH 118/129] Remove support for QList in FBXWriter --- libraries/fbx/src/FBXWriter.cpp | 60 --------------------------------- 1 file changed, 60 deletions(-) diff --git a/libraries/fbx/src/FBXWriter.cpp b/libraries/fbx/src/FBXWriter.cpp index 900fddc0a2..5029b489bc 100644 --- a/libraries/fbx/src/FBXWriter.cpp +++ b/libraries/fbx/src/FBXWriter.cpp @@ -178,66 +178,6 @@ void FBXWriter::encodeFBXProperty(QDataStream& out, const QVariant& prop) { break; } - // TODO Delete? Do we ever use QList instead of QVector? - case QVariant::Type::List: - { - auto list = prop.toList(); - auto listType = prop.userType(); - - switch (listType) { - case QMetaType::Float: - out.device()->write("f", 1); - out << (int32_t)list.length(); - out << (int32_t)0; - out << (int32_t)0; - for (auto& innerProp : list) { - out << innerProp.toFloat(); - } - break; - - case QMetaType::Double: - out.device()->write("d", 1); - out << (int32_t)list.length(); - out << (int32_t)0; - out << (int32_t)0; - for (auto& innerProp : list) { - out << innerProp.toDouble(); - } - break; - - case QMetaType::LongLong: - out.device()->write("l", 1); - out << (int32_t)list.length(); - out << (int32_t)0; - out << (int32_t)0; - for (auto& innerProp : list) { - out << innerProp.toLongLong(); - } - break; - - case QMetaType::Int: - out.device()->write("i", 1); - out << (int32_t)list.length(); - out << (int32_t)0; - out << (int32_t)0; - for (auto& innerProp : list) { - out << innerProp.toInt(); - } - break; - - case QMetaType::Bool: - out.device()->write("b", 1); - out << (int32_t)list.length(); - out << (int32_t)0; - out << (int32_t)0; - for (auto& innerProp : list) { - out << innerProp.toBool(); - } - break; - } - } - break; - default: { if (prop.canConvert>()) { From 02a529923ed4bed757a262a39a85c0f5c8797dd9 Mon Sep 17 00:00:00 2001 From: Stephen Birarda Date: Mon, 18 Sep 2017 14:00:03 -0700 Subject: [PATCH 119/129] don't export embedded textures, remove triangle warning --- libraries/baking/src/FBXBaker.cpp | 60 ++++++++++++++++--------------- libraries/baking/src/FBXBaker.h | 2 +- 2 files changed, 32 insertions(+), 30 deletions(-) diff --git a/libraries/baking/src/FBXBaker.cpp b/libraries/baking/src/FBXBaker.cpp index 3a37a3f31e..120ea3fa80 100644 --- a/libraries/baking/src/FBXBaker.cpp +++ b/libraries/baking/src/FBXBaker.cpp @@ -244,7 +244,7 @@ QString texturePathRelativeToFBX(QUrl fbxURL, QUrl textureURL) { QString FBXBaker::createBakedTextureFileName(const QFileInfo& textureFileInfo) { // first make sure we have a unique base name for this texture // in case another texture referenced by this model has the same base name - auto nameMatches = _textureNameMatchCount[textureFileInfo.baseName()]; + auto& nameMatches = _textureNameMatchCount[textureFileInfo.baseName()]; QString bakedTextureFileName { textureFileInfo.completeBaseName() }; @@ -262,28 +262,32 @@ QString FBXBaker::createBakedTextureFileName(const QFileInfo& textureFileInfo) { return bakedTextureFileName; } -QUrl FBXBaker::getTextureURL(const QFileInfo& textureFileInfo, QString relativeFileName) { +QUrl FBXBaker::getTextureURL(const QFileInfo& textureFileInfo, QString relativeFileName, bool isEmbedded) { + QUrl urlToTexture; - if (textureFileInfo.exists() && textureFileInfo.isFile()) { - // set the texture URL to the local texture that we have confirmed exists - urlToTexture = QUrl::fromLocalFile(textureFileInfo.absoluteFilePath()); - } else { - // external texture that we'll need to download or find + auto apparentRelativePath = QFileInfo(relativeFileName.replace("\\", "/")); - // first check if it the RelativePath to the texture in the FBX was relative - auto apparentRelativePath = QFileInfo(relativeFileName.replace("\\", "/")); - - // this is a relative file path which will require different handling - // depending on the location of the original FBX - if (_fbxURL.isLocalFile() && apparentRelativePath.exists() && apparentRelativePath.isFile()) { - // the absolute path we ran into for the texture in the FBX exists on this machine - // so use that file - urlToTexture = QUrl::fromLocalFile(apparentRelativePath.absoluteFilePath()); + if (isEmbedded) { + urlToTexture = _fbxURL.toString() + "/" + apparentRelativePath.filePath(); + } else { + if (textureFileInfo.exists() && textureFileInfo.isFile()) { + // set the texture URL to the local texture that we have confirmed exists + urlToTexture = QUrl::fromLocalFile(textureFileInfo.absoluteFilePath()); } else { - // we didn't find the texture on this machine at the absolute path - // so assume that it is right beside the FBX to match the behaviour of interface - urlToTexture = _fbxURL.resolved(apparentRelativePath.fileName()); + // external texture that we'll need to download or find + + // this is a relative file path which will require different handling + // depending on the location of the original FBX + if (_fbxURL.isLocalFile() && apparentRelativePath.exists() && apparentRelativePath.isFile()) { + // the absolute path we ran into for the texture in the FBX exists on this machine + // so use that file + urlToTexture = QUrl::fromLocalFile(apparentRelativePath.absoluteFilePath()); + } else { + // we didn't find the texture on this machine at the absolute path + // so assume that it is right beside the FBX to match the behaviour of interface + urlToTexture = _fbxURL.resolved(apparentRelativePath.fileName()); + } } } @@ -335,7 +339,6 @@ void FBXBaker::rewriteAndBakeSceneModels() { } if (numTriangles == 0) { - handleWarning("Skipping compression of mesh because no triangles were found"); continue; } @@ -582,8 +585,12 @@ void FBXBaker::rewriteAndBakeSceneTextures() { qCDebug(model_baking).noquote() << "Re-mapping" << fbxTextureFileName << "to" << bakedTextureFileName; + // check if this was an embedded texture we have already have in-memory content for + auto textureContent = _textureContent.value(fbxTextureFileName.toLocal8Bit()); + // figure out the URL to this texture, embedded or external - auto urlToTexture = getTextureURL(textureFileInfo, fbxTextureFileName); + auto urlToTexture = getTextureURL(textureFileInfo, fbxTextureFileName, + !textureContent.isNull()); // write the new filename into the FBX scene textureChild.properties[0] = bakedTextureFileName.toLocal8Bit(); @@ -595,9 +602,6 @@ void FBXBaker::rewriteAndBakeSceneTextures() { QString textureID { object->properties[0].toByteArray() }; auto textureType = textureTypes[textureID]; - // check if this was an embedded texture we have already have in-memory content for - auto textureContent = _textureContent.value(fbxTextureFileName.toLocal8Bit()); - // bake this texture asynchronously bakeTexture(urlToTexture, textureType, _bakedOutputDir, textureContent); } @@ -649,12 +653,10 @@ void FBXBaker::handleBakedTexture() { // use the path to the texture being baked to determine if this was an embedded or a linked texture - // it is embeddded if the texure being baked was inside the original output folder - // since that is where the FBX SDK places the .fbm folder it generates when importing the FBX + // it is embeddded if the texure being baked was inside a folder with the name of the FBX + // since that is the fake URL we provide when baking external textures - auto originalOutputFolder = QUrl::fromLocalFile(_originalOutputDir); - - if (!originalOutputFolder.isParentOf(bakedTexture->getTextureURL())) { + if (!_fbxURL.isParentOf(bakedTexture->getTextureURL())) { // for linked textures we want to save a copy of original texture beside the original FBX qCDebug(model_baking) << "Saving original texture for" << bakedTexture->getTextureURL(); diff --git a/libraries/baking/src/FBXBaker.h b/libraries/baking/src/FBXBaker.h index 26c1ff2dcc..c611fb712f 100644 --- a/libraries/baking/src/FBXBaker.h +++ b/libraries/baking/src/FBXBaker.h @@ -66,7 +66,7 @@ private: void checkIfTexturesFinished(); QString createBakedTextureFileName(const QFileInfo& textureFileInfo); - QUrl getTextureURL(const QFileInfo& textureFileInfo, QString relativeFileName); + QUrl getTextureURL(const QFileInfo& textureFileInfo, QString relativeFileName, bool isEmbedded = false); void bakeTexture(const QUrl& textureURL, image::TextureUsage::Type textureType, const QDir& outputDir, const QByteArray& textureContent = QByteArray()); From 608f8196c62cde260d052d7ba7f3a33178538740 Mon Sep 17 00:00:00 2001 From: Stephen Birarda Date: Mon, 18 Sep 2017 14:01:48 -0700 Subject: [PATCH 120/129] remove commented out removal of embedded media folder --- libraries/baking/src/FBXBaker.cpp | 9 --------- 1 file changed, 9 deletions(-) diff --git a/libraries/baking/src/FBXBaker.cpp b/libraries/baking/src/FBXBaker.cpp index 120ea3fa80..0f31b878ee 100644 --- a/libraries/baking/src/FBXBaker.cpp +++ b/libraries/baking/src/FBXBaker.cpp @@ -736,20 +736,11 @@ void FBXBaker::exportScene() { } -void FBXBaker::removeEmbeddedMediaFolder() { - // now that the bake is complete, remove the embedded media folder produced by the FBX SDK when it imports an FBX - //auto embeddedMediaFolderName = _fbxURL.fileName().replace(".fbx", ".fbm"); - //QDir(_bakedOutputDir + ORIGINAL_OUTPUT_SUBFOLDER + embeddedMediaFolderName).removeRecursively(); -} - void FBXBaker::checkIfTexturesFinished() { // check if we're done everything we need to do for this FBX // and emit our finished signal if we're done if (_bakingTextures.isEmpty()) { - // remove the embedded media folder that the FBX SDK produces when reading the original - removeEmbeddedMediaFolder(); - if (hasErrors()) { // if we're checking for completion but we have errors // that means one or more of our texture baking operations failed From b9e5957c9f350ad74c47f59fbb7362ace2b54f87 Mon Sep 17 00:00:00 2001 From: Atlante45 Date: Mon, 18 Sep 2017 16:14:06 -0700 Subject: [PATCH 121/129] Remove unused Algorithms.h --- libraries/shared/src/shared/Algorithms.h | 38 ------------------------ 1 file changed, 38 deletions(-) delete mode 100644 libraries/shared/src/shared/Algorithms.h diff --git a/libraries/shared/src/shared/Algorithms.h b/libraries/shared/src/shared/Algorithms.h deleted file mode 100644 index b85a3df6c5..0000000000 --- a/libraries/shared/src/shared/Algorithms.h +++ /dev/null @@ -1,38 +0,0 @@ -// -// Algorithms.h -// libraries/shared/src/shared -// -// Created by Clement Brisset on 8/9/17 -// Copyright 2017 High Fidelity, Inc. -// -// Distributed under the Apache License, Version 2.0. -// See the accompanying file LICENSE or http://www.apache.org/licenses/LICENSE-2.0.html -// - -#ifndef hifi_Algorithms_h -#define hifi_Algorithms_h - -#include -#include -#include - -namespace alg { - -template -auto find(const Container& container, const ValueType& value) -> decltype(std::begin(container)) { - return std::find(std::begin(container), std::end(container), value); -} - -template -auto find_if(const Container& container, Predicate&& predicate) -> decltype(std::begin(container)) { - return std::find_if(std::begin(container), std::end(container), std::forward(predicate)); -} - -template -auto find_if_not(const Container& container, Predicate&& predicate) -> decltype(std::begin(container)) { - return std::find_if_not(std::begin(container), std::end(container), std::forward(predicate)); -} - -} - -#endif // hifi_Algorithms_hpp From 641addf39772a7c41bd06b0ce535453eb547a446 Mon Sep 17 00:00:00 2001 From: Ryan Huffman Date: Tue, 19 Sep 2017 11:13:39 -0700 Subject: [PATCH 122/129] Remove FBX.cpp --- libraries/fbx/src/FBX.cpp | 10 ---------- 1 file changed, 10 deletions(-) delete mode 100644 libraries/fbx/src/FBX.cpp diff --git a/libraries/fbx/src/FBX.cpp b/libraries/fbx/src/FBX.cpp deleted file mode 100644 index 3f18b3e678..0000000000 --- a/libraries/fbx/src/FBX.cpp +++ /dev/null @@ -1,10 +0,0 @@ -// -// FBX.cpp -// libraries/fbx/src -// -// Created by Ryan Huffman on 9/5/17. -// Copyright 2017 High Fidelity, Inc. -// -// Distributed under the Apache License, Version 2.0. -// See the accompanying file LICENSE or http://www.apache.org/licenses/LICENSE-2.0.html -// From 59500061116f856b3b8c6a45bd01ec0a6dc35368 Mon Sep 17 00:00:00 2001 From: Ryan Huffman Date: Tue, 19 Sep 2017 11:16:59 -0700 Subject: [PATCH 123/129] Remove extraneous logging --- .../model-networking/src/model-networking/TextureCache.cpp | 1 - libraries/networking/src/AssetClient.cpp | 1 - libraries/networking/src/ResourceCache.cpp | 2 -- 3 files changed, 4 deletions(-) diff --git a/libraries/model-networking/src/model-networking/TextureCache.cpp b/libraries/model-networking/src/model-networking/TextureCache.cpp index cc29e841ce..e0d00ecd08 100644 --- a/libraries/model-networking/src/model-networking/TextureCache.cpp +++ b/libraries/model-networking/src/model-networking/TextureCache.cpp @@ -194,7 +194,6 @@ NetworkTexturePointer TextureCache::getTexture(const QUrl& url, image::TextureUs if (type == image::TextureUsage::CUBE_TEXTURE) { QUrlQuery query { url.query() }; query.addQueryItem("skybox", ""); - qDebug() << "Updating cubemap texture query from" << url.query() << "to" << query.toString(); modifiedUrl.setQuery(query.toString()); } TextureExtra extra = { type, content, maxNumPixels }; diff --git a/libraries/networking/src/AssetClient.cpp b/libraries/networking/src/AssetClient.cpp index 0d4e7c8388..940daf4d19 100644 --- a/libraries/networking/src/AssetClient.cpp +++ b/libraries/networking/src/AssetClient.cpp @@ -617,7 +617,6 @@ MessageID AssetClient::setBakingEnabled(const AssetPathList& paths, bool enabled _pendingMappingRequests[assetServer][messageID] = callback; return messageID; - } } diff --git a/libraries/networking/src/ResourceCache.cpp b/libraries/networking/src/ResourceCache.cpp index fbe7c05801..a3ac995bcf 100644 --- a/libraries/networking/src/ResourceCache.cpp +++ b/libraries/networking/src/ResourceCache.cpp @@ -734,9 +734,7 @@ void Resource::handleReplyFinished() { qCDebug(networking).noquote() << QString("Request finished for %1%2").arg(_url.toDisplayString(), extraInfo); auto relativePathURL = _request->getRelativePathUrl(); - qDebug() << "Relative path is: " << relativePathURL; if (!relativePathURL.isEmpty()) { - qDebug() << "setting effective path"; _effectiveBaseURL = relativePathURL; } From e1d79ee8f4a8ed0e1f605d57771897366314d94c Mon Sep 17 00:00:00 2001 From: Stephen Birarda Date: Tue, 19 Sep 2017 11:04:01 -0700 Subject: [PATCH 124/129] add abort to Baker and subclasses --- assignment-client/src/assets/AssetServer.cpp | 68 +++++------- assignment-client/src/assets/AssetServer.h | 23 +--- .../src/assets/BakeAssetTask.cpp | 79 ++++++++++++++ assignment-client/src/assets/BakeAssetTask.h | 50 +++++++++ libraries/baking/src/Baker.cpp | 33 +++++- libraries/baking/src/Baker.h | 16 +++ libraries/baking/src/FBXBaker.cpp | 79 +++++++++++--- libraries/baking/src/FBXBaker.h | 6 +- libraries/baking/src/TextureBaker.cpp | 21 +++- libraries/baking/src/TextureBaker.h | 5 + libraries/image/src/image/Image.cpp | 101 +++++++++++------- libraries/image/src/image/Image.h | 54 ++++++---- .../src/model-networking/TextureCache.cpp | 2 +- 13 files changed, 403 insertions(+), 134 deletions(-) create mode 100644 assignment-client/src/assets/BakeAssetTask.cpp create mode 100644 assignment-client/src/assets/BakeAssetTask.h diff --git a/assignment-client/src/assets/AssetServer.cpp b/assignment-client/src/assets/AssetServer.cpp index f1e921c768..17aae74dfd 100644 --- a/assignment-client/src/assets/AssetServer.cpp +++ b/assignment-client/src/assets/AssetServer.cpp @@ -35,6 +35,7 @@ #include #include "AssetServerLogging.h" +#include "BakeAssetTask.h" #include "SendAssetTask.h" #include "UploadAssetTask.h" @@ -53,45 +54,6 @@ static QStringList BAKEABLE_TEXTURE_EXTENSIONS; static const QString BAKED_MODEL_SIMPLE_NAME = "asset.fbx"; static const QString BAKED_TEXTURE_SIMPLE_NAME = "texture.ktx"; -BakeAssetTask::BakeAssetTask(const AssetHash& assetHash, const AssetPath& assetPath, const QString& filePath) - : _assetHash(assetHash), _assetPath(assetPath), _filePath(filePath) { -} - -void BakeAssetTask::run() { - _isBaking.store(true); - - qRegisterMetaType >("QVector"); - TextureBakerThreadGetter fn = []() -> QThread* { return QThread::currentThread(); }; - - std::unique_ptr baker; - - if (_assetPath.endsWith(".fbx")) { - baker = std::unique_ptr { - new FBXBaker(QUrl("file:///" + _filePath), fn, PathUtils::generateTemporaryDir()) - }; - } else { - baker = std::unique_ptr { - new TextureBaker(QUrl("file:///" + _filePath), image::TextureUsage::CUBE_TEXTURE, - PathUtils::generateTemporaryDir()) - }; - } - - QEventLoop loop; - connect(baker.get(), &Baker::finished, &loop, &QEventLoop::quit); - QMetaObject::invokeMethod(baker.get(), "bake", Qt::QueuedConnection); - loop.exec(); - - if (baker->hasErrors()) { - qDebug() << "Failed to bake: " << _assetHash << _assetPath << baker->getErrors(); - auto errors = baker->getErrors().join('\n'); // Join error list into a single string for convenience - emit bakeFailed(_assetHash, _assetPath, errors); - } else { - auto vectorOutputFiles = QVector::fromStdVector(baker->getOutputFiles()); - qDebug() << "Finished baking: " << _assetHash << _assetPath << vectorOutputFiles; - emit bakeComplete(_assetHash, _assetPath, vectorOutputFiles); - } -} - void AssetServer::bakeAsset(const AssetHash& assetHash, const AssetPath& assetPath, const QString& filePath) { qDebug() << "Starting bake for: " << assetPath << assetHash; auto it = _pendingBakes.find(assetHash); @@ -102,6 +64,7 @@ void AssetServer::bakeAsset(const AssetHash& assetHash, const AssetPath& assetPa connect(task.get(), &BakeAssetTask::bakeComplete, this, &AssetServer::handleCompletedBake); connect(task.get(), &BakeAssetTask::bakeFailed, this, &AssetServer::handleFailedBake); + connect(task.get(), &BakeAssetTask::bakeAborted, this, &AssetServer::handleAbortedBake); _bakingTaskPool.start(task.get()); } else { @@ -310,6 +273,28 @@ AssetServer::AssetServer(ReceivedMessage& message) : } void AssetServer::aboutToFinish() { + + // remove pending transfer tasks + _transferTaskPool.clear(); + + // abort each of our still running bake tasks, remove pending bakes that were never put on the thread pool + auto it = _pendingBakes.begin(); + while (it != _pendingBakes.end()) { + auto pendingRunnable = _bakingTaskPool.tryTake(it.value().get()); + + if (pendingRunnable) { + _pendingBakes.erase(it); + } else { + it.value()->abort(); + ++it; + } + } + + // make sure all bakers are finished or aborted + while (_pendingBakes.size() > 0) { + QCoreApplication::processEvents(); + } + // re-set defaults in image library image::setColorTexturesCompressionEnabled(_wasCubeTextureCompressionEnabled); image::setGrayscaleTexturesCompressionEnabled(_wasGrayscaleTextureCompressionEnabled); @@ -1247,6 +1232,11 @@ void AssetServer::handleCompletedBake(QString originalAssetHash, QString origina _pendingBakes.remove(originalAssetHash); } +void AssetServer::handleAbortedBake(QString originalAssetHash, QString assetPath) { + // for an aborted bake we don't do anything but remove the BakeAssetTask from our pending bakes + _pendingBakes.remove(originalAssetHash); +} + static const QString BAKE_VERSION_KEY = "bake_version"; static const QString APP_VERSION_KEY = "app_version"; static const QString FAILED_LAST_BAKE_KEY = "failed_last_bake"; diff --git a/assignment-client/src/assets/AssetServer.h b/assignment-client/src/assets/AssetServer.h index 8641f5498f..10ea067ee5 100644 --- a/assignment-client/src/assets/AssetServer.h +++ b/assignment-client/src/assets/AssetServer.h @@ -29,26 +29,6 @@ namespace std { }; } -class BakeAssetTask : public QObject, public QRunnable { - Q_OBJECT -public: - BakeAssetTask(const AssetHash& assetHash, const AssetPath& assetPath, const QString& filePath); - - bool isBaking() { return _isBaking.load(); } - - void run() override; - -signals: - void bakeComplete(QString assetHash, QString assetPath, QVector outputFiles); - void bakeFailed(QString assetHash, QString assetPath, QString errors); - -private: - std::atomic _isBaking { false }; - AssetHash _assetHash; - AssetPath _assetPath; - QString _filePath; -}; - struct AssetMeta { AssetMeta() { } @@ -59,6 +39,8 @@ struct AssetMeta { QString lastBakeErrors; }; +class BakeAssetTask; + class AssetServer : public ThreadedAssignment { Q_OBJECT public: @@ -121,6 +103,7 @@ private: /// Move baked content for asset to baked directory and update baked status void handleCompletedBake(QString originalAssetHash, QString assetPath, QVector bakedFilePaths); void handleFailedBake(QString originalAssetHash, QString assetPath, QString errors); + void handleAbortedBake(QString originalAssetHash, QString assetPath); /// Create meta file to describe baked content for original asset std::pair readMetaFile(AssetHash hash); diff --git a/assignment-client/src/assets/BakeAssetTask.cpp b/assignment-client/src/assets/BakeAssetTask.cpp new file mode 100644 index 0000000000..9073510f79 --- /dev/null +++ b/assignment-client/src/assets/BakeAssetTask.cpp @@ -0,0 +1,79 @@ +// +// BakeAssetTask.cpp +// assignment-client/src/assets +// +// Created by Stephen Birarda on 9/18/17. +// Copyright 2017 High Fidelity, Inc. +// +// Distributed under the Apache License, Version 2.0. +// See the accompanying file LICENSE or http://www.apache.org/licenses/LICENSE-2.0.html +// + +#include "BakeAssetTask.h" + +#include + +#include +#include + +BakeAssetTask::BakeAssetTask(const AssetHash& assetHash, const AssetPath& assetPath, const QString& filePath) : + _assetHash(assetHash), + _assetPath(assetPath), + _filePath(filePath) +{ + +} + +void BakeAssetTask::run() { + _isBaking.store(true); + + qRegisterMetaType >("QVector"); + TextureBakerThreadGetter fn = []() -> QThread* { return QThread::currentThread(); }; + + if (_assetPath.endsWith(".fbx")) { + _baker = std::unique_ptr { + new FBXBaker(QUrl("file:///" + _filePath), fn, PathUtils::generateTemporaryDir()) + }; + } else { + _baker = std::unique_ptr { + new TextureBaker(QUrl("file:///" + _filePath), image::TextureUsage::CUBE_TEXTURE, + PathUtils::generateTemporaryDir()) + }; + } + + QEventLoop loop; + connect(_baker.get(), &Baker::finished, &loop, &QEventLoop::quit); + connect(_baker.get(), &Baker::aborted, &loop, &QEventLoop::quit); + QMetaObject::invokeMethod(_baker.get(), "bake", Qt::QueuedConnection); + loop.exec(); + + if (_baker->wasAborted()) { + qDebug() << "Aborted baking: " << _assetHash << _assetPath; + + _wasAborted.store(true); + + emit bakeAborted(_assetHash, _assetPath); + } else if (_baker->hasErrors()) { + qDebug() << "Failed to bake: " << _assetHash << _assetPath << _baker->getErrors(); + + auto errors = _baker->getErrors().join('\n'); // Join error list into a single string for convenience + + _didFinish.store(true); + + emit bakeFailed(_assetHash, _assetPath, errors); + } else { + auto vectorOutputFiles = QVector::fromStdVector(_baker->getOutputFiles()); + + qDebug() << "Finished baking: " << _assetHash << _assetPath << vectorOutputFiles; + + _didFinish.store(true); + + emit bakeComplete(_assetHash, _assetPath, vectorOutputFiles); + } +} + +void BakeAssetTask::abort() { + if (_baker) { + _baker->abort(); + } +} diff --git a/assignment-client/src/assets/BakeAssetTask.h b/assignment-client/src/assets/BakeAssetTask.h new file mode 100644 index 0000000000..5c456c4521 --- /dev/null +++ b/assignment-client/src/assets/BakeAssetTask.h @@ -0,0 +1,50 @@ +// +// BakeAssetTask.h +// assignment-client/src/assets +// +// Created by Stephen Birarda on 9/18/17. +// Copyright 2017 High Fidelity, Inc. +// +// Distributed under the Apache License, Version 2.0. +// See the accompanying file LICENSE or http://www.apache.org/licenses/LICENSE-2.0.html +// + +#ifndef hifi_BakeAssetTask_h +#define hifi_BakeAssetTask_h + +#include +#include +#include + +#include +#include + +class BakeAssetTask : public QObject, public QRunnable { + Q_OBJECT +public: + BakeAssetTask(const AssetHash& assetHash, const AssetPath& assetPath, const QString& filePath); + + bool isBaking() { return _isBaking.load(); } + + void run() override; + + void abort(); + bool wasAborted() const { return _wasAborted.load(); } + bool didFinish() const { return _didFinish.load(); } + +signals: + void bakeComplete(QString assetHash, QString assetPath, QVector outputFiles); + void bakeFailed(QString assetHash, QString assetPath, QString errors); + void bakeAborted(QString assetHash, QString assetPath); + +private: + std::atomic _isBaking { false }; + AssetHash _assetHash; + AssetPath _assetPath; + QString _filePath; + std::unique_ptr _baker; + std::atomic _wasAborted { false }; + std::atomic _didFinish { false }; +}; + +#endif // hifi_BakeAssetTask_h diff --git a/libraries/baking/src/Baker.cpp b/libraries/baking/src/Baker.cpp index c0cbd8d124..2adedf08a1 100644 --- a/libraries/baking/src/Baker.cpp +++ b/libraries/baking/src/Baker.cpp @@ -13,20 +13,49 @@ #include "Baker.h" +bool Baker::shouldStop() { + if (_shouldAbort) { + setWasAborted(true); + return true; + } + + if (hasErrors()) { + return true; + } + + return false; +} + void Baker::handleError(const QString& error) { qCCritical(model_baking).noquote() << error; _errorList.append(error); - emit finished(); + setIsFinished(true); } void Baker::handleErrors(const QStringList& errors) { // we're appending errors, presumably from a baking operation we called // add those to our list and emit that we are finished _errorList.append(errors); - emit finished(); + setIsFinished(true); } void Baker::handleWarning(const QString& warning) { qCWarning(model_baking).noquote() << warning; _warningList.append(warning); } + +void Baker::setIsFinished(bool isFinished) { + _isFinished.store(isFinished); + + if (isFinished) { + emit finished(); + } +} + +void Baker::setWasAborted(bool wasAborted) { + _wasAborted.store(wasAborted); + + if (wasAborted) { + emit aborted(); + } +} diff --git a/libraries/baking/src/Baker.h b/libraries/baking/src/Baker.h index fd76246497..2da315c9fc 100644 --- a/libraries/baking/src/Baker.h +++ b/libraries/baking/src/Baker.h @@ -18,6 +18,8 @@ class Baker : public QObject { Q_OBJECT public: + bool shouldStop(); + bool hasErrors() const { return !_errorList.isEmpty(); } QStringList getErrors() const { return _errorList; } @@ -26,11 +28,20 @@ public: std::vector getOutputFiles() const { return _outputFiles; } + virtual void setIsFinished(bool isFinished); + bool isFinished() const { return _isFinished.load(); } + + virtual void setWasAborted(bool wasAborted); + + bool wasAborted() const { return _wasAborted.load(); } + public slots: virtual void bake() = 0; + virtual void abort() { _shouldAbort.store(true); } signals: void finished(); + void aborted(); protected: void handleError(const QString& error); @@ -44,6 +55,11 @@ protected: QStringList _errorList; QStringList _warningList; + + std::atomic _isFinished { false }; + + std::atomic _shouldAbort { false }; + std::atomic _wasAborted { false }; }; #endif // hifi_Baker_h diff --git a/libraries/baking/src/FBXBaker.cpp b/libraries/baking/src/FBXBaker.cpp index 0f31b878ee..b5b6570599 100644 --- a/libraries/baking/src/FBXBaker.cpp +++ b/libraries/baking/src/FBXBaker.cpp @@ -56,7 +56,19 @@ FBXBaker::FBXBaker(const QUrl& fbxURL, TextureBakerThreadGetter textureThreadGet } +void FBXBaker::abort() { + Baker::abort(); + + // tell our underlying TextureBaker instances to abort + // the FBXBaker will wait until all are aborted before emitting its own abort signal + for (auto& textureBaker : _bakingTextures) { + textureBaker->abort(); + } +} + void FBXBaker::bake() { + qDebug() << "FBXBaker" << _fbxURL << "bake starting"; + auto tempDir = PathUtils::generateTemporaryDir(); if (tempDir.isEmpty()) { @@ -73,7 +85,7 @@ void FBXBaker::bake() { // setup the output folder for the results of this bake setupOutputFolder(); - if (hasErrors()) { + if (shouldStop()) { return; } @@ -87,22 +99,27 @@ void FBXBaker::bakeSourceCopy() { // load the scene from the FBX file importScene(); - if (hasErrors()) { + if (shouldStop()) { return; } // enumerate the models and textures found in the scene and start a bake for them - rewriteAndBakeSceneModels(); rewriteAndBakeSceneTextures(); - if (hasErrors()) { + if (shouldStop()) { + return; + } + + rewriteAndBakeSceneModels(); + + if (shouldStop()) { return; } // export the FBX with re-written texture references exportScene(); - if (hasErrors()) { + if (shouldStop()) { return; } @@ -165,7 +182,6 @@ void FBXBaker::loadSourceFBX() { networkRequest.setAttribute(QNetworkRequest::CacheLoadControlAttribute, QNetworkRequest::AlwaysNetwork); networkRequest.setHeader(QNetworkRequest::UserAgentHeader, HIGH_FIDELITY_USER_AGENT); - networkRequest.setUrl(_fbxURL); qCDebug(model_baking) << "Downloading" << _fbxURL; @@ -553,6 +569,11 @@ void FBXBaker::rewriteAndBakeSceneTextures() { while (object != rootChild.children.end()) { if (object->name == "Texture") { + // double check that we didn't get an abort while baking the last texture + if (shouldStop()) { + return; + } + // enumerate the texture children for (FBXNode& textureChild : object->children) { @@ -630,8 +651,9 @@ void FBXBaker::bakeTexture(const QUrl& textureURL, image::TextureUsage::Type tex &TextureBaker::deleteLater }; - // make sure we hear when the baking texture is done + // make sure we hear when the baking texture is done or aborted connect(bakingTexture.data(), &Baker::finished, this, &FBXBaker::handleBakedTexture); + connect(bakingTexture.data(), &TextureBaker::aborted, this, &FBXBaker::handleAbortedTexture); // keep a shared pointer to the baking texture _bakingTextures.insert(textureURL, bakingTexture); @@ -646,7 +668,7 @@ void FBXBaker::handleBakedTexture() { // make sure we haven't already run into errors, and that this is a valid texture if (bakedTexture) { - if (!hasErrors()) { + if (!shouldStop()) { if (!bakedTexture->hasErrors()) { if (!_originalOutputDir.isEmpty()) { // we've been asked to make copies of the originals, so we need to make copies of this if it is a linked texture @@ -698,6 +720,11 @@ void FBXBaker::handleBakedTexture() { // now that this texture has been baked, even though it failed, we can remove that TextureBaker from our list _bakingTextures.remove(bakedTexture->getTextureURL()); + // abort any other ongoing texture bakes since we know we'll end up failing + for (auto& bakingTexture : _bakingTextures) { + bakingTexture->abort(); + } + checkIfTexturesFinished(); } } else { @@ -711,6 +738,25 @@ void FBXBaker::handleBakedTexture() { } } +void FBXBaker::handleAbortedTexture() { + // grab the texture bake that was aborted and remove it from our hash since we don't need to track it anymore + TextureBaker* bakedTexture = qobject_cast(sender()); + + if (bakedTexture) { + _bakingTextures.remove(bakedTexture->getTextureURL()); + } + + // since a texture we were baking aborted, our status is also aborted + _shouldAbort.store(true); + + // abort any other ongoing texture bakes since we know we'll end up failing + for (auto& bakingTexture : _bakingTextures) { + bakingTexture->abort(); + } + + checkIfTexturesFinished(); +} + void FBXBaker::exportScene() { // save the relative path to this FBX inside our passed output folder auto fileName = _fbxURL.fileName(); @@ -735,25 +781,34 @@ void FBXBaker::exportScene() { qCDebug(model_baking) << "Exported" << _fbxURL << "with re-written paths to" << _bakedFBXFilePath; } - void FBXBaker::checkIfTexturesFinished() { // check if we're done everything we need to do for this FBX // and emit our finished signal if we're done if (_bakingTextures.isEmpty()) { - if (hasErrors()) { + if (shouldStop()) { // if we're checking for completion but we have errors // that means one or more of our texture baking operations failed if (_pendingErrorEmission) { - emit finished(); + setIsFinished(true); } return; } else { qCDebug(model_baking) << "Finished baking, emitting finsihed" << _fbxURL; - emit finished(); + setIsFinished(true); + } + } +} + +void FBXBaker::setWasAborted(bool wasAborted) { + if (wasAborted != _wasAborted.load()) { + Baker::setWasAborted(wasAborted); + + if (wasAborted) { + qCDebug(model_baking) << "Aborted baking" << _fbxURL; } } } diff --git a/libraries/baking/src/FBXBaker.h b/libraries/baking/src/FBXBaker.h index c611fb712f..26471a29b3 100644 --- a/libraries/baking/src/FBXBaker.h +++ b/libraries/baking/src/FBXBaker.h @@ -39,10 +39,11 @@ public: QUrl getFBXUrl() const { return _fbxURL; } QString getBakedFBXFilePath() const { return _bakedFBXFilePath; } + virtual void setWasAborted(bool wasAborted) override; + public slots: - // all calls to FBXBaker::bake for FBXBaker instances must be from the same thread - // because the Autodesk SDK will cause a crash if it is called from multiple threads virtual void bake() override; + virtual void abort() override; signals: void sourceCopyReadyToLoad(); @@ -51,6 +52,7 @@ private slots: void bakeSourceCopy(); void handleFBXNetworkReply(); void handleBakedTexture(); + void handleAbortedTexture(); private: void setupOutputFolder(); diff --git a/libraries/baking/src/TextureBaker.cpp b/libraries/baking/src/TextureBaker.cpp index cdf21a0290..febc7ea092 100644 --- a/libraries/baking/src/TextureBaker.cpp +++ b/libraries/baking/src/TextureBaker.cpp @@ -50,6 +50,13 @@ void TextureBaker::bake() { } } +void TextureBaker::abort() { + Baker::abort(); + + // flip our atomic bool so any ongoing texture processing is stopped + _abortProcessing.store(true); +} + const QStringList TextureBaker::getSupportedFormats() { auto formats = QImageReader::supportedImageFormats(); QStringList stringFormats; @@ -111,7 +118,11 @@ void TextureBaker::handleTextureNetworkReply() { void TextureBaker::processTexture() { auto processedTexture = image::processImage(_originalTexture, _textureURL.toString().toStdString(), - ABSOLUTE_MAX_TEXTURE_NUM_PIXELS, _textureType); + ABSOLUTE_MAX_TEXTURE_NUM_PIXELS, _textureType, _abortProcessing); + + if (shouldStop()) { + return; + } if (!processedTexture) { handleError("Could not process texture " + _textureURL.toString()); @@ -145,5 +156,11 @@ void TextureBaker::processTexture() { } qCDebug(model_baking) << "Baked texture" << _textureURL; - emit finished(); + setIsFinished(true); +} + +void TextureBaker::setWasAborted(bool wasAborted) { + Baker::setWasAborted(wasAborted); + + qCDebug(model_baking) << "Aborted baking" << _textureURL; } diff --git a/libraries/baking/src/TextureBaker.h b/libraries/baking/src/TextureBaker.h index 5ea696dda0..e5bd41cf0d 100644 --- a/libraries/baking/src/TextureBaker.h +++ b/libraries/baking/src/TextureBaker.h @@ -39,8 +39,11 @@ public: QString getDestinationFilePath() const { return _outputDirectory.absoluteFilePath(_bakedTextureFileName); } QString getBakedTextureFileName() const { return _bakedTextureFileName; } + virtual void setWasAborted(bool wasAborted) override; + public slots: virtual void bake() override; + virtual void abort() override; signals: void originalTextureLoaded(); @@ -58,6 +61,8 @@ private: QDir _outputDirectory; QString _bakedTextureFileName; + + std::atomic _abortProcessing { false }; }; #endif // hifi_TextureBaker_h diff --git a/libraries/image/src/image/Image.cpp b/libraries/image/src/image/Image.cpp index 30299663de..9d49efcba0 100644 --- a/libraries/image/src/image/Image.cpp +++ b/libraries/image/src/image/Image.cpp @@ -97,52 +97,64 @@ TextureUsage::TextureLoader TextureUsage::getTextureLoaderForType(Type type, con } } -gpu::TexturePointer TextureUsage::createStrict2DTextureFromImage(const QImage& srcImage, const std::string& srcImageName) { - return process2DTextureColorFromImage(srcImage, srcImageName, true); +gpu::TexturePointer TextureUsage::createStrict2DTextureFromImage(const QImage& srcImage, const std::string& srcImageName, + const std::atomic& abortProcessing) { + return process2DTextureColorFromImage(srcImage, srcImageName, true, abortProcessing); } -gpu::TexturePointer TextureUsage::create2DTextureFromImage(const QImage& srcImage, const std::string& srcImageName) { - return process2DTextureColorFromImage(srcImage, srcImageName, false); +gpu::TexturePointer TextureUsage::create2DTextureFromImage(const QImage& srcImage, const std::string& srcImageName, + const std::atomic& abortProcessing) { + return process2DTextureColorFromImage(srcImage, srcImageName, false, abortProcessing); } -gpu::TexturePointer TextureUsage::createAlbedoTextureFromImage(const QImage& srcImage, const std::string& srcImageName) { - return process2DTextureColorFromImage(srcImage, srcImageName, false); +gpu::TexturePointer TextureUsage::createAlbedoTextureFromImage(const QImage& srcImage, const std::string& srcImageName, + const std::atomic& abortProcessing) { + return process2DTextureColorFromImage(srcImage, srcImageName, false, abortProcessing); } -gpu::TexturePointer TextureUsage::createEmissiveTextureFromImage(const QImage& srcImage, const std::string& srcImageName) { - return process2DTextureColorFromImage(srcImage, srcImageName, false); +gpu::TexturePointer TextureUsage::createEmissiveTextureFromImage(const QImage& srcImage, const std::string& srcImageName, + const std::atomic& abortProcessing) { + return process2DTextureColorFromImage(srcImage, srcImageName, false, abortProcessing); } -gpu::TexturePointer TextureUsage::createLightmapTextureFromImage(const QImage& srcImage, const std::string& srcImageName) { - return process2DTextureColorFromImage(srcImage, srcImageName, false); +gpu::TexturePointer TextureUsage::createLightmapTextureFromImage(const QImage& srcImage, const std::string& srcImageName, + const std::atomic& abortProcessing) { + return process2DTextureColorFromImage(srcImage, srcImageName, false, abortProcessing); } -gpu::TexturePointer TextureUsage::createNormalTextureFromNormalImage(const QImage& srcImage, const std::string& srcImageName) { - return process2DTextureNormalMapFromImage(srcImage, srcImageName, false); +gpu::TexturePointer TextureUsage::createNormalTextureFromNormalImage(const QImage& srcImage, const std::string& srcImageName, + const std::atomic& abortProcessing) { + return process2DTextureNormalMapFromImage(srcImage, srcImageName, false, abortProcessing); } -gpu::TexturePointer TextureUsage::createNormalTextureFromBumpImage(const QImage& srcImage, const std::string& srcImageName) { - return process2DTextureNormalMapFromImage(srcImage, srcImageName, true); +gpu::TexturePointer TextureUsage::createNormalTextureFromBumpImage(const QImage& srcImage, const std::string& srcImageName, + const std::atomic& abortProcessing) { + return process2DTextureNormalMapFromImage(srcImage, srcImageName, true, abortProcessing); } -gpu::TexturePointer TextureUsage::createRoughnessTextureFromImage(const QImage& srcImage, const std::string& srcImageName) { - return process2DTextureGrayscaleFromImage(srcImage, srcImageName, false); +gpu::TexturePointer TextureUsage::createRoughnessTextureFromImage(const QImage& srcImage, const std::string& srcImageName, + const std::atomic& abortProcessing) { + return process2DTextureGrayscaleFromImage(srcImage, srcImageName, false, abortProcessing); } -gpu::TexturePointer TextureUsage::createRoughnessTextureFromGlossImage(const QImage& srcImage, const std::string& srcImageName) { - return process2DTextureGrayscaleFromImage(srcImage, srcImageName, true); +gpu::TexturePointer TextureUsage::createRoughnessTextureFromGlossImage(const QImage& srcImage, const std::string& srcImageName, + const std::atomic& abortProcessing) { + return process2DTextureGrayscaleFromImage(srcImage, srcImageName, true, abortProcessing); } -gpu::TexturePointer TextureUsage::createMetallicTextureFromImage(const QImage& srcImage, const std::string& srcImageName) { - return process2DTextureGrayscaleFromImage(srcImage, srcImageName, false); +gpu::TexturePointer TextureUsage::createMetallicTextureFromImage(const QImage& srcImage, const std::string& srcImageName, + const std::atomic& abortProcessing) { + return process2DTextureGrayscaleFromImage(srcImage, srcImageName, false, abortProcessing); } -gpu::TexturePointer TextureUsage::createCubeTextureFromImage(const QImage& srcImage, const std::string& srcImageName) { - return processCubeTextureColorFromImage(srcImage, srcImageName, true); +gpu::TexturePointer TextureUsage::createCubeTextureFromImage(const QImage& srcImage, const std::string& srcImageName, + const std::atomic& abortProcessing) { + return processCubeTextureColorFromImage(srcImage, srcImageName, true, abortProcessing); } -gpu::TexturePointer TextureUsage::createCubeTextureFromImageWithoutIrradiance(const QImage& srcImage, const std::string& srcImageName) { - return processCubeTextureColorFromImage(srcImage, srcImageName, false); +gpu::TexturePointer TextureUsage::createCubeTextureFromImageWithoutIrradiance(const QImage& srcImage, const std::string& srcImageName, + const std::atomic& abortProcessing) { + return processCubeTextureColorFromImage(srcImage, srcImageName, false, abortProcessing); } @@ -194,8 +206,9 @@ void setCubeTexturesCompressionEnabled(bool enabled) { compressCubeTextures.store(enabled); } - -gpu::TexturePointer processImage(const QByteArray& content, const std::string& filename, int maxNumPixels, TextureUsage::Type textureType) { +gpu::TexturePointer processImage(const QByteArray& content, const std::string& filename, + int maxNumPixels, TextureUsage::Type textureType, + const std::atomic& abortProcessing) { // Help the QImage loader by extracting the image file format from the url filename ext. // Some tga are not created properly without it. auto filenameExtension = filename.substr(filename.find_last_of('.') + 1); @@ -245,7 +258,7 @@ gpu::TexturePointer processImage(const QByteArray& content, const std::string& f } auto loader = TextureUsage::getTextureLoaderForType(textureType); - auto texture = loader(image, filename); + auto texture = loader(image, filename, abortProcessing); return texture; } @@ -321,7 +334,7 @@ struct MyErrorHandler : public nvtt::ErrorHandler { } }; -void generateMips(gpu::Texture* texture, QImage& image, int face = -1) { +void generateMips(gpu::Texture* texture, QImage& image, const std::atomic& abortProcessing = false, int face = -1) { #if CPU_MIPMAPS PROFILE_RANGE(resource_parse, "generateMips"); @@ -439,14 +452,20 @@ void generateMips(gpu::Texture* texture, QImage& image, int face = -1) { class SequentialTaskDispatcher : public nvtt::TaskDispatcher { public: + SequentialTaskDispatcher(const std::atomic& abortProcessing) : _abortProcessing(abortProcessing) {}; + + const std::atomic& _abortProcessing; + virtual void dispatch(nvtt::Task* task, void* context, int count) override { for (int i = 0; i < count; i++) { - task(context, i); + if (!_abortProcessing.load()) { + task(context, i); + } } } }; - SequentialTaskDispatcher dispatcher; + SequentialTaskDispatcher dispatcher(abortProcessing); nvtt::Compressor compressor; compressor.setTaskDispatcher(&dispatcher); compressor.process(inputOptions, compressionOptions, outputOptions); @@ -482,7 +501,8 @@ void processTextureAlpha(const QImage& srcImage, bool& validAlpha, bool& alphaAs validAlpha = (numOpaques != NUM_PIXELS); } -gpu::TexturePointer TextureUsage::process2DTextureColorFromImage(const QImage& srcImage, const std::string& srcImageName, bool isStrict) { +gpu::TexturePointer TextureUsage::process2DTextureColorFromImage(const QImage& srcImage, const std::string& srcImageName, + bool isStrict, const std::atomic& abortProcessing) { PROFILE_RANGE(resource_parse, "process2DTextureColorFromImage"); QImage image = processSourceImage(srcImage, false); bool validAlpha = image.hasAlphaChannel(); @@ -530,7 +550,7 @@ gpu::TexturePointer TextureUsage::process2DTextureColorFromImage(const QImage& s } theTexture->setUsage(usage.build()); theTexture->setStoredMipFormat(formatMip); - generateMips(theTexture.get(), image); + generateMips(theTexture.get(), image, abortProcessing); } return theTexture; @@ -606,7 +626,8 @@ QImage processBumpMap(QImage& image) { return result; } -gpu::TexturePointer TextureUsage::process2DTextureNormalMapFromImage(const QImage& srcImage, const std::string& srcImageName, bool isBumpMap) { +gpu::TexturePointer TextureUsage::process2DTextureNormalMapFromImage(const QImage& srcImage, const std::string& srcImageName, + bool isBumpMap, const std::atomic& abortProcessing) { PROFILE_RANGE(resource_parse, "process2DTextureNormalMapFromImage"); QImage image = processSourceImage(srcImage, false); @@ -631,13 +652,15 @@ gpu::TexturePointer TextureUsage::process2DTextureNormalMapFromImage(const QImag theTexture = gpu::Texture::create2D(formatGPU, image.width(), image.height(), gpu::Texture::MAX_NUM_MIPS, gpu::Sampler(gpu::Sampler::FILTER_MIN_MAG_MIP_LINEAR)); theTexture->setSource(srcImageName); theTexture->setStoredMipFormat(formatMip); - generateMips(theTexture.get(), image); + generateMips(theTexture.get(), image, abortProcessing); } return theTexture; } -gpu::TexturePointer TextureUsage::process2DTextureGrayscaleFromImage(const QImage& srcImage, const std::string& srcImageName, bool isInvertedPixels) { +gpu::TexturePointer TextureUsage::process2DTextureGrayscaleFromImage(const QImage& srcImage, const std::string& srcImageName, + bool isInvertedPixels, + const std::atomic& abortProcessing) { PROFILE_RANGE(resource_parse, "process2DTextureGrayscaleFromImage"); QImage image = processSourceImage(srcImage, false); @@ -665,7 +688,7 @@ gpu::TexturePointer TextureUsage::process2DTextureGrayscaleFromImage(const QImag theTexture = gpu::Texture::create2D(formatGPU, image.width(), image.height(), gpu::Texture::MAX_NUM_MIPS, gpu::Sampler(gpu::Sampler::FILTER_MIN_MAG_MIP_LINEAR)); theTexture->setSource(srcImageName); theTexture->setStoredMipFormat(formatMip); - generateMips(theTexture.get(), image); + generateMips(theTexture.get(), image, abortProcessing); } return theTexture; @@ -927,7 +950,9 @@ const CubeLayout CubeLayout::CUBEMAP_LAYOUTS[] = { }; const int CubeLayout::NUM_CUBEMAP_LAYOUTS = sizeof(CubeLayout::CUBEMAP_LAYOUTS) / sizeof(CubeLayout); -gpu::TexturePointer TextureUsage::processCubeTextureColorFromImage(const QImage& srcImage, const std::string& srcImageName, bool generateIrradiance) { +gpu::TexturePointer TextureUsage::processCubeTextureColorFromImage(const QImage& srcImage, const std::string& srcImageName, + bool generateIrradiance, + const std::atomic& abortProcessing) { PROFILE_RANGE(resource_parse, "processCubeTextureColorFromImage"); gpu::TexturePointer theTexture = nullptr; @@ -986,7 +1011,7 @@ gpu::TexturePointer TextureUsage::processCubeTextureColorFromImage(const QImage& theTexture->setStoredMipFormat(formatMip); for (uint8 face = 0; face < faces.size(); ++face) { - generateMips(theTexture.get(), faces[face], face); + generateMips(theTexture.get(), faces[face], abortProcessing, face); } // Generate irradiance while we are at it diff --git a/libraries/image/src/image/Image.h b/libraries/image/src/image/Image.h index 3bf45ace98..856dc009cf 100644 --- a/libraries/image/src/image/Image.h +++ b/libraries/image/src/image/Image.h @@ -41,26 +41,42 @@ enum Type { UNUSED_TEXTURE }; -using TextureLoader = std::function; +using TextureLoader = std::function&)>; TextureLoader getTextureLoaderForType(Type type, const QVariantMap& options = QVariantMap()); -gpu::TexturePointer create2DTextureFromImage(const QImage& image, const std::string& srcImageName); -gpu::TexturePointer createStrict2DTextureFromImage(const QImage& image, const std::string& srcImageName); -gpu::TexturePointer createAlbedoTextureFromImage(const QImage& image, const std::string& srcImageName); -gpu::TexturePointer createEmissiveTextureFromImage(const QImage& image, const std::string& srcImageName); -gpu::TexturePointer createNormalTextureFromNormalImage(const QImage& image, const std::string& srcImageName); -gpu::TexturePointer createNormalTextureFromBumpImage(const QImage& image, const std::string& srcImageName); -gpu::TexturePointer createRoughnessTextureFromImage(const QImage& image, const std::string& srcImageName); -gpu::TexturePointer createRoughnessTextureFromGlossImage(const QImage& image, const std::string& srcImageName); -gpu::TexturePointer createMetallicTextureFromImage(const QImage& image, const std::string& srcImageName); -gpu::TexturePointer createCubeTextureFromImage(const QImage& image, const std::string& srcImageName); -gpu::TexturePointer createCubeTextureFromImageWithoutIrradiance(const QImage& image, const std::string& srcImageName); -gpu::TexturePointer createLightmapTextureFromImage(const QImage& image, const std::string& srcImageName); +gpu::TexturePointer create2DTextureFromImage(const QImage& image, const std::string& srcImageName, + const std::atomic& abortProcessing); +gpu::TexturePointer createStrict2DTextureFromImage(const QImage& image, const std::string& srcImageName, + const std::atomic& abortProcessing); +gpu::TexturePointer createAlbedoTextureFromImage(const QImage& image, const std::string& srcImageName, + const std::atomic& abortProcessing); +gpu::TexturePointer createEmissiveTextureFromImage(const QImage& image, const std::string& srcImageName, + const std::atomic& abortProcessing); +gpu::TexturePointer createNormalTextureFromNormalImage(const QImage& image, const std::string& srcImageName, + const std::atomic& abortProcessing); +gpu::TexturePointer createNormalTextureFromBumpImage(const QImage& image, const std::string& srcImageName, + const std::atomic& abortProcessing); +gpu::TexturePointer createRoughnessTextureFromImage(const QImage& image, const std::string& srcImageName, + const std::atomic& abortProcessing); +gpu::TexturePointer createRoughnessTextureFromGlossImage(const QImage& image, const std::string& srcImageName, + const std::atomic& abortProcessing); +gpu::TexturePointer createMetallicTextureFromImage(const QImage& image, const std::string& srcImageName, + const std::atomic& abortProcessing); +gpu::TexturePointer createCubeTextureFromImage(const QImage& image, const std::string& srcImageName, + const std::atomic& abortProcessing); +gpu::TexturePointer createCubeTextureFromImageWithoutIrradiance(const QImage& image, const std::string& srcImageName, + const std::atomic& abortProcessing); +gpu::TexturePointer createLightmapTextureFromImage(const QImage& image, const std::string& srcImageName, + const std::atomic& abortProcessing); -gpu::TexturePointer process2DTextureColorFromImage(const QImage& srcImage, const std::string& srcImageName, bool isStrict); -gpu::TexturePointer process2DTextureNormalMapFromImage(const QImage& srcImage, const std::string& srcImageName, bool isBumpMap); -gpu::TexturePointer process2DTextureGrayscaleFromImage(const QImage& srcImage, const std::string& srcImageName, bool isInvertedPixels); -gpu::TexturePointer processCubeTextureColorFromImage(const QImage& srcImage, const std::string& srcImageName, bool generateIrradiance); +gpu::TexturePointer process2DTextureColorFromImage(const QImage& srcImage, const std::string& srcImageName, bool isStrict, + const std::atomic& abortProcessing); +gpu::TexturePointer process2DTextureNormalMapFromImage(const QImage& srcImage, const std::string& srcImageName, bool isBumpMap, + const std::atomic& abortProcessing); +gpu::TexturePointer process2DTextureGrayscaleFromImage(const QImage& srcImage, const std::string& srcImageName, bool isInvertedPixels, + const std::atomic& abortProcessing); +gpu::TexturePointer processCubeTextureColorFromImage(const QImage& srcImage, const std::string& srcImageName, bool generateIrradiance, + const std::atomic& abortProcessing); } // namespace TextureUsage @@ -74,7 +90,9 @@ void setNormalTexturesCompressionEnabled(bool enabled); void setGrayscaleTexturesCompressionEnabled(bool enabled); void setCubeTexturesCompressionEnabled(bool enabled); -gpu::TexturePointer processImage(const QByteArray& content, const std::string& url, int maxNumPixels, TextureUsage::Type textureType); +gpu::TexturePointer processImage(const QByteArray& content, const std::string& url, + int maxNumPixels, TextureUsage::Type textureType, + const std::atomic& abortProcessing = false); } // namespace image diff --git a/libraries/model-networking/src/model-networking/TextureCache.cpp b/libraries/model-networking/src/model-networking/TextureCache.cpp index cc29e841ce..fed5c69ce2 100644 --- a/libraries/model-networking/src/model-networking/TextureCache.cpp +++ b/libraries/model-networking/src/model-networking/TextureCache.cpp @@ -265,7 +265,7 @@ gpu::TexturePointer getFallbackTextureForType(image::TextureUsage::Type type) { gpu::TexturePointer TextureCache::getImageTexture(const QString& path, image::TextureUsage::Type type, QVariantMap options) { QImage image = QImage(path); auto loader = image::TextureUsage::getTextureLoaderForType(type, options); - return gpu::TexturePointer(loader(image, QUrl::fromLocalFile(path).fileName().toStdString())); + return gpu::TexturePointer(loader(image, QUrl::fromLocalFile(path).fileName().toStdString(), false)); } QSharedPointer TextureCache::createResource(const QUrl& url, const QSharedPointer& fallback, From a8ac015bc4e350ffc768699633f757af51a1f03c Mon Sep 17 00:00:00 2001 From: Stephen Birarda Date: Tue, 19 Sep 2017 13:56:42 -0700 Subject: [PATCH 125/129] address code review comments --- assignment-client/src/assets/AssetServer.cpp | 4 ++-- libraries/image/src/image/Image.cpp | 2 ++ 2 files changed, 4 insertions(+), 2 deletions(-) diff --git a/assignment-client/src/assets/AssetServer.cpp b/assignment-client/src/assets/AssetServer.cpp index 17aae74dfd..fea9d8329d 100644 --- a/assignment-client/src/assets/AssetServer.cpp +++ b/assignment-client/src/assets/AssetServer.cpp @@ -280,10 +280,10 @@ void AssetServer::aboutToFinish() { // abort each of our still running bake tasks, remove pending bakes that were never put on the thread pool auto it = _pendingBakes.begin(); while (it != _pendingBakes.end()) { - auto pendingRunnable = _bakingTaskPool.tryTake(it.value().get()); + auto pendingRunnable = _bakingTaskPool.tryTake(it->get()); if (pendingRunnable) { - _pendingBakes.erase(it); + it = _pendingBakes.erase(it); } else { it.value()->abort(); ++it; diff --git a/libraries/image/src/image/Image.cpp b/libraries/image/src/image/Image.cpp index 9d49efcba0..6081355eaa 100644 --- a/libraries/image/src/image/Image.cpp +++ b/libraries/image/src/image/Image.cpp @@ -460,6 +460,8 @@ void generateMips(gpu::Texture* texture, QImage& image, const std::atomic& for (int i = 0; i < count; i++) { if (!_abortProcessing.load()) { task(context, i); + } else { + break; } } } From d4e35edc84f300f83f5352b0e4257a5f576409a4 Mon Sep 17 00:00:00 2001 From: Stephen Birarda Date: Tue, 19 Sep 2017 15:08:44 -0700 Subject: [PATCH 126/129] add memory for unique_ptr in BakeAssetTask --- assignment-client/src/assets/BakeAssetTask.h | 2 ++ 1 file changed, 2 insertions(+) diff --git a/assignment-client/src/assets/BakeAssetTask.h b/assignment-client/src/assets/BakeAssetTask.h index 5c456c4521..45e7ec8702 100644 --- a/assignment-client/src/assets/BakeAssetTask.h +++ b/assignment-client/src/assets/BakeAssetTask.h @@ -12,6 +12,8 @@ #ifndef hifi_BakeAssetTask_h #define hifi_BakeAssetTask_h +#include + #include #include #include From 29a9331f145e4b72fe9f6ba68e14fade548784be Mon Sep 17 00:00:00 2001 From: Stephen Birarda Date: Wed, 20 Sep 2017 12:55:50 -0700 Subject: [PATCH 127/129] change include for QUrlQuery to be consistent --- assignment-client/src/assets/AssetServer.cpp | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/assignment-client/src/assets/AssetServer.cpp b/assignment-client/src/assets/AssetServer.cpp index fea9d8329d..b4287a0161 100644 --- a/assignment-client/src/assets/AssetServer.cpp +++ b/assignment-client/src/assets/AssetServer.cpp @@ -26,7 +26,7 @@ #include #include #include -#include +#include #include #include From f6d3960160bde08c156230fd6052649115177981 Mon Sep 17 00:00:00 2001 From: Stephen Birarda Date: Wed, 20 Sep 2017 15:58:52 -0700 Subject: [PATCH 128/129] fix call to maybeBake when looking at skybox --- assignment-client/src/assets/AssetServer.cpp | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/assignment-client/src/assets/AssetServer.cpp b/assignment-client/src/assets/AssetServer.cpp index b4287a0161..8f361515c0 100644 --- a/assignment-client/src/assets/AssetServer.cpp +++ b/assignment-client/src/assets/AssetServer.cpp @@ -530,7 +530,7 @@ void AssetServer::handleGetMappingOperation(ReceivedMessage& message, SharedNode bool isSkybox = query.hasQueryItem("skybox"); if (isSkybox) { writeMetaFile(originalAssetHash); - maybeBake(originalAssetHash, assetPath); + maybeBake(assetPath, originalAssetHash); } } } else { From dd050e103f5e35c9bcc29eaa1646fd7577206ab1 Mon Sep 17 00:00:00 2001 From: Atlante45 Date: Wed, 20 Sep 2017 18:23:28 -0700 Subject: [PATCH 129/129] Don't redirect to the same hash. --- assignment-client/src/assets/AssetServer.cpp | 18 +++++++++++++----- 1 file changed, 13 insertions(+), 5 deletions(-) diff --git a/assignment-client/src/assets/AssetServer.cpp b/assignment-client/src/assets/AssetServer.cpp index 8f361515c0..6e3db69c63 100644 --- a/assignment-client/src/assets/AssetServer.cpp +++ b/assignment-client/src/assets/AssetServer.cpp @@ -494,6 +494,7 @@ void AssetServer::handleGetMappingOperation(ReceivedMessage& message, SharedNode QString redirectedAssetHash; QString bakedAssetPath; quint8 wasRedirected = false; + bool bakingDisabled = false; if (!bakedRootFile.isEmpty()) { // we ran into an asset for which we could have a baked version, let's check if it's ready @@ -501,10 +502,15 @@ void AssetServer::handleGetMappingOperation(ReceivedMessage& message, SharedNode auto bakedIt = _fileMappings.find(bakedAssetPath); if (bakedIt != _fileMappings.end()) { - qDebug() << "Did find baked version for: " << originalAssetHash << assetPath; - // we found a baked version of the requested asset to serve, redirect to that - redirectedAssetHash = bakedIt->second; - wasRedirected = true; + if (bakedIt->second != originalAssetHash) { + qDebug() << "Did find baked version for: " << originalAssetHash << assetPath; + // we found a baked version of the requested asset to serve, redirect to that + redirectedAssetHash = bakedIt->second; + wasRedirected = true; + } else { + qDebug() << "Did not find baked version for: " << originalAssetHash << assetPath << " (disabled)"; + bakingDisabled = true; + } } else { qDebug() << "Did not find baked version for: " << originalAssetHash << assetPath; } @@ -530,7 +536,9 @@ void AssetServer::handleGetMappingOperation(ReceivedMessage& message, SharedNode bool isSkybox = query.hasQueryItem("skybox"); if (isSkybox) { writeMetaFile(originalAssetHash); - maybeBake(assetPath, originalAssetHash); + if (!bakingDisabled) { + maybeBake(assetPath, originalAssetHash); + } } } } else {