Mirrors/overte-HifiExperiments
3
0
Fork 0
mirror of https://github.com/HifiExperiments/overte.git synced 2025-04-07 10:02:24 +02:00
Code Issues Projects Releases Packages Wiki Activity Actions

Improve baking interface and add automatic baking to asset server

Browse source
This commit is contained in:
Ryan Huffman 2017-08-16 13:51:40 -07:00
parent 8d4ab5f751
commit 39f04adc8d
25 changed files with 1139 additions and 38 deletions
Show stats Download patch file Download diff file Expand all files Collapse all files

2
assignment-client/CMakeLists.txt
View file

@ -13,7 +13,7 @@ setup_memory_debugger()
link_hifi_libraries(
audio avatars octree gpu model fbx entities
networking animation recording shared script-engine embedded-webserver
controllers physics plugins midi
controllers physics plugins midi baking image
)
if (WIN32)

190
assignment-client/src/assets/AssetServer.cpp
View file

@ -24,6 +24,8 @@
#include <QtCore/QJsonDocument>
#include <QtCore/QString>
#include <QtGui/QImageReader>
#include <QtCore/QVector>
#include <qurlquery.h>
#include <SharedUtil.h>
#include <PathUtils.h>
@ -33,6 +35,9 @@
#include "SendAssetTask.h"
#include "UploadAssetTask.h"
#include <ClientServerUtils.h>
#include <FBXBaker.h>
#include <memory>
static const uint8_t MIN_CORES_FOR_MULTICORE = 4;
static const uint8_t CPU_AFFINITY_COUNT_HIGH = 2;
@ -43,6 +48,128 @@ static const int INTERFACE_RUNNING_CHECK_FREQUENCY_MS = 1000;
const QString ASSET_SERVER_LOGGING_TARGET_NAME = "asset-server";
static const QStringList BAKEABLE_MODEL_EXTENSIONS = { "fbx" };
static const QList<QByteArray> BAKEABLE_TEXTURE_EXTENSIONS = QImageReader::supportedImageFormats();
static const QString BAKED_MODEL_SIMPLE_NAME = "asset.fbx";
static const QString BAKED_TEXTURE_SIMPLE_NAME = "texture.ktx";
BakeAssetTask::BakeAssetTask(const QString& assetHash, const QString& assetPath, const QString& filePath)
: _assetHash(assetHash), _assetPath(assetPath), _filePath(filePath) {
}
void BakeAssetTask::run() {
qRegisterMetaType<QVector<QString> >("QVector<QString>");
TextureBakerThreadGetter fn = []() -> QThread* { return QThread::currentThread(); };
if (_filePath.endsWith(".fbx")) {
FBXBaker baker(QUrl("file:///" + _filePath), fn, PathUtils::generateTemporaryDir());
QEventLoop loop;
connect(&baker, &Baker::finished, &loop, &QEventLoop::quit);
QMetaObject::invokeMethod(&baker, "bake", Qt::QueuedConnection);
qDebug() << "Running the bake!";
loop.exec();
qDebug() << "Finished baking: " << _assetHash << _assetPath << baker.getOutputFiles();
emit bakeComplete(_assetHash, _assetPath, QVector<QString>::fromStdVector(baker.getOutputFiles()));
} else {
TextureBaker baker(QUrl("file:///" + _filePath), image::TextureUsage::CUBE_TEXTURE, PathUtils::generateTemporaryDir());
QEventLoop loop;
connect(&baker, &Baker::finished, &loop, &QEventLoop::quit);
QMetaObject::invokeMethod(&baker, "bake", Qt::QueuedConnection);
qDebug() << "Running the bake!";
loop.exec();
qDebug() << "Finished baking: " << _assetHash << _assetPath << baker.getBakedTextureFileName();
emit bakeComplete(_assetHash, _assetPath, { baker.getDestinationFilePath() });
}
}
void AssetServer::bakeAsset(const QString& assetHash, const QString& assetPath, const QString& filePath) {
qDebug() << "Starting bake for: " << assetPath << assetHash;
auto it = _pendingBakes.find(assetHash);
if (it == _pendingBakes.end()) {
auto task = std::make_shared<BakeAssetTask>(assetHash, assetPath, filePath);
task->setAutoDelete(false);
_pendingBakes[assetHash] = task;
connect(task.get(), &BakeAssetTask::bakeComplete, this, [this, assetPath](QString assetHash, QString assetPath, QVector<QString> outputFiles) {
handleCompletedBake(assetPath, assetHash, outputFiles);
});
_bakingTaskPool.start(task.get());
} else {
qDebug() << "Already in queue";
}
}
QString AssetServer::getPathToAssetHash(const AssetHash& assetHash) {
return _filesDirectory.absoluteFilePath(assetHash);
}
void AssetServer::bakeAssets() {
auto it = _fileMappings.cbegin();
for (; it != _fileMappings.cend(); ++it) {
auto path = it.key();
auto hash = it.value().toString();
maybeBake(path, hash);
}
}
void AssetServer::maybeBake(const AssetPath& path, const AssetHash& hash) {
if (needsToBeBaked(path, hash)) {
qDebug() << "Queuing bake of: " << path;
bakeAsset(hash, path, getPathToAssetHash(hash));
}
}
void AssetServer::createEmptyMetaFile(const AssetHash& hash) {
QString metaFilePath = "atp:/" + hash + "/meta.json";
QFile metaFile { metaFilePath };
if (!metaFile.exists()) {
qDebug() << "Creating metfaile for " << hash;
if (metaFile.open(QFile::WriteOnly)) {
qDebug() << "Created metfaile for " << hash;
metaFile.write("{}");
}
}
}
bool AssetServer::hasMetaFile(const AssetHash& hash) {
QString metaFilePath = "/.baked/" + hash + "/meta.json";
qDebug() << "in mappings?" << metaFilePath;
return _fileMappings.contains(metaFilePath);
}
bool AssetServer::needsToBeBaked(const AssetPath& path, const AssetHash& assetHash) {
if (path.startsWith("/.baked/")) {
return false;
}
auto dotIndex = path.lastIndexOf(".");
if (dotIndex == -1) {
return false;
}
auto extension = path.mid(dotIndex + 1);
QString bakedFilename;
if (BAKEABLE_MODEL_EXTENSIONS.contains(extension)) {
bakedFilename = BAKED_MODEL_SIMPLE_NAME;
} else if (BAKEABLE_TEXTURE_EXTENSIONS.contains(extension.toLocal8Bit()) && hasMetaFile(assetHash)) {
bakedFilename = BAKED_TEXTURE_SIMPLE_NAME;
} else {
return false;
}
auto bakedPath = "/.baked/" + assetHash + "/" + bakedFilename;
return !_fileMappings.contains(bakedPath);
}
bool interfaceRunning() {
bool result = false;
@ -76,13 +203,15 @@ void updateConsumedCores() {
AssetServer::AssetServer(ReceivedMessage& message) :
ThreadedAssignment(message),
_taskPool(this)
_transferTaskPool(this),
_bakingTaskPool(this)
{
// Most of the work will be I/O bound, reading from disk and constructing packet objects,
// so the ideal is greater than the number of cores on the system.
static const int TASK_POOL_THREAD_COUNT = 50;
_taskPool.setMaxThreadCount(TASK_POOL_THREAD_COUNT);
_transferTaskPool.setMaxThreadCount(TASK_POOL_THREAD_COUNT);
_bakingTaskPool.setMaxThreadCount(1);
auto& packetReceiver = DependencyManager::get<NodeList>()->getPacketReceiver();
packetReceiver.registerListener(PacketType::AssetGet, this, "handleAssetGet");
@ -196,6 +325,8 @@ void AssetServer::completeSetup() {
}
nodeList->addSetOfNodeTypesToNodeInterestSet({ NodeType::Agent, NodeType::EntityScriptServer });
bakeAssets();
} else {
qCritical() << "Asset Server assignment will not continue because mapping file could not be loaded.";
setFinished(true);
@ -265,27 +396,27 @@ void AssetServer::handleAssetMappingOperation(QSharedPointer<ReceivedMessage> me
nodeList->sendPacketList(std::move(replyPacket), *senderNode);
}
static const QStringList BAKEABLE_MODEL_EXTENSIONS = { ".fbx" };
static const QString BAKED_MODEL_SIMPLE_NAME = "asset.fbx";
static const QString BAKED_TEXTURE_SIMPLE_NAME = "texture.ktx";
void AssetServer::handleGetMappingOperation(ReceivedMessage& message, SharedNodePointer senderNode, NLPacketList& replyPacket) {
QString assetPath = message.readString();
QUrl url { assetPath };
assetPath = url.path();
auto it = _fileMappings.find(assetPath);
if (it != _fileMappings.end()) {
// check if we should re-direct to a baked asset
// first, figure out from the mapping extension what type of file this is
auto assetPathExtension = assetPath.mid(assetPath.lastIndexOf('.')).toLower();
auto assetPathExtension = assetPath.mid(assetPath.lastIndexOf('.') + 1).toLower();
QString bakedRootFile;
if (BAKEABLE_MODEL_EXTENSIONS.contains(assetPathExtension)) {
bakedRootFile = BAKED_MODEL_SIMPLE_NAME;
} else if (QImageReader::supportedImageFormats().contains(assetPathExtension.toLocal8Bit())) {
} else if (BAKEABLE_TEXTURE_EXTENSIONS.contains(assetPathExtension.toLocal8Bit())) {
bakedRootFile = BAKED_TEXTURE_SIMPLE_NAME;
}
qDebug() << bakedRootFile << assetPathExtension;
auto originalAssetHash = it->toString();
QString redirectedAssetHash;
@ -298,9 +429,12 @@ void AssetServer::handleGetMappingOperation(ReceivedMessage& message, SharedNode
auto bakedIt = _fileMappings.find(bakedAssetPath);
if (bakedIt != _fileMappings.end()) {
qDebug() << "Did find baked version for: " << originalAssetHash << assetPath;
// we found a baked version of the requested asset to serve, redirect to that
redirectedAssetHash = bakedIt->toString();
wasRedirected = true;
} else {
qDebug() << "Did not find baked version for: " << originalAssetHash << assetPath;
}
}
@ -319,6 +453,15 @@ void AssetServer::handleGetMappingOperation(ReceivedMessage& message, SharedNode
} else {
replyPacket.write(QByteArray::fromHex(originalAssetHash.toUtf8()));
replyPacket.writePrimitive(wasRedirected);
auto query = QUrlQuery(url.query());
bool isSkybox = query.hasQueryItem("skybox");
qDebug() << "Is skybox? " << isSkybox;
if (isSkybox) {
createMetaFile(originalAssetHash);
maybeBake(originalAssetHash, assetPath);
}
}
} else {
replyPacket.writePrimitive(AssetServerError::AssetNotFound);
@ -437,7 +580,7 @@ void AssetServer::handleAssetGet(QSharedPointer<ReceivedMessage> message, Shared
// Queue task
auto task = new SendAssetTask(message, senderNode, _filesDirectory);
_taskPool.start(task);
_transferTaskPool.start(task);
}
void AssetServer::handleAssetUpload(QSharedPointer<ReceivedMessage> message, SharedNodePointer senderNode) {
@ -446,7 +589,7 @@ void AssetServer::handleAssetUpload(QSharedPointer<ReceivedMessage> message, Sha
qDebug() << "Starting an UploadAssetTask for upload from" << uuidStringWithoutCurlyBraces(senderNode->getUUID());
auto task = new UploadAssetTask(message, senderNode, _filesDirectory);
_taskPool.start(task);
_transferTaskPool.start(task);
} else {
// this is a node the domain told us is not allowed to rez entities
// for now this also means it isn't allowed to add assets
@ -632,6 +775,7 @@ bool AssetServer::setMapping(AssetPath path, AssetHash hash) {
if (writeMappingsToFile()) {
// persistence succeeded, we are good to go
qDebug() << "Set mapping:" << path << "=>" << hash;
maybeBake(path, hash);
return true;
} else {
// failed to persist this mapping to file - put back the old one in our in-memory representation
@ -836,18 +980,17 @@ bool AssetServer::renameMapping(AssetPath oldPath, AssetPath newPath) {
static const QString HIDDEN_BAKED_CONTENT_FOLDER = "/.baked/";
void AssetServer::handleCompletedBake(AssetHash originalAssetHash, QDir temporaryOutputDir) {
// enumerate the baking result files in the temporary directory
QDirIterator dirIterator(temporaryOutputDir.absolutePath(), QDir::Files, QDirIterator::Subdirectories);
void AssetServer::handleCompletedBake(AssetPath originalAssetPath, AssetHash originalAssetHash, QVector<QString> bakedFilePaths) {
bool errorCompletingBake { false };
while (dirIterator.hasNext()) {
QString filePath = dirIterator.next();
qDebug() << "Completing bake for " << originalAssetHash;
for (auto& filePath: bakedFilePaths) {
// figure out the hash for the contents of this file
QFile file(filePath);
qDebug() << "File path: " << filePath;
AssetHash bakedFileHash;
if (file.open(QIODevice::ReadOnly)) {
@ -873,26 +1016,33 @@ void AssetServer::handleCompletedBake(AssetHash originalAssetHash, QDir temporar
}
// setup the mapping for this bake file
auto relativeFilePath = temporaryOutputDir.relativeFilePath(filePath);
static const QString BAKED_ASSET_SIMPLE_NAME = "asset.fbx";
auto relativeFilePath = QUrl(filePath).fileName();
qDebug() << "Relative file path is: " << relativeFilePath;
static const QString BAKED_ASSET_SIMPLE_FBX_NAME = "asset.fbx";
static const QString BAKED_ASSET_SIMPLE_TEXTURE_NAME = "texture.ktx";
if (relativeFilePath.endsWith(".fbx", Qt::CaseInsensitive)) {
// for an FBX file, we replace the filename with the simple name
// (to handle the case where two mapped assets have the same hash but different names)
relativeFilePath = BAKED_ASSET_SIMPLE_NAME;
relativeFilePath = BAKED_ASSET_SIMPLE_FBX_NAME;
} else if (!originalAssetPath.endsWith(".fbx")) {
relativeFilePath = BAKED_ASSET_SIMPLE_TEXTURE_NAME;
}
QString bakeMapping = HIDDEN_BAKED_CONTENT_FOLDER + originalAssetHash + "/" + relativeFilePath;
// add a mapping (under the hidden baked folder) for this file resulting from the bake
if (setMapping(bakeMapping , bakedFileHash)) {
if (setMapping(bakeMapping, bakedFileHash)) {
qDebug() << "Added" << bakeMapping << "for bake file" << bakedFileHash << "from bake of" << originalAssetHash;
} else {
qDebug() << "Failed to set mapping";
// stop handling this bake, couldn't add a mapping for this bake file
errorCompletingBake = true;
break;
}
} else {
qDebug() << "Failed to open baked file: " << filePath;
// stop handling this bake, we couldn't open one of the files for reading
errorCompletingBake = true;
break;

35
assignment-client/src/assets/AssetServer.h
View file

@ -14,12 +14,29 @@
#include <QtCore/QDir>
#include <QtCore/QThreadPool>
#include <QRunnable>
#include <ThreadedAssignment.h>
#include "AssetUtils.h"
#include "ReceivedMessage.h"
class BakeAssetTask : public QObject, public QRunnable {
Q_OBJECT
public:
BakeAssetTask(const QString& assetHash, const QString& assetPath, const QString& filePath);
void run() override;
signals:
void bakeComplete(QString assetHash, QString assetPath, QVector<QString> outputFiles);
private:
QString _assetHash;
QString _assetPath;
QString _filePath;
};
class AssetServer : public ThreadedAssignment {
Q_OBJECT
public:
@ -63,8 +80,17 @@ private:
/// Delete any unmapped files from the local asset directory
void cleanupUnmappedFiles();
QString getPathToAssetHash(const AssetHash& assetHash);
void bakeAssets();
void maybeBake(const AssetPath& path, const AssetHash& hash);
void createEmptyMetaFile(const AssetHash& hash);
bool hasMetaFile(const AssetHash& hash);
bool needsToBeBaked(const AssetPath& path, const AssetHash& assetHash);
void bakeAsset(const QString& assetHash, const QString& assetPath, const QString& filePath);
/// Move baked content for asset to baked directory and update baked status
void handleCompletedBake(AssetHash originalAssetHash, QDir temporaryOutputDir);
void handleCompletedBake(AssetPath assetPath, AssetHash originalAssetHash, QVector<QString> bakedFilePaths);
/// Create meta file to describe baked content for original asset
bool createMetaFile(AssetHash originalAssetHash);
@ -73,7 +99,12 @@ private:
QDir _resourcesDirectory;
QDir _filesDirectory;
QThreadPool _taskPool;
/// Task pool for handling uploads and downloads of assets
QThreadPool _transferTaskPool;
QHash<QString, std::shared_ptr<BakeAssetTask>> _pendingBakes;
QThreadPool _bakingTaskPool;
};
#endif

2
cmake/modules/FindFBX.cmake
View file

@ -57,7 +57,7 @@ endif()
function(_fbx_find_library _name _lib _suffix)
if (MSVC_VERSION EQUAL 1910)
set(VS_PREFIX vs2017)
set(VS_PREFIX vs2015)
elseif (MSVC_VERSION EQUAL 1900)
set(VS_PREFIX vs2015)
elseif (MSVC_VERSION EQUAL 1800)

17
libraries/baking/CMakeLists.txt Normal file
View file

@ -0,0 +1,17 @@
set(TARGET_NAME baking)
setup_hifi_library(Concurrent)
find_package(FBX)
if (FBX_FOUND)
if (CMAKE_THREAD_LIBS_INIT)
target_link_libraries(${TARGET_NAME} ${FBX_LIBRARIES} "${CMAKE_THREAD_LIBS_INIT}")
else ()
target_link_libraries(${TARGET_NAME} ${FBX_LIBRARIES})
endif ()
target_include_directories(${TARGET_NAME} SYSTEM PRIVATE ${FBX_INCLUDE_DIR})
endif ()
set_target_properties(${TARGET_NAME} PROPERTIES EXCLUDE_FROM_ALL TRUE EXCLUDE_FROM_DEFAULT_BUILD TRUE)
link_hifi_libraries(shared model networking ktx image)
include_hifi_library_headers(gpu)

0
tools/oven/src/Baker.cpp → libraries/baking/src/Baker.cpp
View file

0
tools/oven/src/Baker.h → libraries/baking/src/Baker.h
View file

568
libraries/baking/src/FBXBaker.cpp Normal file
View file

@ -0,0 +1,568 @@
//
// FBXBaker.cpp
// tools/oven/src
//
// Created by Stephen Birarda on 3/30/17.
// Copyright 2017 High Fidelity, Inc.
//
// Distributed under the Apache License, Version 2.0.
// See the accompanying file LICENSE or http://www.apache.org/licenses/LICENSE-2.0.html
//
#include <cmath> // need this include so we don't get an error looking for std::isnan
#include <fbxsdk.h>
#include <QtConcurrent>
#include <QtCore/QCoreApplication>
#include <QtCore/QDir>
#include <QtCore/QEventLoop>
#include <QtCore/QFileInfo>
#include <QtCore/QThread>
#include <mutex>
#include <NetworkAccessManager.h>
#include <SharedUtil.h>
#include <PathUtils.h>
#include "ModelBakingLoggingCategory.h"
#include "TextureBaker.h"
#include "FBXBaker.h"
std::once_flag onceFlag;
FBXSDKManagerUniquePointer FBXBaker::_sdkManager { nullptr };
FBXBaker::FBXBaker(const QUrl& fbxURL, TextureBakerThreadGetter textureThreadGetter,
const QString& bakedOutputDir, const QString& originalOutputDir) :
_fbxURL(fbxURL),
_bakedOutputDir(bakedOutputDir),
_originalOutputDir(originalOutputDir),
_textureThreadGetter(textureThreadGetter)
{
std::call_once(onceFlag, [](){
// create the static FBX SDK manager
_sdkManager = FBXSDKManagerUniquePointer(FbxManager::Create(), [](FbxManager* manager){
manager->Destroy();
});
});
}
void FBXBaker::bake() {
auto tempDir = PathUtils::generateTemporaryDir();
if (tempDir.isEmpty()) {
handleError("Failed to create a temporary directory.");
return;
}
_tempDir = tempDir;
_originalFBXFilePath = _tempDir.filePath(_fbxURL.fileName());
qDebug() << "Made temporary dir " << _tempDir;
qDebug() << "Origin file path: " << _originalFBXFilePath;
// setup the output folder for the results of this bake
setupOutputFolder();
if (hasErrors()) {
return;
}
connect(this, &FBXBaker::sourceCopyReadyToLoad, this, &FBXBaker::bakeSourceCopy);
// make a local copy of the FBX file
loadSourceFBX();
}
void FBXBaker::bakeSourceCopy() {
// load the scene from the FBX file
importScene();
if (hasErrors()) {
return;
}
// enumerate the textures found in the scene and start a bake for them
rewriteAndBakeSceneTextures();
if (hasErrors()) {
return;
}
// export the FBX with re-written texture references
exportScene();
if (hasErrors()) {
return;
}
// check if we're already done with textures (in case we had none to re-write)
checkIfTexturesFinished();
}
void FBXBaker::setupOutputFolder() {
// make sure there isn't already an output directory using the same name
int iteration = 0;
if (QDir(_bakedOutputDir).exists()) {
qWarning() << "Output path" << _bakedOutputDir << "already exists. Continuing.";
//_bakedOutputDir = _baseOutputPath + "/" + _fbxName + "-" + QString::number(++iteration) + "/";
} else {
qCDebug(model_baking) << "Creating FBX output folder" << _bakedOutputDir;
// attempt to make the output folder
if (!QDir().mkpath(_bakedOutputDir)) {
handleError("Failed to create FBX output folder " + _bakedOutputDir);
return;
}
// attempt to make the output folder
if (!QDir().mkpath(_originalOutputDir)) {
handleError("Failed to create FBX output folder " + _bakedOutputDir);
return;
}
}
}
void FBXBaker::loadSourceFBX() {
// check if the FBX is local or first needs to be downloaded
if (_fbxURL.isLocalFile()) {
// load up the local file
QFile localFBX { _fbxURL.toLocalFile() };
qDebug() << "Local file url: " << _fbxURL << _fbxURL.toString() << _fbxURL.toLocalFile() << ", copying to: " << _originalFBXFilePath;
if (!localFBX.exists()) {
//QMessageBox::warning(this, "Could not find " + _fbxURL.toString(), "");
handleError("Could not find " + _fbxURL.toString());
return;
}
// make a copy in the output folder
if (!_originalOutputDir.isEmpty()) {
qDebug() << "Copying to: " << _originalOutputDir << "/" << _fbxURL.fileName();
localFBX.copy(_originalOutputDir + "/" + _fbxURL.fileName());
}
localFBX.copy(_originalFBXFilePath);
// emit our signal to start the import of the FBX source copy
emit sourceCopyReadyToLoad();
} else {
// remote file, kick off a download
auto& networkAccessManager = NetworkAccessManager::getInstance();
QNetworkRequest networkRequest;
// setup the request to follow re-directs and always hit the network
networkRequest.setAttribute(QNetworkRequest::FollowRedirectsAttribute, true);
networkRequest.setAttribute(QNetworkRequest::CacheLoadControlAttribute, QNetworkRequest::AlwaysNetwork);
networkRequest.setHeader(QNetworkRequest::UserAgentHeader, HIGH_FIDELITY_USER_AGENT);
networkRequest.setUrl(_fbxURL);
qCDebug(model_baking) << "Downloading" << _fbxURL;
auto networkReply = networkAccessManager.get(networkRequest);
connect(networkReply, &QNetworkReply::finished, this, &FBXBaker::handleFBXNetworkReply);
}
}
void FBXBaker::handleFBXNetworkReply() {
auto requestReply = qobject_cast<QNetworkReply*>(sender());
if (requestReply->error() == QNetworkReply::NoError) {
qCDebug(model_baking) << "Downloaded" << _fbxURL;
// grab the contents of the reply and make a copy in the output folder
QFile copyOfOriginal(_originalFBXFilePath);
qDebug(model_baking) << "Writing copy of original FBX to" << _originalFBXFilePath << copyOfOriginal.fileName();
if (!copyOfOriginal.open(QIODevice::WriteOnly)) {
// add an error to the error list for this FBX stating that a duplicate of the original FBX could not be made
handleError("Could not create copy of " + _fbxURL.toString() + " (Failed to open " + _originalFBXFilePath + ")");
return;
}
if (copyOfOriginal.write(requestReply->readAll()) == -1) {
handleError("Could not create copy of " + _fbxURL.toString() + " (Failed to write)");
return;
}
// close that file now that we are done writing to it
copyOfOriginal.close();
if (!_originalOutputDir.isEmpty()) {
copyOfOriginal.copy(_originalOutputDir + "/" + _fbxURL.fileName());
}
// emit our signal to start the import of the FBX source copy
emit sourceCopyReadyToLoad();
} else {
// add an error to our list stating that the FBX could not be downloaded
handleError("Failed to download " + _fbxURL.toString());
}
}
void FBXBaker::importScene() {
// create an FBX SDK importer
FbxImporter* importer = FbxImporter::Create(_sdkManager.get(), "");
qDebug() << "file path: " << _originalFBXFilePath.toLocal8Bit().data() << QDir(_originalFBXFilePath).exists();
// import the copy of the original FBX file
bool importStatus = importer->Initialize(_originalFBXFilePath.toLocal8Bit().data());
if (!importStatus) {
// failed to initialize importer, print an error and return
handleError("Failed to import " + _fbxURL.toString() + " - " + importer->GetStatus().GetErrorString());
return;
} else {
qCDebug(model_baking) << "Imported" << _fbxURL << "to FbxScene";
}
// setup a new scene to hold the imported file
_scene = FbxScene::Create(_sdkManager.get(), "bakeScene");
// import the file to the created scene
importer->Import(_scene);
// destroy the importer that is no longer needed
importer->Destroy();
}
QString texturePathRelativeToFBX(QUrl fbxURL, QUrl textureURL) {
auto fbxPath = fbxURL.toString(QUrl::RemoveFilename | QUrl::RemoveQuery | QUrl::RemoveFragment);
auto texturePath = textureURL.toString(QUrl::RemoveFilename | QUrl::RemoveQuery | QUrl::RemoveFragment);
if (texturePath.startsWith(fbxPath)) {
// texture path is a child of the FBX path, return the texture path without the fbx path
return texturePath.mid(fbxPath.length());
} else {
// the texture path was not a child of the FBX path, return the empty string
return "";
}
}
QString FBXBaker::createBakedTextureFileName(const QFileInfo& textureFileInfo) {
// first make sure we have a unique base name for this texture
// in case another texture referenced by this model has the same base name
auto nameMatches = _textureNameMatchCount[textureFileInfo.baseName()];
QString bakedTextureFileName { textureFileInfo.completeBaseName() };
if (nameMatches > 0) {
// there are already nameMatches texture with this name
// append - and that number to our baked texture file name so that it is unique
bakedTextureFileName += "-" + QString::number(nameMatches);
}
bakedTextureFileName += BAKED_TEXTURE_EXT;
// increment the number of name matches
++nameMatches;
return bakedTextureFileName;
}
QUrl FBXBaker::getTextureURL(const QFileInfo& textureFileInfo, FbxFileTexture* fileTexture) {
QUrl urlToTexture;
if (textureFileInfo.exists() && textureFileInfo.isFile()) {
// set the texture URL to the local texture that we have confirmed exists
urlToTexture = QUrl::fromLocalFile(textureFileInfo.absoluteFilePath());
} else {
// external texture that we'll need to download or find
// first check if it the RelativePath to the texture in the FBX was relative
QString relativeFileName = fileTexture->GetRelativeFileName();
auto apparentRelativePath = QFileInfo(relativeFileName.replace("\\", "/"));
// this is a relative file path which will require different handling
// depending on the location of the original FBX
if (_fbxURL.isLocalFile() && apparentRelativePath.exists() && apparentRelativePath.isFile()) {
// the absolute path we ran into for the texture in the FBX exists on this machine
// so use that file
urlToTexture = QUrl::fromLocalFile(apparentRelativePath.absoluteFilePath());
} else {
// we didn't find the texture on this machine at the absolute path
// so assume that it is right beside the FBX to match the behaviour of interface
urlToTexture = _fbxURL.resolved(apparentRelativePath.fileName());
}
}
return urlToTexture;
}
image::TextureUsage::Type textureTypeForMaterialProperty(FbxProperty& property, FbxSurfaceMaterial* material) {
using namespace image::TextureUsage;
// this is a property we know has a texture, we need to match it to a High Fidelity known texture type
// since that information is passed to the baking process
// grab the hierarchical name for this property and lowercase it for case-insensitive compare
auto propertyName = QString(property.GetHierarchicalName()).toLower();
// figure out the type of the property based on what known value string it matches
if ((propertyName.contains("diffuse") && !propertyName.contains("tex_global_diffuse"))
|| propertyName.contains("tex_color_map")) {
return ALBEDO_TEXTURE;
} else if (propertyName.contains("transparentcolor") || propertyName.contains("transparencyfactor")) {
return ALBEDO_TEXTURE;
} else if (propertyName.contains("bump")) {
return BUMP_TEXTURE;
} else if (propertyName.contains("normal")) {
return NORMAL_TEXTURE;
} else if ((propertyName.contains("specular") && !propertyName.contains("tex_global_specular"))
|| propertyName.contains("reflection")) {
return SPECULAR_TEXTURE;
} else if (propertyName.contains("tex_metallic_map")) {
return METALLIC_TEXTURE;
} else if (propertyName.contains("shininess")) {
return GLOSS_TEXTURE;
} else if (propertyName.contains("tex_roughness_map")) {
return ROUGHNESS_TEXTURE;
} else if (propertyName.contains("emissive")) {
return EMISSIVE_TEXTURE;
} else if (propertyName.contains("ambientcolor")) {
return LIGHTMAP_TEXTURE;
} else if (propertyName.contains("ambientfactor")) {
// we need to check what the ambient factor is, since that tells Interface to process this texture
// either as an occlusion texture or a light map
auto lambertMaterial = FbxCast<FbxSurfaceLambert>(material);
if (lambertMaterial->AmbientFactor == 0) {
return LIGHTMAP_TEXTURE;
} else if (lambertMaterial->AmbientFactor > 0) {
return OCCLUSION_TEXTURE;
} else {
return UNUSED_TEXTURE;
}
} else if (propertyName.contains("tex_ao_map")) {
return OCCLUSION_TEXTURE;
}
return UNUSED_TEXTURE;
}
void FBXBaker::rewriteAndBakeSceneTextures() {
// enumerate the surface materials to find the textures used in the scene
int numMaterials = _scene->GetMaterialCount();
for (int i = 0; i < numMaterials; i++) {
FbxSurfaceMaterial* material = _scene->GetMaterial(i);
if (material) {
// enumerate the properties of this material to see what texture channels it might have
FbxProperty property = material->GetFirstProperty();
while (property.IsValid()) {
// first check if this property has connected textures, if not we don't need to bother with it here
if (property.GetSrcObjectCount<FbxTexture>() > 0) {
// figure out the type of texture from the material property
auto textureType = textureTypeForMaterialProperty(property, material);
if (textureType != image::TextureUsage::UNUSED_TEXTURE) {
int numTextures = property.GetSrcObjectCount<FbxFileTexture>();
for (int j = 0; j < numTextures; j++) {
FbxFileTexture* fileTexture = property.GetSrcObject<FbxFileTexture>(j);
// use QFileInfo to easily split up the existing texture filename into its components
QString fbxTextureFileName { fileTexture->GetFileName() };
QFileInfo textureFileInfo { fbxTextureFileName.replace("\\", "/") };
// make sure this texture points to something and isn't one we've already re-mapped
if (!textureFileInfo.filePath().isEmpty()
&& textureFileInfo.suffix() != BAKED_TEXTURE_EXT.mid(1)) {
// construct the new baked texture file name and file path
// ensuring that the baked texture will have a unique name
// even if there was another texture with the same name at a different path
auto bakedTextureFileName = createBakedTextureFileName(textureFileInfo);
QString bakedTextureFilePath {
_bakedOutputDir + "/" + bakedTextureFileName
};
_outputFiles.push_back(bakedTextureFilePath);
qCDebug(model_baking).noquote() << "Re-mapping" << fileTexture->GetFileName()
<< "to" << bakedTextureFilePath;
// figure out the URL to this texture, embedded or external
auto urlToTexture = getTextureURL(textureFileInfo, fileTexture);
// write the new filename into the FBX scene
fileTexture->SetFileName(bakedTextureFilePath.toUtf8().data());
// write the relative filename to be the baked texture file name since it will
// be right beside the FBX
fileTexture->SetRelativeFileName(bakedTextureFileName.toLocal8Bit().constData());
if (!_bakingTextures.contains(urlToTexture)) {
// bake this texture asynchronously
bakeTexture(urlToTexture, textureType, _bakedOutputDir);
}
}
}
}
}
property = material->GetNextProperty(property);
}
}
}
}
void FBXBaker::bakeTexture(const QUrl& textureURL, image::TextureUsage::Type textureType, const QDir& outputDir) {
// start a bake for this texture and add it to our list to keep track of
QSharedPointer<TextureBaker> bakingTexture {
new TextureBaker(textureURL, textureType, outputDir),
&TextureBaker::deleteLater
};
// make sure we hear when the baking texture is done
connect(bakingTexture.data(), &Baker::finished, this, &FBXBaker::handleBakedTexture);
// keep a shared pointer to the baking texture
_bakingTextures.insert(textureURL, bakingTexture);
// start baking the texture on one of our available worker threads
bakingTexture->moveToThread(_textureThreadGetter());
QMetaObject::invokeMethod(bakingTexture.data(), "bake");
}
void FBXBaker::handleBakedTexture() {
TextureBaker* bakedTexture = qobject_cast<TextureBaker*>(sender());
// make sure we haven't already run into errors, and that this is a valid texture
if (bakedTexture) {
if (!hasErrors()) {
if (!bakedTexture->hasErrors()) {
if (!_originalOutputDir.isEmpty()) {
// we've been asked to make copies of the originals, so we need to make copies of this if it is a linked texture
// use the path to the texture being baked to determine if this was an embedded or a linked texture
// it is embeddded if the texure being baked was inside the original output folder
// since that is where the FBX SDK places the .fbm folder it generates when importing the FBX
auto originalOutputFolder = QUrl::fromLocalFile(_originalOutputDir);
if (!originalOutputFolder.isParentOf(bakedTexture->getTextureURL())) {
// for linked textures we want to save a copy of original texture beside the original FBX
qCDebug(model_baking) << "Saving original texture for" << bakedTexture->getTextureURL();
// check if we have a relative path to use for the texture
auto relativeTexturePath = texturePathRelativeToFBX(_fbxURL, bakedTexture->getTextureURL());
QFile originalTextureFile {
_originalOutputDir + "/" + relativeTexturePath + bakedTexture->getTextureURL().fileName()
};
if (relativeTexturePath.length() > 0) {
// make the folders needed by the relative path
}
if (originalTextureFile.open(QIODevice::WriteOnly) && originalTextureFile.write(bakedTexture->getOriginalTexture()) != -1) {
qCDebug(model_baking) << "Saved original texture file" << originalTextureFile.fileName()
<< "for" << _fbxURL;
} else {
handleError("Could not save original external texture " + originalTextureFile.fileName()
+ " for " + _fbxURL.toString());
return;
}
}
}
// now that this texture has been baked and handled, we can remove that TextureBaker from our hash
_bakingTextures.remove(bakedTexture->getTextureURL());
checkIfTexturesFinished();
} else {
// there was an error baking this texture - add it to our list of errors
_errorList.append(bakedTexture->getErrors());
// we don't emit finished yet so that the other textures can finish baking first
_pendingErrorEmission = true;
// now that this texture has been baked, even though it failed, we can remove that TextureBaker from our list
_bakingTextures.remove(bakedTexture->getTextureURL());
checkIfTexturesFinished();
}
} else {
// we have errors to attend to, so we don't do extra processing for this texture
// but we do need to remove that TextureBaker from our list
// and then check if we're done with all textures
_bakingTextures.remove(bakedTexture->getTextureURL());
checkIfTexturesFinished();
}
}
}
void FBXBaker::exportScene() {
// setup the exporter
FbxExporter* exporter = FbxExporter::Create(_sdkManager.get(), "");
// save the relative path to this FBX inside our passed output folder
auto fileName = _fbxURL.fileName();
auto baseName = fileName.left(fileName.lastIndexOf('.'));
auto bakedFilename = baseName + BAKED_FBX_EXTENSION;
_bakedFBXFilePath = _bakedOutputDir + "/" + bakedFilename;
bool exportStatus = exporter->Initialize(_bakedFBXFilePath.toLocal8Bit().data());
if (!exportStatus) {
// failed to initialize exporter, print an error and return
handleError("Failed to export FBX file at " + _fbxURL.toString() + " to " + _bakedFBXFilePath
+ "- error: " + exporter->GetStatus().GetErrorString());
}
_outputFiles.push_back(_bakedFBXFilePath);
// export the scene
exporter->Export(_scene);
qCDebug(model_baking) << "Exported" << _fbxURL << "with re-written paths to" << _bakedFBXFilePath;
}
void FBXBaker::removeEmbeddedMediaFolder() {
// now that the bake is complete, remove the embedded media folder produced by the FBX SDK when it imports an FBX
//auto embeddedMediaFolderName = _fbxURL.fileName().replace(".fbx", ".fbm");
//QDir(_bakedOutputDir + ORIGINAL_OUTPUT_SUBFOLDER + embeddedMediaFolderName).removeRecursively();
}
void FBXBaker::checkIfTexturesFinished() {
// check if we're done everything we need to do for this FBX
// and emit our finished signal if we're done
if (_bakingTextures.isEmpty()) {
// remove the embedded media folder that the FBX SDK produces when reading the original
removeEmbeddedMediaFolder();
if (hasErrors()) {
// if we're checking for completion but we have errors
// that means one or more of our texture baking operations failed
if (_pendingErrorEmission) {
emit finished();
}
return;
} else {
qCDebug(model_baking) << "Finished baking, emitting finsihed" << _fbxURL;
emit finished();
}
}
}

105
libraries/baking/src/FBXBaker.h Normal file
View file

@ -0,0 +1,105 @@
//
// FBXBaker.h
// tools/oven/src
//
// Created by Stephen Birarda on 3/30/17.
// Copyright 2017 High Fidelity, Inc.
//
// Distributed under the Apache License, Version 2.0.
// See the accompanying file LICENSE or http://www.apache.org/licenses/LICENSE-2.0.html
//
#ifndef hifi_FBXBaker_h
#define hifi_FBXBaker_h
#include <QtCore/QFutureSynchronizer>
#include <QtCore/QDir>
#include <QtCore/QUrl>
#include <QtNetwork/QNetworkReply>
#include "Baker.h"
#include "TextureBaker.h"
#include "ModelBakingLoggingCategory.h"
#include <gpu/Texture.h>
namespace fbxsdk {
class FbxManager;
class FbxProperty;
class FbxScene;
class FbxFileTexture;
}
static const QString BAKED_FBX_EXTENSION = ".baked.fbx";
using FBXSDKManagerUniquePointer = std::unique_ptr<fbxsdk::FbxManager, std::function<void (fbxsdk::FbxManager *)>>;
using TextureBakerThreadGetter = std::function<QThread*()>;
class FBXBaker : public Baker {
Q_OBJECT
public:
FBXBaker(const QUrl& fbxURL, TextureBakerThreadGetter textureThreadGetter,
const QString& bakedOutputDir, const QString& originalOutputDir = "");
QUrl getFBXUrl() const { return _fbxURL; }
QString getBakedFBXFilePath() const { return _bakedFBXFilePath; }
std::vector<QString> getOutputFiles() const { return _outputFiles; }
public slots:
// all calls to FBXBaker::bake for FBXBaker instances must be from the same thread
// because the Autodesk SDK will cause a crash if it is called from multiple threads
virtual void bake() override;
signals:
void sourceCopyReadyToLoad();
private slots:
void bakeSourceCopy();
void handleFBXNetworkReply();
void handleBakedTexture();
private:
void setupOutputFolder();
void loadSourceFBX();
void importScene();
void rewriteAndBakeSceneTextures();
void exportScene();
void removeEmbeddedMediaFolder();
void checkIfTexturesFinished();
QString createBakedTextureFileName(const QFileInfo& textureFileInfo);
QUrl getTextureURL(const QFileInfo& textureFileInfo, fbxsdk::FbxFileTexture* fileTexture);
void bakeTexture(const QUrl& textureURL, image::TextureUsage::Type textureType, const QDir& outputDir);
QUrl _fbxURL;
QString _bakedFBXFilePath;
QString _bakedOutputDir;
// If set, the original FBX and textures will also be copied here
QString _originalOutputDir;
QDir _tempDir;
QString _originalFBXFilePath;
// List of baked output files, includes the FBX and textures
std::vector<QString> _outputFiles;
static FBXSDKManagerUniquePointer _sdkManager;
fbxsdk::FbxScene* _scene { nullptr };
QMultiHash<QUrl, QSharedPointer<TextureBaker>> _bakingTextures;
QHash<QString, int> _textureNameMatchCount;
TextureBakerThreadGetter _textureThreadGetter;
bool _pendingErrorEmission { false };
};
#endif // hifi_FBXBaker_h

0
tools/oven/src/ModelBakingLoggingCategory.cpp → libraries/baking/src/ModelBakingLoggingCategory.cpp
View file

0
tools/oven/src/ModelBakingLoggingCategory.h → libraries/baking/src/ModelBakingLoggingCategory.h
View file

131
libraries/baking/src/TextureBaker.cpp Normal file
View file

@ -0,0 +1,131 @@
//
// TextureBaker.cpp
// tools/oven/src
//
// Created by Stephen Birarda on 4/5/17.
// Copyright 2017 High Fidelity, Inc.
//
// Distributed under the Apache License, Version 2.0.
// See the accompanying file LICENSE or http://www.apache.org/licenses/LICENSE-2.0.html
//
#include <QtCore/QDir>
#include <QtCore/QEventLoop>
#include <QtCore/QFile>
#include <QtNetwork/QNetworkReply>
#include <image/Image.h>
#include <ktx/KTX.h>
#include <NetworkAccessManager.h>
#include <SharedUtil.h>
#include "ModelBakingLoggingCategory.h"
#include "TextureBaker.h"
const QString BAKED_TEXTURE_EXT = ".ktx";
TextureBaker::TextureBaker(const QUrl& textureURL, image::TextureUsage::Type textureType, const QDir& outputDirectory) :
_textureURL(textureURL),
_textureType(textureType),
_outputDirectory(outputDirectory)
{
// figure out the baked texture filename
auto originalFilename = textureURL.fileName();
_bakedTextureFileName = originalFilename.left(originalFilename.lastIndexOf('.')) + BAKED_TEXTURE_EXT;
}
void TextureBaker::bake() {
// once our texture is loaded, kick off a the processing
connect(this, &TextureBaker::originalTextureLoaded, this, &TextureBaker::processTexture);
// first load the texture (either locally or remotely)
loadTexture();
}
void TextureBaker::loadTexture() {
// check if the texture is local or first needs to be downloaded
if (_textureURL.isLocalFile()) {
// load up the local file
QFile localTexture { _textureURL.toLocalFile() };
if (!localTexture.open(QIODevice::ReadOnly)) {
handleError("Unable to open texture " + _textureURL.toString());
return;
}
_originalTexture = localTexture.readAll();
emit originalTextureLoaded();
} else {
// remote file, kick off a download
auto& networkAccessManager = NetworkAccessManager::getInstance();
QNetworkRequest networkRequest;
// setup the request to follow re-directs and always hit the network
networkRequest.setAttribute(QNetworkRequest::FollowRedirectsAttribute, true);
networkRequest.setAttribute(QNetworkRequest::CacheLoadControlAttribute, QNetworkRequest::AlwaysNetwork);
networkRequest.setHeader(QNetworkRequest::UserAgentHeader, HIGH_FIDELITY_USER_AGENT);
networkRequest.setUrl(_textureURL);
qCDebug(model_baking) << "Downloading" << _textureURL;
// kickoff the download, wait for slot to tell us it is done
auto networkReply = networkAccessManager.get(networkRequest);
connect(networkReply, &QNetworkReply::finished, this, &TextureBaker::handleTextureNetworkReply);
}
}
void TextureBaker::handleTextureNetworkReply() {
auto requestReply = qobject_cast<QNetworkReply*>(sender());
if (requestReply->error() == QNetworkReply::NoError) {
qCDebug(model_baking) << "Downloaded texture" << _textureURL;
// store the original texture so it can be passed along for the bake
_originalTexture = requestReply->readAll();
emit originalTextureLoaded();
} else {
// add an error to our list stating that this texture could not be downloaded
handleError("Error downloading " + _textureURL.toString() + " - " + requestReply->errorString());
}
}
void TextureBaker::processTexture() {
auto processedTexture = image::processImage(_originalTexture, _textureURL.toString().toStdString(),
ABSOLUTE_MAX_TEXTURE_NUM_PIXELS, _textureType);
if (!processedTexture) {
handleError("Could not process texture " + _textureURL.toString());
return;
}
// the baked textures need to have the source hash added for cache checks in Interface
// so we add that to the processed texture before handling it off to be serialized
auto hashData = QCryptographicHash::hash(_originalTexture, QCryptographicHash::Md5);
std::string hash = hashData.toHex().toStdString();
processedTexture->setSourceHash(hash);
auto memKTX = gpu::Texture::serialize(*processedTexture);
if (!memKTX) {
handleError("Could not serialize " + _textureURL.toString() + " to KTX");
return;
}
const char* data = reinterpret_cast<const char*>(memKTX->_storage->data());
const size_t length = memKTX->_storage->size();
// attempt to write the baked texture to the destination file path
QFile bakedTextureFile { _outputDirectory.absoluteFilePath(_bakedTextureFileName) };
if (!bakedTextureFile.open(QIODevice::WriteOnly) || bakedTextureFile.write(data, length) == -1) {
handleError("Could not write baked texture for " + _textureURL.toString());
}
qCDebug(model_baking) << "Baked texture" << _textureURL;
emit finished();
}

59
libraries/baking/src/TextureBaker.h Normal file
View file

@ -0,0 +1,59 @@
//
// TextureBaker.h
// tools/oven/src
//
// Created by Stephen Birarda on 4/5/17.
// Copyright 2017 High Fidelity, Inc.
//
// Distributed under the Apache License, Version 2.0.
// See the accompanying file LICENSE or http://www.apache.org/licenses/LICENSE-2.0.html
//
#ifndef hifi_TextureBaker_h
#define hifi_TextureBaker_h
#include <QtCore/QObject>
#include <QtCore/QUrl>
#include <QtCore/QRunnable>
#include <image/Image.h>
#include "Baker.h"
extern const QString BAKED_TEXTURE_EXT;
class TextureBaker : public Baker {
Q_OBJECT
public:
TextureBaker(const QUrl& textureURL, image::TextureUsage::Type textureType, const QDir& outputDirectory);
const QByteArray& getOriginalTexture() const { return _originalTexture; }
QUrl getTextureURL() const { return _textureURL; }
QString getDestinationFilePath() const { return _outputDirectory.absoluteFilePath(_bakedTextureFileName); }
QString getBakedTextureFileName() const { return _bakedTextureFileName; }
public slots:
virtual void bake() override;
signals:
void originalTextureLoaded();
private slots:
void processTexture();
private:
void loadTexture();
void handleTextureNetworkReply();
QUrl _textureURL;
QByteArray _originalTexture;
image::TextureUsage::Type _textureType;
QDir _outputDirectory;
QString _bakedTextureFileName;
};
#endif // hifi_TextureBaker_h

7
libraries/fbx/CMakeLists.txt
View file

@ -1,4 +1,7 @@
set(TARGET_NAME fbx)
setup_hifi_library()
link_hifi_libraries(shared model networking)
include_hifi_library_headers(gpu)
link_hifi_libraries(shared model networking image)
include_hifi_library_headers(gpu image)

4
libraries/fbx/src/FBXReader.cpp
View file

@ -846,12 +846,14 @@ FBXGeometry* FBXReader::extractFBXGeometry(const QVariantHash& mapping, const QS
QByteArray filename = subobject.properties.at(0).toByteArray();
QByteArray filepath = filename.replace('\\', '/');
filename = fileOnUrl(filepath, url);
qDebug() << "Filename" << filepath << filename;
_textureFilepaths.insert(getID(object.properties), filepath);
_textureFilenames.insert(getID(object.properties), filename);
} else if (subobject.name == "TextureName" && subobject.properties.length() >= TEXTURE_NAME_MIN_SIZE) {
// trim the name from the timestamp
QString name = QString(subobject.properties.at(0).toByteArray());
name = name.left(name.indexOf('['));
qDebug() << "Filename" << name;
_textureNames.insert(getID(object.properties), name);
} else if (subobject.name == "Texture_Alpha_Source" && subobject.properties.length() >= TEXTURE_ALPHA_SOURCE_MIN_SIZE) {
tex.assign<uint8_t>(tex.alphaSource, subobject.properties.at(0).value<int>());
@ -1840,5 +1842,7 @@ FBXGeometry* readFBX(QIODevice* device, const QVariantHash& mapping, const QStri
reader._loadLightmaps = loadLightmaps;
reader._lightmapLevel = lightmapLevel;
qDebug() << "Reading FBX: " << url;
return reader.extractFBXGeometry(mapping, url);
}

11
libraries/model-networking/src/model-networking/ModelCache.cpp
View file

@ -77,6 +77,8 @@ void GeometryMappingResource::downloadFinished(const QByteArray& data) {
texdir += '/';
}
_textureBaseUrl = resolveTextureBaseUrl(url, _url.resolved(texdir));
} else {
_textureBaseUrl = _effectiveBaseURL;
}
auto animGraphVariant = mapping.value("animGraphUrl");
@ -239,7 +241,10 @@ private:
};
void GeometryDefinitionResource::downloadFinished(const QByteArray& data) {
QThreadPool::globalInstance()->start(new GeometryReader(_self, _url, _mapping, data, _combineParts));
qDebug() << "Processing geometry: " << _effectiveBaseURL;
_url = _effectiveBaseURL;
_textureBaseUrl = _effectiveBaseURL;
QThreadPool::globalInstance()->start(new GeometryReader(_self, _effectiveBaseURL, _mapping, data, _combineParts));
}
void GeometryDefinitionResource::setGeometryDefinition(FBXGeometry::Pointer fbxGeometry) {
@ -250,6 +255,7 @@ void GeometryDefinitionResource::setGeometryDefinition(FBXGeometry::Pointer fbxG
QHash<QString, size_t> materialIDAtlas;
for (const FBXMaterial& material : _fbxGeometry->materials) {
materialIDAtlas[material.materialID] = _materials.size();
qDebug() << "setGeometryDefinition() " << _textureBaseUrl;
_materials.push_back(std::make_shared<NetworkMaterial>(material, _textureBaseUrl));
}
@ -342,6 +348,7 @@ Geometry::Geometry(const Geometry& geometry) {
_materials.reserve(geometry._materials.size());
for (const auto& material : geometry._materials) {
qDebug() << "Geometry() no base url...";
_materials.push_back(std::make_shared<NetworkMaterial>(*material));
}
@ -427,6 +434,7 @@ void GeometryResource::deleter() {
void GeometryResource::setTextures() {
if (_fbxGeometry) {
for (const FBXMaterial& material : _fbxGeometry->materials) {
qDebug() << "setTextures() " << _textureBaseUrl;
_materials.push_back(std::make_shared<NetworkMaterial>(material, _textureBaseUrl));
}
}
@ -528,6 +536,7 @@ model::TextureMapPointer NetworkMaterial::fetchTextureMap(const QUrl& url, image
NetworkMaterial::NetworkMaterial(const FBXMaterial& material, const QUrl& textureBaseUrl) :
model::Material(*material._material)
{
qDebug() << "Created network material with base url: " << textureBaseUrl;
_textures = Textures(MapChannel::NUM_MAP_CHANNELS);
if (!material.albedoTexture.filename.isEmpty()) {
auto map = fetchTextureMap(textureBaseUrl, material.albedoTexture, image::TextureUsage::ALBEDO_TEXTURE, MapChannel::ALBEDO_MAP);

1
libraries/model-networking/src/model-networking/ModelCache.h
View file

@ -131,6 +131,7 @@ private:
Geometry::Pointer& _geometryRef;
};
/// Stores cached model geometries.
class ModelCache : public ResourceCache, public Dependency {
Q_OBJECT

10
libraries/model-networking/src/model-networking/TextureCache.cpp
View file

@ -21,6 +21,7 @@
#include <QThreadPool>
#include <QNetworkReply>
#include <QPainter>
#include <QUrlQuery>
#if DEBUG_DUMP_TEXTURE_LOADS
#include <QtCore/QFile>
@ -189,8 +190,15 @@ NetworkTexturePointer TextureCache::getTexture(const QUrl& url, image::TextureUs
if (url.scheme() == RESOURCE_SCHEME) {
return getResourceTexture(url);
}
auto modifiedUrl = url;
if (type == image::TextureUsage::CUBE_TEXTURE) {
QUrlQuery query { url.query() };
query.addQueryItem("skybox", "");
qDebug() << "Updating cubemap texture query from" << url.query() << "to" << query.toString();
modifiedUrl.setQuery(query.toString());
}
TextureExtra extra = { type, content, maxNumPixels };
return ResourceCache::getResource(url, QUrl(), &extra).staticCast<NetworkTexture>();
return ResourceCache::getResource(modifiedUrl, QUrl(), &extra).staticCast<NetworkTexture>();
}
gpu::TexturePointer TextureCache::getTextureByHash(const std::string& hash) {

6
libraries/networking/src/AssetResourceRequest.cpp
View file

@ -63,7 +63,7 @@ void AssetResourceRequest::doSend() {
// This is an ATP path, we'll need to figure out what the mapping is.
// This may incur a roundtrip to the asset-server, or it may return immediately from the cache in AssetClient.
auto path = _url.path();
auto path = _url.path() + (_url.hasQuery() ? "?" + _url.query() : "");
requestMappingForPath(path);
}
}
@ -93,10 +93,8 @@ void AssetResourceRequest::requestMappingForPath(const AssetPath& path) {
// if we got a redirected path we need to store that with the resource request as relative path URL
if (request->wasRedirected()) {
qDebug() << "Request was redirected";
_relativePathURL = ATP_SCHEME + request->getRedirectedPath();
// truncate the filename for the re-directed asset so we actually have a path
_relativePathURL = _relativePathURL.adjusted(QUrl::RemoveFilename);
}
break;

3
libraries/networking/src/MappingRequest.cpp
View file

@ -96,6 +96,9 @@ void GetMappingRequest::doStart() {
// if it did grab that re-directed path
if (_wasRedirected) {
_redirectedPath = message->readString();
qDebug() << "Got redirected from " << _path << " to " << _redirectedPath;
} else {
qDebug() << "Not redirected: " << _path;
}
}

7
libraries/networking/src/ResourceCache.cpp
View file

@ -731,6 +731,13 @@ void Resource::handleReplyFinished() {
if (result == ResourceRequest::Success) {
auto extraInfo = _url == _activeUrl ? "" : QString(", %1").arg(_activeUrl.toDisplayString());
qCDebug(networking).noquote() << QString("Request finished for %1%2").arg(_url.toDisplayString(), extraInfo);
auto relativePathURL = _request->getRelativePathUrl();
qDebug() << "Relative path is: " << relativePathURL;
if (!relativePathURL.isEmpty()) {
qDebug() << "setting effective path";
_effectiveBaseURL = relativePathURL;
}
auto data = _request->getData();
emit loaded(data);

1
libraries/networking/src/ResourceCache.h
View file

@ -452,6 +452,7 @@ protected:
bool handleFailedRequest(ResourceRequest::Result result);
QUrl _url;
QUrl _effectiveBaseURL{ _url };
QUrl _activeUrl;
ByteRange _requestByteRange;

1
libraries/networking/src/ResourceRequest.h
View file

@ -66,6 +66,7 @@ public:
Result getResult() const { return _result; }
QString getResultString() const;
QUrl getUrl() const { return _url; }
QUrl getRelativePathUrl() const { return _relativePathURL; }
bool loadedFromCache() const { return _loadedFromCache; }
bool getRangeRequestSuccessful() const { return _rangeRequestSuccessful; }
bool getTotalSizeOfResource() const { return _totalSizeOfResource; }

15
libraries/networking/src/udt/PacketHeaders.h
View file

@ -139,11 +139,16 @@ public:
const static QSet<PacketTypeEnum::Value> getNonVerifiedPackets() {
const static QSet<PacketTypeEnum::Value> NON_VERIFIED_PACKETS = QSet<PacketTypeEnum::Value>()
<< PacketTypeEnum::Value::NodeJsonStats << PacketTypeEnum::Value::EntityQuery
<< PacketTypeEnum::Value::OctreeDataNack << PacketTypeEnum::Value::EntityEditNack
<< PacketTypeEnum::Value::DomainListRequest << PacketTypeEnum::Value::StopNode
<< PacketTypeEnum::Value::DomainDisconnectRequest << PacketTypeEnum::Value::UsernameFromIDRequest
<< PacketTypeEnum::Value::NodeKickRequest << PacketTypeEnum::Value::NodeMuteRequest;
<< PacketTypeEnum::Value::NodeJsonStats
<< PacketTypeEnum::Value::EntityQuery
<< PacketTypeEnum::Value::OctreeDataNack
<< PacketTypeEnum::Value::EntityEditNack
<< PacketTypeEnum::Value::DomainListRequest
<< PacketTypeEnum::Value::StopNode
<< PacketTypeEnum::Value::DomainDisconnectRequest
<< PacketTypeEnum::Value::UsernameFromIDRequest
<< PacketTypeEnum::Value::NodeKickRequest
<< PacketTypeEnum::Value::NodeMuteRequest;
return NON_VERIFIED_PACKETS;
}

2
tools/oven/CMakeLists.txt
View file

@ -2,7 +2,7 @@ set(TARGET_NAME oven)
setup_hifi_project(Widgets Gui Concurrent)
link_hifi_libraries(networking shared image gpu ktx)
link_hifi_libraries(networking shared image gpu ktx fbx baking)
setup_memory_debugger()