From 8cb0d0f8fd982d4ba5029358b4400f1a8e8ab0b3 Mon Sep 17 00:00:00 2001 From: Brad Davis Date: Tue, 23 Jan 2018 13:30:07 -0800 Subject: [PATCH] Don't use the KTX cache for local files --- .../src/model-networking/TextureCache.cpp | 109 +++++++++++++++--- .../src/model-networking/TextureCache.h | 4 +- 2 files changed, 96 insertions(+), 17 deletions(-) diff --git a/libraries/model-networking/src/model-networking/TextureCache.cpp b/libraries/model-networking/src/model-networking/TextureCache.cpp index fb75123b43..0a9e5f94a0 100644 --- a/libraries/model-networking/src/model-networking/TextureCache.cpp +++ b/libraries/model-networking/src/model-networking/TextureCache.cpp @@ -37,7 +37,7 @@ #include #include - +#include #include #include @@ -50,10 +50,8 @@ Q_LOGGING_CATEGORY(trace_resource_parse_image, "trace.resource.parse.image") Q_LOGGING_CATEGORY(trace_resource_parse_image_raw, "trace.resource.parse.image.raw") Q_LOGGING_CATEGORY(trace_resource_parse_image_ktx, "trace.resource.parse.image.ktx") -#if !defined(DISABLE_KTX_CACHE) const std::string TextureCache::KTX_DIRNAME { "ktx_cache" }; const std::string TextureCache::KTX_EXT { "ktx" }; -#endif static const QString RESOURCE_SCHEME = "resource"; static const QUrl SPECTATOR_CAMERA_FRAME_URL("resource://spectatorCameraFrame"); @@ -62,9 +60,12 @@ static const QUrl HMD_PREVIEW_FRAME_URL("resource://hmdPreviewFrame"); static const float SKYBOX_LOAD_PRIORITY { 10.0f }; // Make sure skybox loads first static const float HIGH_MIPS_LOAD_PRIORITY { 9.0f }; // Make sure high mips loads after skybox but before models +#pragma optimize("", off) + TextureCache::TextureCache() { -#if !defined(DISABLE_KTX_CACHE) _ktxCache->initialize(); +#if defined(DISABLE_KTX_CACHE) + _ktxCache->wipe(); #endif setUnusedResourceCacheSize(0); setObjectName("TextureCache"); @@ -294,6 +295,10 @@ _maxNumPixels(100) _loaded = true; } +static bool isLocalUrl(const QUrl& url) { + auto scheme = url.scheme(); + return (scheme == URL_SCHEME_FILE || scheme == URL_SCHEME_QRC || scheme == RESOURCE_SCHEME); +} NetworkTexture::NetworkTexture(const QUrl& url, image::TextureUsage::Type type, const QByteArray& content, int maxNumPixels) : Resource(url), @@ -387,6 +392,21 @@ void NetworkTexture::makeRequest() { return; } + if (isLocalUrl(_url)) { + auto self = _self; + QtConcurrent::run(QThreadPool::globalInstance(), [self] { + auto resource = self.lock(); + if (!resource) { + return; + } + + NetworkTexture* networkTexture = static_cast(resource.data()); + networkTexture->makeLocalRequest(); + }); + return; + } + + // We special-handle ktx requests to run 2 concurrent requests right off the bat PROFILE_ASYNC_BEGIN(resource, "Resource:" + getType(), QString::number(_requestID), { { "url", _url.toString() }, { "activeURL", _activeUrl.toString() } }); @@ -435,6 +455,74 @@ void NetworkTexture::makeRequest() { } +void NetworkTexture::makeLocalRequest() { + const QString scheme = _url.scheme(); + QString path; + if (scheme == URL_SCHEME_FILE) { + path = _url.toLocalFile(); + } else { + path = ":" + _url.path(); + } + + path = FileUtils::selectFile(path); + + auto storage = std::make_shared(path); + std::unique_ptr ktxFile; + if (storage) { + ktxFile = ktx::KTX::create(storage); + } + std::shared_ptr ktxDescriptor; + if (ktxFile) { + ktxDescriptor = std::make_shared(ktxFile->toDescriptor()); + } + + gpu::TexturePointer texture; + if (ktxDescriptor) { + std::string hash; + // Create bare ktx in memory + auto found = std::find_if(ktxDescriptor->keyValues.begin(), ktxDescriptor->keyValues.end(), [](const ktx::KeyValue& val) -> bool { + return val._key.compare(gpu::SOURCE_HASH_KEY) == 0; + }); + + if (found == ktxDescriptor->keyValues.end() || found->_value.size() != gpu::SOURCE_HASH_BYTES) { + hash = _url.toString().toLocal8Bit().toHex().toStdString(); + } else { + // at this point the source hash is in binary 16-byte form + // and we need it in a hexadecimal string + auto binaryHash = QByteArray(reinterpret_cast(found->_value.data()), gpu::SOURCE_HASH_BYTES); + hash = binaryHash.toHex().toStdString(); + } + + auto textureCache = DependencyManager::get(); + texture = textureCache->getTextureByHash(hash); + if (!texture) { + texture = gpu::Texture::build(*ktxDescriptor); + if (texture) { + texture->setKtxBacking(path.toStdString()); + texture->setSource(path.toStdString()); + texture = textureCache->cacheTextureByHash(hash, texture); + } + } + } + + if (!texture) { + qCDebug(networking).noquote() << "Failed load local KTX from" << path; + QMetaObject::invokeMethod(this, "setImage", + Q_ARG(gpu::TexturePointer, nullptr), + Q_ARG(int, 0), + Q_ARG(int, 0)); + return; + } + + _ktxResourceState = PENDING_MIP_REQUEST; + _lowestKnownPopulatedMip = texture->minAvailableMipLevel(); + QMetaObject::invokeMethod(this, "setImage", + Q_ARG(gpu::TexturePointer, texture), + Q_ARG(int, texture->getWidth()), + Q_ARG(int, texture->getHeight())); + +} + bool NetworkTexture::handleFailedRequest(ResourceRequest::Result result) { if (!_sourceIsKTX && result == ResourceRequest::Result::RedirectFail) { auto newPath = _request->getRelativePathUrl(); @@ -750,7 +838,6 @@ void NetworkTexture::handleFinishedInitialLoad() { gpu::TexturePointer texture = textureCache->getTextureByHash(hash); -#if !defined(DISABLE_KTX_CACHE) if (!texture) { auto ktxFile = textureCache->_ktxCache->getFile(hash); if (ktxFile) { @@ -763,9 +850,8 @@ void NetworkTexture::handleFinishedInitialLoad() { } } } -#endif - if (!texture) { + if (!texture) { auto memKtx = ktx::KTX::createBare(*header, keyValues); if (!memKtx) { qWarning() << " Ktx could not be created, bailing"; @@ -778,7 +864,6 @@ void NetworkTexture::handleFinishedInitialLoad() { // Move ktx to file const char* data = reinterpret_cast(memKtx->_storage->data()); -#if !defined(DISABLE_KTX_CACHE) size_t length = memKtx->_storage->size(); cache::FilePointer file; auto& ktxCache = textureCache->_ktxCache; @@ -790,14 +875,11 @@ void NetworkTexture::handleFinishedInitialLoad() { Q_ARG(int, 0)); return; } -#endif auto newKtxDescriptor = memKtx->toDescriptor(); texture = gpu::Texture::build(newKtxDescriptor); -#if !defined(DISABLE_KTX_CACHE) texture->setKtxBacking(file); -#endif texture->setSource(filename); auto& images = originalKtxDescriptor->images; @@ -940,7 +1022,6 @@ void ImageReader::read() { // If we already have a live texture with the same hash, use it auto texture = textureCache->getTextureByHash(hash); -#if !defined(DISABLE_KTX_CACHE) // If there is no live texture, check if there's an existing KTX file if (!texture) { auto ktxFile = textureCache->_ktxCache->getFile(hash); @@ -953,7 +1034,6 @@ void ImageReader::read() { } } } -#endif // If we found the texture either because it's in use or via KTX deserialization, // set the image and return immediately. @@ -989,7 +1069,6 @@ void ImageReader::read() { // Save the image into a KTXFile if (texture && textureCache) { -#if !defined(DISABLE_KTX_CACHE) auto memKtx = gpu::Texture::serialize(*texture); // Move the texture into a memory mapped file @@ -1006,7 +1085,7 @@ void ImageReader::read() { } else { qCWarning(modelnetworking) << "Unable to serialize texture to KTX " << _url; } -#endif + // We replace the texture with the one stored in the cache. This deals with the possible race condition of two different // images with the same hash being loaded concurrently. Only one of them will make it into the cache by hash first and will // be the winner diff --git a/libraries/model-networking/src/model-networking/TextureCache.h b/libraries/model-networking/src/model-networking/TextureCache.h index 74395d8948..5b33b42226 100644 --- a/libraries/model-networking/src/model-networking/TextureCache.h +++ b/libraries/model-networking/src/model-networking/TextureCache.h @@ -70,6 +70,7 @@ public slots: protected: void makeRequest() override; + void makeLocalRequest(); virtual bool isCacheable() const override { return _loaded; } @@ -194,12 +195,11 @@ private: TextureCache(); virtual ~TextureCache(); -#if !defined(DISABLE_KTX_CACHE) static const std::string KTX_DIRNAME; static const std::string KTX_EXT; std::shared_ptr _ktxCache { std::make_shared(KTX_DIRNAME, KTX_EXT) }; -#endif + // Map from image hashes to texture weak pointers std::unordered_map> _texturesByHashes; std::mutex _texturesByHashesMutex;