Merge pull request #10689 from Atlante45/fix/ktx

Fix KTX performance problems

libraries/ktx/src/ktx/KTX.cpp

@@ -16,6 +16,8 @@
 
 using namespace ktx;
 
+int ktxDescriptorMetaTypeId = qRegisterMetaType<KTXDescriptor*>();
+
 const Header::Identifier ktx::Header::IDENTIFIER {{
     0xAB, 0x4B, 0x54, 0x58, 0x20, 0x31, 0x31, 0xBB, 0x0D, 0x0A, 0x1A, 0x0A
 }};

libraries/ktx/src/ktx/KTX.h

@@ -387,4 +387,6 @@ namespace ktx {
 
 }
 
+Q_DECLARE_METATYPE(ktx::KTXDescriptor*);
+
 #endif // hifi_ktx_KTX_h

libraries/model-networking/src/model-networking/TextureCache.cpp

@@ -13,6 +13,8 @@
 
 #include <mutex>
 
+#include <QtConcurrent/QtConcurrentRun>
+
 #include <QCryptographicHash>
 #include <QImageReader>
 #include <QRunnable>
@@ -303,14 +305,12 @@ void NetworkTexture::setImage(gpu::TexturePointer texture, int originalWidth,
         _width = texture->getWidth();
         _height = texture->getHeight();
         setSize(texture->getStoredSize());
+        finishedLoading(true);
     } else {
-        // FIXME: If !gpuTexture, we failed to load!
         _width = _height = 0;
-        qWarning() << "Texture did not load";
+        finishedLoading(false);
     }
 
-    finishedLoading(true);
-
     emit networkTextureCreated(qWeakPointerCast<NetworkTexture, Resource> (_self));
 }
 
@@ -382,8 +382,7 @@ void NetworkTexture::makeRequest() {
 
         emit loading();
 
-        connect(_ktxHeaderRequest, &ResourceRequest::progress, this, &NetworkTexture::ktxHeaderRequestProgress);
+        connect(_ktxHeaderRequest, &ResourceRequest::finished, this, &NetworkTexture::ktxInitialDataRequestFinished);
-        connect(_ktxHeaderRequest, &ResourceRequest::finished, this, &NetworkTexture::ktxHeaderRequestFinished);
 
         _bytesReceived = _bytesTotal = _bytes = 0;
 
@@ -407,18 +406,18 @@ void NetworkTexture::makeRequest() {
 }
 
 void NetworkTexture::startRequestForNextMipLevel() {
-    if (_lowestKnownPopulatedMip == 0) {
+    auto self = _self.lock();
-        qWarning(networking) << "Requesting next mip level but all have been fulfilled: " << _lowestKnownPopulatedMip
+    if (!self) {
-            << " " << _textureSource->getGPUTexture()->minAvailableMipLevel() << " " << _url;
         return;
     }
 
-    if (_ktxResourceState == WAITING_FOR_MIP_REQUEST) {
+    auto texture = _textureSource->getGPUTexture();
-        auto self = _self.lock();
+    if (!texture || _ktxResourceState != WAITING_FOR_MIP_REQUEST) {
-        if (!self) {
+        return;
-            return;
+    }
-        }
 
+    _lowestKnownPopulatedMip = texture->minAvailableMipLevel();
+    if (_lowestRequestedMipLevel < _lowestKnownPopulatedMip) {
         _ktxResourceState = PENDING_MIP_REQUEST;
 
         init(false);
@@ -453,6 +452,8 @@ void NetworkTexture::startMipRangeRequest(uint16_t low, uint16_t high) {
         ByteRange range;
         range.fromInclusive = -HIGH_MIP_MAX_SIZE;
         _ktxMipRequest->setByteRange(range);
+
+        connect(_ktxMipRequest, &ResourceRequest::finished, this, &NetworkTexture::ktxInitialDataRequestFinished);
     } else {
         ByteRange range;
         range.fromInclusive = ktx::KTX_HEADER_SIZE + _originalKtxDescriptor->header.bytesOfKeyValueData
@@ -460,229 +461,315 @@ void NetworkTexture::startMipRangeRequest(uint16_t low, uint16_t high) {
         range.toExclusive = ktx::KTX_HEADER_SIZE + _originalKtxDescriptor->header.bytesOfKeyValueData
                               + _originalKtxDescriptor->images[high + 1]._imageOffset;
         _ktxMipRequest->setByteRange(range);
-    }
 
-    connect(_ktxMipRequest, &ResourceRequest::progress, this, &NetworkTexture::ktxMipRequestProgress);
+        connect(_ktxMipRequest, &ResourceRequest::finished, this, &NetworkTexture::ktxMipRequestFinished);
-    connect(_ktxMipRequest, &ResourceRequest::finished, this, &NetworkTexture::ktxMipRequestFinished);
+    }
 
     _ktxMipRequest->send();
 }
 
 
-void NetworkTexture::ktxHeaderRequestFinished() {
+// This is called when the header or top mips have been loaded
-    Q_ASSERT(_ktxResourceState == LOADING_INITIAL_DATA);
+void NetworkTexture::ktxInitialDataRequestFinished() {
-
+    if (!_ktxHeaderRequest || _ktxHeaderRequest->getState() != ResourceRequest::Finished ||
-    if (!_ktxHeaderRequest) {
+        !_ktxMipRequest ||  _ktxMipRequest->getState() != ResourceRequest::Finished) {
+        // Wait for both request to be finished
         return;
     }
 
-    _ktxHeaderRequestFinished = true;
+    Q_ASSERT(_ktxResourceState == LOADING_INITIAL_DATA);
-    maybeHandleFinishedInitialLoad();
+    Q_ASSERT_X(_ktxHeaderRequest && _ktxMipRequest, __FUNCTION__, "Request should not be null while in ktxInitialDataRequestFinished");
+
+    PROFILE_ASYNC_END(resource, "Resource:" + getType(), QString::number(_requestID), {
+        { "from_cache", _ktxHeaderRequest->loadedFromCache() },
+        { "size_mb", _bytesTotal / 1000000.0 }
+    });
+
+    PROFILE_RANGE_EX(resource_parse_image, __FUNCTION__, 0xffff0000, 0, { { "url", _url.toString() } });
+
+    setSize(_bytesTotal);
+
+    TextureCache::requestCompleted(_self);
+
+    auto result = _ktxHeaderRequest->getResult();
+    if (result == ResourceRequest::Success) {
+        result = _ktxMipRequest->getResult();
+    }
+
+    if (result == ResourceRequest::Success) {
+        auto extraInfo = _url == _activeUrl ? "" : QString(", %1").arg(_activeUrl.toDisplayString());
+        qCDebug(networking).noquote() << QString("Request finished for %1%2").arg(_url.toDisplayString(), extraInfo);
+
+        _ktxHeaderData = _ktxHeaderRequest->getData();
+        _ktxHighMipData = _ktxMipRequest->getData();
+        handleFinishedInitialLoad();
+    } else {
+        if (handleFailedRequest(result)) {
+            _ktxResourceState = PENDING_INITIAL_LOAD;
+        } else {
+            _ktxResourceState = FAILED_TO_LOAD;
+        }
+    }
+
+    _ktxHeaderRequest->disconnect(this);
+    _ktxHeaderRequest->deleteLater();
+    _ktxHeaderRequest = nullptr;
+    _ktxMipRequest->disconnect(this);
+    _ktxMipRequest->deleteLater();
+    _ktxMipRequest = nullptr;
 }
 
 void NetworkTexture::ktxMipRequestFinished() {
-    Q_ASSERT(_ktxResourceState == LOADING_INITIAL_DATA || _ktxResourceState == REQUESTING_MIP);
+    Q_ASSERT_X(_ktxMipRequest, __FUNCTION__, "Request should not be null while in ktxMipRequestFinished");
+    Q_ASSERT(_ktxResourceState == REQUESTING_MIP);
 
-    if (!_ktxMipRequest) {
+    PROFILE_ASYNC_END(resource, "Resource:" + getType(), QString::number(_requestID), {
+        { "from_cache", _ktxMipRequest->loadedFromCache() },
+        { "size_mb", _bytesTotal / 1000000.0 }
+    });
+
+    PROFILE_RANGE_EX(resource_parse_image, __FUNCTION__, 0xffff0000, 0, { { "url", _url.toString() } });
+
+
+    setSize(_bytesTotal);
+
+    if (!_ktxMipRequest || _ktxMipRequest != sender()) {
+        // This can happen in the edge case that a request is timed out, but a `finished` signal is emitted before it is deleted.
+        qWarning(networking) << "Received signal NetworkTexture::ktxMipRequestFinished from ResourceRequest that is not the current"
+            << " request: " << sender() << ", " << _ktxMipRequest;
         return;
     }
 
-    if (_ktxResourceState == LOADING_INITIAL_DATA) {
+    TextureCache::requestCompleted(_self);
-        _ktxHighMipRequestFinished = true;
-        maybeHandleFinishedInitialLoad();
-    } else if (_ktxResourceState == REQUESTING_MIP) {
-        Q_ASSERT(_ktxMipLevelRangeInFlight.first != NULL_MIP_LEVEL);
-        TextureCache::requestCompleted(_self);
 
-        if (_ktxMipRequest->getResult() == ResourceRequest::Success) {
+    auto result = _ktxMipRequest->getResult();
+    if (result == ResourceRequest::Success) {
+        auto extraInfo = _url == _activeUrl ? "" : QString(", %1").arg(_activeUrl.toDisplayString());
+        qCDebug(networking).noquote() << QString("Request finished for %1%2").arg(_url.toDisplayString(), extraInfo);
+
+        if (_ktxResourceState == REQUESTING_MIP) {
+            Q_ASSERT(_ktxMipLevelRangeInFlight.first != NULL_MIP_LEVEL);
             Q_ASSERT(_ktxMipLevelRangeInFlight.second - _ktxMipLevelRangeInFlight.first == 0);
 
+            _ktxResourceState = WAITING_FOR_MIP_REQUEST;
+
+            auto self = _self;
+            auto url = _url;
+            auto data = _ktxMipRequest->getData();
+            auto mipLevel = _ktxMipLevelRangeInFlight.first;
             auto texture = _textureSource->getGPUTexture();
-            if (texture) {
+            DependencyManager::get<StatTracker>()->incrementStat("PendingProcessing");
-                texture->assignStoredMip(_ktxMipLevelRangeInFlight.first,
+            QtConcurrent::run(QThreadPool::globalInstance(), [self, data, mipLevel, url, texture] {
-                    _ktxMipRequest->getData().size(), reinterpret_cast<uint8_t*>(_ktxMipRequest->getData().data()));
+                PROFILE_RANGE_EX(resource_parse_image, "NetworkTexture - Processing Mip Data", 0xffff0000, 0, { { "url", url.toString() } });
+                DependencyManager::get<StatTracker>()->decrementStat("PendingProcessing");
+                CounterStat counter("Processing");
 
-                if (texture->minAvailableMipLevel() <= _ktxMipLevelRangeInFlight.first) {
+                auto originalPriority = QThread::currentThread()->priority();
-                    _lowestKnownPopulatedMip = texture->minAvailableMipLevel();
+                if (originalPriority == QThread::InheritPriority) {
-                    _ktxResourceState = WAITING_FOR_MIP_REQUEST;
+                    originalPriority = QThread::NormalPriority;
-                } else {
-                    qWarning(networking) << "Failed to load mip: " << _url << ":" << _ktxMipLevelRangeInFlight.first;
-                    _ktxResourceState = FAILED_TO_LOAD;
                 }
-            } else {
+                QThread::currentThread()->setPriority(QThread::LowPriority);
-                _ktxResourceState = WAITING_FOR_MIP_REQUEST;
+                Finally restorePriority([originalPriority] { QThread::currentThread()->setPriority(originalPriority); });
-                qWarning(networking) << "Trying to update mips but texture is null";
+
-            }
+                auto resource = self.lock();
-            finishedLoading(true);
+                if (!resource) {
+                    // Resource no longer exists, bail
+                    return;
+                }
+
+                Q_ASSERT_X(texture, "Async - NetworkTexture::ktxMipRequestFinished", "NetworkTexture should have been assigned a GPU texture by now.");
+
+                texture->assignStoredMip(mipLevel, data.size(), reinterpret_cast<const uint8_t*>(data.data()));
+
+                QMetaObject::invokeMethod(resource.data(), "setImage",
+                    Q_ARG(gpu::TexturePointer, texture),
+                    Q_ARG(int, texture->getWidth()),
+                    Q_ARG(int, texture->getHeight()));
+
+                QMetaObject::invokeMethod(resource.data(), "startRequestForNextMipLevel");
+            });
         } else {
+            qWarning(networking) << "Mip request finished in an unexpected state: " << _ktxResourceState;
             finishedLoading(false);
-            if (handleFailedRequest(_ktxMipRequest->getResult())) {
-                _ktxResourceState = PENDING_MIP_REQUEST;
-            } else {
-                qWarning(networking) << "Failed to load mip: " << _url;
-                _ktxResourceState = FAILED_TO_LOAD;
-            }
-        }
-
-        _ktxMipRequest->deleteLater();
-        _ktxMipRequest = nullptr;
-
-        if (_ktxResourceState == WAITING_FOR_MIP_REQUEST && _lowestRequestedMipLevel < _lowestKnownPopulatedMip) {
-            startRequestForNextMipLevel();
         }
     } else {
-        qWarning() << "Mip request finished in an unexpected state: " << _ktxResourceState;
+        if (handleFailedRequest(result)) {
+            _ktxResourceState = PENDING_MIP_REQUEST;
+        } else {
+            _ktxResourceState = FAILED_TO_LOAD;
+        }
     }
+
+    _ktxMipRequest->disconnect(this);
+    _ktxMipRequest->deleteLater();
+    _ktxMipRequest = nullptr;
 }
 
-// This is called when the header or top mips have been loaded
+// This is called when the header and top mips have been loaded
-void NetworkTexture::maybeHandleFinishedInitialLoad() {
+void NetworkTexture::handleFinishedInitialLoad() {
     Q_ASSERT(_ktxResourceState == LOADING_INITIAL_DATA);
+    Q_ASSERT(!_ktxHeaderData.isEmpty() && !_ktxHighMipData.isEmpty());
 
-    if (_ktxHeaderRequestFinished && _ktxHighMipRequestFinished) {
+    // create ktx...
+    auto ktxHeaderData = _ktxHeaderData;
+    auto ktxHighMipData = _ktxHighMipData;
+    _ktxHeaderData.clear();
+    _ktxHighMipData.clear();
 
-        TextureCache::requestCompleted(_self);
+    _ktxResourceState = WAITING_FOR_MIP_REQUEST;
 
-        if (_ktxHeaderRequest->getResult() != ResourceRequest::Success || _ktxMipRequest->getResult() != ResourceRequest::Success) {
+    auto self = _self;
-            if (handleFailedRequest(_ktxMipRequest->getResult())) {
+    auto url = _url;
-                _ktxResourceState = PENDING_INITIAL_LOAD;
+    DependencyManager::get<StatTracker>()->incrementStat("PendingProcessing");
-            }
+    QtConcurrent::run(QThreadPool::globalInstance(), [self, ktxHeaderData, ktxHighMipData, url] {
-            else {
+        PROFILE_RANGE_EX(resource_parse_image, "NetworkTexture - Processing Initial Data", 0xffff0000, 0, { { "url", url.toString() } });
-                _ktxResourceState = FAILED_TO_LOAD;
+        DependencyManager::get<StatTracker>()->decrementStat("PendingProcessing");
-            }
+        CounterStat counter("Processing");
 
-            _ktxHeaderRequest->deleteLater();
+        auto originalPriority = QThread::currentThread()->priority();
-            _ktxHeaderRequest = nullptr;
+        if (originalPriority == QThread::InheritPriority) {
-            _ktxMipRequest->deleteLater();
+            originalPriority = QThread::NormalPriority;
-            _ktxMipRequest = nullptr;
-        } else {
-            // create ktx...
-            auto ktxHeaderData = _ktxHeaderRequest->getData();
-            auto ktxHighMipData = _ktxMipRequest->getData();
-
-            auto header = reinterpret_cast<const ktx::Header*>(ktxHeaderData.data());
-
-            if (!ktx::checkIdentifier(header->identifier)) {
-                qWarning() << "Cannot load " << _url << ", invalid header identifier";
-                _ktxResourceState = FAILED_TO_LOAD;
-                finishedLoading(false);
-                return;
-            }
-
-            auto kvSize = header->bytesOfKeyValueData;
-            if (kvSize > (ktxHeaderData.size() - ktx::KTX_HEADER_SIZE)) {
-                qWarning() << "Cannot load " << _url << ", did not receive all kv data with initial request";
-                _ktxResourceState = FAILED_TO_LOAD;
-                finishedLoading(false);
-                return;
-            }
-
-            auto keyValues = ktx::KTX::parseKeyValues(header->bytesOfKeyValueData, reinterpret_cast<const ktx::Byte*>(ktxHeaderData.data()) + ktx::KTX_HEADER_SIZE);
-
-            auto imageDescriptors = header->generateImageDescriptors();
-            if (imageDescriptors.size() == 0) {
-                qWarning(networking) << "Failed to process ktx file " << _url;
-                _ktxResourceState = FAILED_TO_LOAD;
-                finishedLoading(false);
-            }
-            _originalKtxDescriptor.reset(new ktx::KTXDescriptor(*header, keyValues, imageDescriptors));
-
-            // Create bare ktx in memory
-            auto found = std::find_if(keyValues.begin(), keyValues.end(), [](const ktx::KeyValue& val) -> bool {
-                return val._key.compare(gpu::SOURCE_HASH_KEY) == 0;
-            });
-            std::string filename;
-            std::string hash;
-            if (found == keyValues.end() || found->_value.size() != gpu::SOURCE_HASH_BYTES) {
-                qWarning("Invalid source hash key found, bailing");
-                _ktxResourceState = FAILED_TO_LOAD;
-                finishedLoading(false);
-                return;
-            } else {
-                // at this point the source hash is in binary 16-byte form
-                // and we need it in a hexadecimal string
-                auto binaryHash = QByteArray(reinterpret_cast<char*>(found->_value.data()), gpu::SOURCE_HASH_BYTES);
-                hash = filename = binaryHash.toHex().toStdString();
-            }
-
-            auto textureCache = DependencyManager::get<TextureCache>();
-
-            gpu::TexturePointer texture = textureCache->getTextureByHash(hash);
-
-            if (!texture) {
-                KTXFilePointer ktxFile = textureCache->_ktxCache.getFile(hash);
-                if (ktxFile) {
-                    texture = gpu::Texture::unserialize(ktxFile);
-                    if (texture) {
-                        texture = textureCache->cacheTextureByHash(hash, texture);
-                        _file = ktxFile;
-                    }
-                }
-            }
-
-            if (!texture) {
-
-                auto memKtx = ktx::KTX::createBare(*header, keyValues);
-                if (!memKtx) {
-                    qWarning() << " Ktx could not be created, bailing";
-                    finishedLoading(false);
-                    return;
-                }
-
-                // Move ktx to file
-                const char* data = reinterpret_cast<const char*>(memKtx->_storage->data());
-                size_t length = memKtx->_storage->size();
-                KTXFilePointer file;
-                auto& ktxCache = textureCache->_ktxCache;
-                if (!memKtx || !(file = ktxCache.writeFile(data, KTXCache::Metadata(filename, length)))) {
-                    qCWarning(modelnetworking) << _url << " failed to write cache file";
-                    _ktxResourceState = FAILED_TO_LOAD;
-                    finishedLoading(false);
-                    return;
-                } else {
-                    _file = file;
-                }
-
-                auto newKtxDescriptor = memKtx->toDescriptor();
-
-                texture = gpu::Texture::build(newKtxDescriptor);
-                texture->setKtxBacking(file);
-                texture->setSource(filename);
-
-                auto& images = _originalKtxDescriptor->images;
-                size_t imageSizeRemaining = ktxHighMipData.size();
-                uint8_t* ktxData = reinterpret_cast<uint8_t*>(ktxHighMipData.data());
-                ktxData += ktxHighMipData.size();
-                // TODO Move image offset calculation to ktx ImageDescriptor
-                for (int level = static_cast<int>(images.size()) - 1; level >= 0; --level) {
-                    auto& image = images[level];
-                    if (image._imageSize > imageSizeRemaining) {
-                        break;
-                    }
-                    ktxData -= image._imageSize;
-                    texture->assignStoredMip(static_cast<gpu::uint16>(level), image._imageSize, ktxData);
-                    ktxData -= ktx::IMAGE_SIZE_WIDTH;
-                    imageSizeRemaining -= (image._imageSize + ktx::IMAGE_SIZE_WIDTH);
-                }
-
-                // We replace the texture with the one stored in the cache.  This deals with the possible race condition of two different 
-                // images with the same hash being loaded concurrently.  Only one of them will make it into the cache by hash first and will
-                // be the winner
-                texture = textureCache->cacheTextureByHash(filename, texture);
-            }
-
-            _lowestKnownPopulatedMip = texture->minAvailableMipLevel();
-
-            _ktxResourceState = WAITING_FOR_MIP_REQUEST;
-            setImage(texture, header->getPixelWidth(), header->getPixelHeight());
-
-            _ktxHeaderRequest->deleteLater();
-            _ktxHeaderRequest = nullptr;
-            _ktxMipRequest->deleteLater();
-            _ktxMipRequest = nullptr;
         }
-        startRequestForNextMipLevel();
+        QThread::currentThread()->setPriority(QThread::LowPriority);
-    }
+        Finally restorePriority([originalPriority] { QThread::currentThread()->setPriority(originalPriority); });
+
+        auto resource = self.lock();
+        if (!resource) {
+            // Resource no longer exists, bail
+            return;
+        }
+
+        auto header = reinterpret_cast<const ktx::Header*>(ktxHeaderData.data());
+
+        if (!ktx::checkIdentifier(header->identifier)) {
+            qWarning() << "Cannot load " << url << ", invalid header identifier";
+            QMetaObject::invokeMethod(resource.data(), "setImage",
+                Q_ARG(gpu::TexturePointer, nullptr),
+                Q_ARG(int, 0),
+                Q_ARG(int, 0));
+            return;
+        }
+
+        auto kvSize = header->bytesOfKeyValueData;
+        if (kvSize > (ktxHeaderData.size() - ktx::KTX_HEADER_SIZE)) {
+            qWarning() << "Cannot load " << url << ", did not receive all kv data with initial request";
+            QMetaObject::invokeMethod(resource.data(), "setImage",
+                Q_ARG(gpu::TexturePointer, nullptr),
+                Q_ARG(int, 0),
+                Q_ARG(int, 0));
+            return;
+        }
+
+        auto keyValues = ktx::KTX::parseKeyValues(header->bytesOfKeyValueData, reinterpret_cast<const ktx::Byte*>(ktxHeaderData.data()) + ktx::KTX_HEADER_SIZE);
+
+        auto imageDescriptors = header->generateImageDescriptors();
+        if (imageDescriptors.size() == 0) {
+            qWarning(networking) << "Failed to process ktx file " << url;
+            QMetaObject::invokeMethod(resource.data(), "setImage",
+                Q_ARG(gpu::TexturePointer, nullptr),
+                Q_ARG(int, 0),
+                Q_ARG(int, 0));
+            return;
+        }
+        auto originalKtxDescriptor = new ktx::KTXDescriptor(*header, keyValues, imageDescriptors);
+        QMetaObject::invokeMethod(resource.data(), "setOriginalDescriptor",
+            Q_ARG(ktx::KTXDescriptor*, originalKtxDescriptor));
+
+        // Create bare ktx in memory
+        auto found = std::find_if(keyValues.begin(), keyValues.end(), [](const ktx::KeyValue& val) -> bool {
+            return val._key.compare(gpu::SOURCE_HASH_KEY) == 0;
+        });
+        std::string filename;
+        std::string hash;
+        if (found == keyValues.end() || found->_value.size() != gpu::SOURCE_HASH_BYTES) {
+            qWarning("Invalid source hash key found, bailing");
+            QMetaObject::invokeMethod(resource.data(), "setImage",
+                Q_ARG(gpu::TexturePointer, nullptr),
+                Q_ARG(int, 0),
+                Q_ARG(int, 0));
+            return;
+        } else {
+            // at this point the source hash is in binary 16-byte form
+            // and we need it in a hexadecimal string
+            auto binaryHash = QByteArray(reinterpret_cast<char*>(found->_value.data()), gpu::SOURCE_HASH_BYTES);
+            hash = filename = binaryHash.toHex().toStdString();
+        }
+
+        auto textureCache = DependencyManager::get<TextureCache>();
+
+        gpu::TexturePointer texture = textureCache->getTextureByHash(hash);
+
+        if (!texture) {
+            KTXFilePointer ktxFile = textureCache->_ktxCache.getFile(hash);
+            if (ktxFile) {
+                texture = gpu::Texture::unserialize(ktxFile);
+                if (texture) {
+                    texture = textureCache->cacheTextureByHash(hash, texture);
+                }
+            }
+        }
+
+        if (!texture) {
+
+            auto memKtx = ktx::KTX::createBare(*header, keyValues);
+            if (!memKtx) {
+                qWarning() << " Ktx could not be created, bailing";
+                QMetaObject::invokeMethod(resource.data(), "setImage",
+                    Q_ARG(gpu::TexturePointer, nullptr),
+                    Q_ARG(int, 0),
+                    Q_ARG(int, 0));
+                return;
+            }
+
+            // Move ktx to file
+            const char* data = reinterpret_cast<const char*>(memKtx->_storage->data());
+            size_t length = memKtx->_storage->size();
+            KTXFilePointer file;
+            auto& ktxCache = textureCache->_ktxCache;
+            if (!memKtx || !(file = ktxCache.writeFile(data, KTXCache::Metadata(filename, length)))) {
+                qCWarning(modelnetworking) << url << " failed to write cache file";
+                QMetaObject::invokeMethod(resource.data(), "setImage",
+                    Q_ARG(gpu::TexturePointer, nullptr),
+                    Q_ARG(int, 0),
+                    Q_ARG(int, 0));
+                return;
+            }
+
+            auto newKtxDescriptor = memKtx->toDescriptor();
+
+            texture = gpu::Texture::build(newKtxDescriptor);
+            texture->setKtxBacking(file);
+            texture->setSource(filename);
+
+            auto& images = originalKtxDescriptor->images;
+            size_t imageSizeRemaining = ktxHighMipData.size();
+            const uint8_t* ktxData = reinterpret_cast<const uint8_t*>(ktxHighMipData.data());
+            ktxData += ktxHighMipData.size();
+            // TODO Move image offset calculation to ktx ImageDescriptor
+            for (int level = static_cast<int>(images.size()) - 1; level >= 0; --level) {
+                auto& image = images[level];
+                if (image._imageSize > imageSizeRemaining) {
+                    break;
+                }
+                ktxData -= image._imageSize;
+                texture->assignStoredMip(static_cast<gpu::uint16>(level), image._imageSize, ktxData);
+                ktxData -= ktx::IMAGE_SIZE_WIDTH;
+                imageSizeRemaining -= (image._imageSize + ktx::IMAGE_SIZE_WIDTH);
+            }
+
+            // We replace the texture with the one stored in the cache.  This deals with the possible race condition of two different
+            // images with the same hash being loaded concurrently.  Only one of them will make it into the cache by hash first and will
+            // be the winner
+            texture = textureCache->cacheTextureByHash(filename, texture);
+        }
+
+        QMetaObject::invokeMethod(resource.data(), "setImage",
+            Q_ARG(gpu::TexturePointer, texture),
+            Q_ARG(int, texture->getWidth()),
+            Q_ARG(int, texture->getHeight()));
+
+        QMetaObject::invokeMethod(resource.data(), "startRequestForNextMipLevel");
+    });
 }
 
 void NetworkTexture::downloadFinished(const QByteArray& data) {
@@ -845,11 +932,11 @@ void ImageReader::read() {
             const char* data = reinterpret_cast<const char*>(memKtx->_storage->data());
             size_t length = memKtx->_storage->size();
             auto& ktxCache = textureCache->_ktxCache;
-            networkTexture->_file = ktxCache.writeFile(data, KTXCache::Metadata(hash, length)); // 
+            auto file = ktxCache.writeFile(data, KTXCache::Metadata(hash, length));
-            if (!networkTexture->_file) {
+            if (!file) {
                 qCWarning(modelnetworking) << _url << "file cache failed";
             } else {
-                texture->setKtxBacking(networkTexture->_file);
+                texture->setKtxBacking(file);
             }
         } else {
             qCWarning(modelnetworking) << "Unable to serialize texture to KTX " << _url;

libraries/model-networking/src/model-networking/TextureCache.h

@@ -58,14 +58,13 @@ public:
 
     void refresh() override;
 
+    Q_INVOKABLE void setOriginalDescriptor(ktx::KTXDescriptor* descriptor) { _originalKtxDescriptor.reset(descriptor); }
+
 signals:
     void networkTextureCreated(const QWeakPointer<NetworkTexture>& self);
 
 public slots:
-    void ktxHeaderRequestProgress(uint64_t bytesReceived, uint64_t bytesTotal) { }
+    void ktxInitialDataRequestFinished();
-    void ktxHeaderRequestFinished();
-
-    void ktxMipRequestProgress(uint64_t bytesReceived, uint64_t bytesTotal) { }
     void ktxMipRequestFinished();
 
 protected:
@@ -74,14 +73,14 @@ protected:
     virtual bool isCacheable() const override { return _loaded; }
 
     virtual void downloadFinished(const QByteArray& data) override;
-
+    
     Q_INVOKABLE void loadContent(const QByteArray& content);
     Q_INVOKABLE void setImage(gpu::TexturePointer texture, int originalWidth, int originalHeight);
 
-    void startRequestForNextMipLevel();
+    Q_INVOKABLE void startRequestForNextMipLevel();
 
     void startMipRangeRequest(uint16_t low, uint16_t high);
-    void maybeHandleFinishedInitialLoad();
+    void handleFinishedInitialLoad();
 
 private:
     friend class KTXReader;
@@ -102,16 +101,13 @@ private:
     bool _sourceIsKTX { false };
     KTXResourceState _ktxResourceState { PENDING_INITIAL_LOAD };
 
-    // TODO Can this be removed?
-    KTXFilePointer _file;
-
     // The current mips that are currently being requested w/ _ktxMipRequest
     std::pair<uint16_t, uint16_t> _ktxMipLevelRangeInFlight{ NULL_MIP_LEVEL, NULL_MIP_LEVEL };
 
     ResourceRequest* _ktxHeaderRequest { nullptr };
     ResourceRequest* _ktxMipRequest { nullptr };
-    bool _ktxHeaderRequestFinished{ false };
+    QByteArray _ktxHeaderData;
-    bool _ktxHighMipRequestFinished{ false };
+    QByteArray _ktxHighMipData;
 
     uint16_t _lowestRequestedMipLevel { NULL_MIP_LEVEL };
     uint16_t _lowestKnownPopulatedMip { NULL_MIP_LEVEL };