From ccd9c4697b2bf67f1c6912effcd5f0a40f9144aa Mon Sep 17 00:00:00 2001 From: Ryan Huffman Date: Tue, 11 Apr 2017 22:44:43 -0700 Subject: [PATCH] Add extended ktx header/high-mip request handling to NetworkTexture --- interface/src/Application.cpp | 2 +- libraries/gpu/src/gpu/Texture.h | 13 +- libraries/gpu/src/gpu/Texture_ktx.cpp | 20 +++ .../src/model-networking/TextureCache.cpp | 160 ++++++++++++++++++ .../src/model-networking/TextureCache.h | 21 +++ .../networking/src/HTTPResourceRequest.cpp | 7 +- libraries/networking/src/ResourceCache.cpp | 60 ++++--- libraries/networking/src/ResourceCache.h | 19 ++- libraries/networking/src/ResourceRequest.h | 2 +- 9 files changed, 257 insertions(+), 47 deletions(-) diff --git a/interface/src/Application.cpp b/interface/src/Application.cpp index 8703c09497..886487603e 100644 --- a/interface/src/Application.cpp +++ b/interface/src/Application.cpp @@ -626,7 +626,7 @@ Application::Application(int& argc, char** argv, QElapsedTimer& startupTimer, bo proxy.setType(QNetworkProxy::HttpProxy); proxy.setHostName("127.0.0.1"); proxy.setPort(8888); - //QNetworkProxy::setApplicationProxy(proxy); + QNetworkProxy::setApplicationProxy(proxy); // make sure the debug draw singleton is initialized on the main thread. DebugDraw::getInstance().removeMarker(""); diff --git a/libraries/gpu/src/gpu/Texture.h b/libraries/gpu/src/gpu/Texture.h index b01ec8f0bc..e82566b852 100755 --- a/libraries/gpu/src/gpu/Texture.h +++ b/libraries/gpu/src/gpu/Texture.h @@ -270,7 +270,6 @@ public: virtual void reset() = 0; virtual PixelsPointer getMipFace(uint16 level, uint8 face = 0) const = 0; virtual Size getMipFaceSize(uint16 level, uint8 face = 0) const = 0; - virtual void assignMipData(uint16 level, const char* data, const size_t length) = 0; virtual void assignMipData(uint16 level, const storage::StoragePointer& storage) = 0; virtual void assignMipFaceData(uint16 level, uint8 face, const storage::StoragePointer& storage) = 0; virtual bool isMipAvailable(uint16 level, uint8 face = 0) const = 0; @@ -297,7 +296,6 @@ public: void reset() override; PixelsPointer getMipFace(uint16 level, uint8 face = 0) const override; Size getMipFaceSize(uint16 level, uint8 face = 0) const override; - void assignMipData(uint16 level, const char* data, const size_t length) override; void assignMipData(uint16 level, const storage::StoragePointer& storage) override; void assignMipFaceData(uint16 level, uint8 face, const storage::StoragePointer& storage) override; bool isMipAvailable(uint16 level, uint8 face = 0) const override; @@ -313,15 +311,12 @@ public: PixelsPointer getMipFace(uint16 level, uint8 face = 0) const override; Size getMipFaceSize(uint16 level, uint8 face = 0) const override; // By convention, all mip levels and faces MUST be populated when using KTX backing - bool isMipAvailable(uint16 level, uint8 face = 0) const override { return true; } + bool isMipAvailable(uint16 level, uint8 face = 0) const override; - void assignMipData(uint16 level, const storage::StoragePointer& storage) override { - throw std::runtime_error("Invalid call"); - } + void assignMipData(uint16 level, const storage::StoragePointer& storage) override; + + void assignMipFaceData(uint16 level, uint8 face, const storage::StoragePointer& storage) override; - void assignMipFaceData(uint16 level, uint8 face, const storage::StoragePointer& storage) override { - throw std::runtime_error("Invalid call"); - } void reset() override { } protected: diff --git a/libraries/gpu/src/gpu/Texture_ktx.cpp b/libraries/gpu/src/gpu/Texture_ktx.cpp index db6808a866..487d32f91d 100644 --- a/libraries/gpu/src/gpu/Texture_ktx.cpp +++ b/libraries/gpu/src/gpu/Texture_ktx.cpp @@ -72,6 +72,25 @@ Size KtxStorage::getMipFaceSize(uint16 level, uint8 face) const { return _ktxDescriptor->getMipFaceTexelsSize(level, face); } + +bool KtxStorage::isMipAvailable(uint16 level, uint8 face) const { + auto numLevels = _ktxDescriptor->header.numberOfMipmapLevels; + auto minLevel = 7 > numLevels ? 0 : numLevels - 7; + auto avail = level >= minLevel; + qDebug() << "isMipAvailable: " << level << " " << face << avail << minLevel << " " << _ktxDescriptor->header.numberOfMipmapLevels; + //return true; + return level > _ktxDescriptor->header.numberOfMipmapLevels - 7; +} + +void KtxStorage::assignMipData(uint16 level, const storage::StoragePointer& storage) { + throw std::runtime_error("Invalid call"); +} + +void KtxStorage::assignMipFaceData(uint16 level, uint8 face, const storage::StoragePointer& storage) { + throw std::runtime_error("Invalid call"); +} + + void Texture::setKtxBacking(const std::string& filename) { // Check the KTX file for validity before using it as backing storage { @@ -86,6 +105,7 @@ void Texture::setKtxBacking(const std::string& filename) { setStorage(newBacking); } + ktx::KTXUniquePointer Texture::serialize(const Texture& texture) { ktx::Header header; diff --git a/libraries/model-networking/src/model-networking/TextureCache.cpp b/libraries/model-networking/src/model-networking/TextureCache.cpp index 593a7c163e..f4fe192a4a 100644 --- a/libraries/model-networking/src/model-networking/TextureCache.cpp +++ b/libraries/model-networking/src/model-networking/TextureCache.cpp @@ -330,6 +330,166 @@ private: int _maxNumPixels; }; +const uint16_t NetworkTexture::NULL_MIP_LEVEL = std::numeric_limits::max(); +void NetworkTexture::makeRequest() { + if (!_sourceIsKTX) { + Resource::makeRequest(); + return; + } + + // We special-handle ktx requests to run 2 concurrent requests right off the bat + PROFILE_ASYNC_BEGIN(resource, "Resource:" + getType(), QString::number(_requestID), { { "url", _url.toString() }, { "activeURL", _activeUrl.toString() } }); + + if (!_ktxHeaderLoaded) { + qDebug() << ">>> Making request to " << _url << " for header"; + _ktxHeaderRequest = ResourceManager::createResourceRequest(this, _activeUrl); + + if (!_ktxHeaderRequest) { + //qCDebug(networking).noquote() << "Failed to get request for" << _url.toDisplayString(); + ResourceCache::requestCompleted(_self); + finishedLoading(false); + PROFILE_ASYNC_END(resource, "Resource:" + getType(), QString::number(_requestID)); + return; + } + + ByteRange range; + range.fromInclusive = 0; + range.toExclusive = 1000; + _ktxHeaderRequest->setByteRange(range); + + //qCDebug(resourceLog).noquote() << "Starting request for:" << _url.toDisplayString(); + emit loading(); + + connect(_ktxHeaderRequest, &ResourceRequest::progress, this, &NetworkTexture::ktxHeaderRequestProgress); + //connect(this, &Resource::onProgress, this, &NetworkTexture::ktxHeaderRequestFinished); + + connect(_ktxHeaderRequest, &ResourceRequest::finished, this, &NetworkTexture::ktxHeaderRequestFinished); + + _bytesReceived = _bytesTotal = _bytes = 0; + + _ktxHeaderRequest->send(); + } + + startMipRangeRequest(NULL_MIP_LEVEL, NULL_MIP_LEVEL); +} + +// Load mips in the range [low, high] (inclusive) +void NetworkTexture::startMipRangeRequest(uint16_t low, uint16_t high) { + if (_ktxMipRequest) { + return; + } + + bool isHighMipRequest = low == NULL_MIP_LEVEL && high == NULL_MIP_LEVEL; + + if (!isHighMipRequest && !_ktxHeaderLoaded) { + return; + } + + _ktxMipRequest = ResourceManager::createResourceRequest(this, _activeUrl); + qDebug() << ">>> Making request to " << _url << " for " << low << " to " << high; + + if (isHighMipRequest) { + // This is a special case where we load the high 7 mips + ByteRange range; + range.fromInclusive = -15000; + _ktxMipRequest->setByteRange(range); + } else { + // TODO: Discover range for other mips + } + + connect(_ktxMipRequest, &ResourceRequest::progress, this, &NetworkTexture::ktxMipRequestProgress); + connect(_ktxMipRequest, &ResourceRequest::finished, this, &NetworkTexture::ktxMipRequestFinished); + + _ktxMipRequest->send(); +} + + +void NetworkTexture::ktxHeaderRequestFinished() { + assert(!_ktxHeaderLoaded); + + if (_ktxHeaderRequest->getResult() == ResourceRequest::Success) { + _ktxHeaderLoaded = true; + _ktxHeaderData = _ktxHeaderRequest->getData(); + maybeCreateKTX(); + } else { + handleFailedRequest(_ktxHeaderRequest->getResult()); + } + _ktxHeaderRequest->deleteLater(); + _ktxHeaderRequest = nullptr; +} + +void NetworkTexture::ktxMipRequestFinished() { + bool isHighMipRequest = _ktxMipLevelRangeInFlight.first == NULL_MIP_LEVEL + && _ktxMipLevelRangeInFlight.second == NULL_MIP_LEVEL; + + if (_ktxMipRequest->getResult() == ResourceRequest::Success) { + _ktxHighMipData = _ktxMipRequest->getData(); + maybeCreateKTX(); + } else { + handleFailedRequest(_ktxHeaderRequest->getResult()); + } + _ktxMipRequest->deleteLater(); + _ktxMipRequest = nullptr; +} + +// This is called when the header or top mips have been loaded +void NetworkTexture::maybeCreateKTX() { + qDebug() << "Maybe create ktx..."; + if (_ktxHeaderData.size() > 0 && _ktxHighMipData.size() > 0) { + // create ktx... + auto header = reinterpret_cast(_ktxHeaderData.data()); + + qDebug() << "Identifier:" << QString(QByteArray((char*)header->identifier, 12)); + qDebug() << "Type:" << header->glType; + qDebug() << "TypeSize:" << header->glTypeSize; + qDebug() << "numberOfArrayElements:" << header->numberOfArrayElements; + qDebug() << "numberOfFaces:" << header->numberOfFaces; + qDebug() << "numberOfMipmapLevels:" << header->numberOfMipmapLevels; + auto kvSize = header->bytesOfKeyValueData; + if (kvSize > _ktxHeaderData.size() - ktx::KTX_HEADER_SIZE) { + qWarning() << "Cannot load " << _url << ", did not receive all kv data with initial request"; + return; + } + + auto keyValues = ktx::KTX::parseKeyValues(header->bytesOfKeyValueData, reinterpret_cast(_ktxHeaderData.data()) + ktx::KTX_HEADER_SIZE); + + // Create bare ktx in memory + std::string filename = "test"; + auto memKtx = ktx::KTX::createBare(*header, keyValues); + + auto d = const_cast(memKtx->getStorage()->data()); + memcpy(d + memKtx->_storage->size() - _ktxHighMipData.size(), _ktxHighMipData.data(), _ktxHighMipData.size()); + + auto textureCache = DependencyManager::get(); + + // Move ktx to file + const char* data = reinterpret_cast(memKtx->_storage->data()); + size_t length = memKtx->_storage->size(); + KTXFilePointer file; + auto& ktxCache = textureCache->_ktxCache; + if (!memKtx || !(file = ktxCache.writeFile(data, KTXCache::Metadata(filename, length)))) { + qCWarning(modelnetworking) << _url << "file cache failed"; + } else { + _file = file; + } + + //auto texture = gpu::Texture::serializeHeader("test.ktx", *header, keyValues); + gpu::TexturePointer texture; + texture.reset(gpu::Texture::unserialize(_file->getFilepath(), memKtx->toDescriptor())); + texture->setKtxBacking(file->getFilepath()); + + // We replace the texture with the one stored in the cache. This deals with the possible race condition of two different + // images with the same hash being loaded concurrently. Only one of them will make it into the cache by hash first and will + // be the winner + if (textureCache) { + texture = textureCache->cacheTextureByHash(filename, texture); + } + + setImage(texture, header->getPixelWidth(), header->getPixelHeight()); + + } +} + void NetworkTexture::downloadFinished(const QByteArray& data) { loadContent(data); } diff --git a/libraries/model-networking/src/model-networking/TextureCache.h b/libraries/model-networking/src/model-networking/TextureCache.h index 40eb29de35..3998ba3408 100644 --- a/libraries/model-networking/src/model-networking/TextureCache.h +++ b/libraries/model-networking/src/model-networking/TextureCache.h @@ -59,7 +59,16 @@ public: signals: void networkTextureCreated(const QWeakPointer& self); +public slots: + void ktxHeaderRequestProgress(uint64_t bytesReceived, uint64_t bytesTotal) { } + void ktxHeaderRequestFinished(); + + void ktxMipRequestProgress(uint64_t bytesReceived, uint64_t bytesTotal) { } + void ktxMipRequestFinished(); + protected: + void makeRequest() override; + virtual bool isCacheable() const override { return _loaded; } virtual void downloadFinished(const QByteArray& data) override; @@ -67,6 +76,9 @@ protected: Q_INVOKABLE void loadContent(const QByteArray& content); Q_INVOKABLE void setImage(gpu::TexturePointer texture, int originalWidth, int originalHeight); + void startMipRangeRequest(uint16_t low, uint16_t high); + void maybeCreateKTX(); + private: friend class KTXReader; friend class ImageReader; @@ -79,9 +91,18 @@ private: DONE_LOADING }; + KTXLoadState _ktxLoadState { LOADING_HEADER }; KTXFilePointer _file; + static const uint16_t NULL_MIP_LEVEL; bool _sourceIsKTX { false }; + bool _ktxHeaderLoaded { false }; + std::pair _ktxMipLevelRangeInFlight{ NULL_MIP_LEVEL, NULL_MIP_LEVEL }; + ResourceRequest* _ktxHeaderRequest { nullptr }; + ResourceRequest* _ktxMipRequest { nullptr }; + QByteArray _ktxHeaderData; + QByteArray _ktxHighMipData; + int _originalWidth { 0 }; int _originalHeight { 0 }; int _width { 0 }; diff --git a/libraries/networking/src/HTTPResourceRequest.cpp b/libraries/networking/src/HTTPResourceRequest.cpp index 5932193830..1bb292a3ef 100644 --- a/libraries/networking/src/HTTPResourceRequest.cpp +++ b/libraries/networking/src/HTTPResourceRequest.cpp @@ -60,7 +60,12 @@ void HTTPResourceRequest::doSend() { } if (_byteRange.isSet()) { - auto byteRange = QString("bytes=%1-%2").arg(_byteRange.fromInclusive).arg(_byteRange.toExclusive); + QString byteRange; + if (_byteRange.fromInclusive < 0) { + auto byteRange = QString("bytes=%1").arg(_byteRange.fromInclusive); + } else { + auto byteRange = QString("bytes=%1-%2").arg(_byteRange.fromInclusive).arg(_byteRange.toExclusive); + } networkRequest.setRawHeader("Range", byteRange.toLatin1()); } networkRequest.setAttribute(QNetworkRequest::HttpPipeliningAllowedAttribute, true); diff --git a/libraries/networking/src/ResourceCache.cpp b/libraries/networking/src/ResourceCache.cpp index 8373a9cf99..62ea85bc86 100644 --- a/libraries/networking/src/ResourceCache.cpp +++ b/libraries/networking/src/ResourceCache.cpp @@ -725,34 +725,7 @@ void Resource::handleReplyFinished() { emit loaded(data); downloadFinished(data); } else { - switch (result) { - case ResourceRequest::Result::Timeout: { - qCDebug(networking) << "Timed out loading" << _url << "received" << _bytesReceived << "total" << _bytesTotal; - // Fall through to other cases - } - case ResourceRequest::Result::ServerUnavailable: { - // retry with increasing delays - const int BASE_DELAY_MS = 1000; - if (_attempts++ < MAX_ATTEMPTS) { - auto waitTime = BASE_DELAY_MS * (int)pow(2.0, _attempts); - - qCDebug(networking).noquote() << "Server unavailable for" << _url << "- may retry in" << waitTime << "ms" - << "if resource is still needed"; - - QTimer::singleShot(waitTime, this, &Resource::attemptRequest); - break; - } - // fall through to final failure - } - default: { - qCDebug(networking) << "Error loading " << _url; - auto error = (result == ResourceRequest::Timeout) ? QNetworkReply::TimeoutError - : QNetworkReply::UnknownNetworkError; - emit failed(error); - finishedLoading(false); - break; - } - } + handleFailedRequest(result); } _request->disconnect(this); @@ -760,6 +733,37 @@ void Resource::handleReplyFinished() { _request = nullptr; } +void Resource::handleFailedRequest(ResourceRequest::Result result) { + switch (result) { + case ResourceRequest::Result::Timeout: { + qCDebug(networking) << "Timed out loading" << _url << "received" << _bytesReceived << "total" << _bytesTotal; + // Fall through to other cases + } + case ResourceRequest::Result::ServerUnavailable: { + // retry with increasing delays + const int BASE_DELAY_MS = 1000; + if (_attempts++ < MAX_ATTEMPTS) { + auto waitTime = BASE_DELAY_MS * (int)pow(2.0, _attempts); + + qCDebug(networking).noquote() << "Server unavailable for" << _url << "- may retry in" << waitTime << "ms" + << "if resource is still needed"; + + QTimer::singleShot(waitTime, this, &Resource::attemptRequest); + break; + } + // fall through to final failure + } + default: { + qCDebug(networking) << "Error loading " << _url; + auto error = (result == ResourceRequest::Timeout) ? QNetworkReply::TimeoutError + : QNetworkReply::UnknownNetworkError; + emit failed(error); + finishedLoading(false); + break; + } + } +} + uint qHash(const QPointer& value, uint seed) { return qHash(value.data(), seed); } diff --git a/libraries/networking/src/ResourceCache.h b/libraries/networking/src/ResourceCache.h index 32364bf71c..e699c3d198 100644 --- a/libraries/networking/src/ResourceCache.h +++ b/libraries/networking/src/ResourceCache.h @@ -424,6 +424,8 @@ protected slots: protected: virtual void init(); + virtual void makeRequest(); + /// Checks whether the resource is cacheable. virtual bool isCacheable() const { return true; } @@ -440,6 +442,8 @@ protected: Q_INVOKABLE void allReferencesCleared(); + void handleFailedRequest(ResourceRequest::Result result); + QUrl _url; QUrl _activeUrl; ByteRange _requestByteRange; @@ -449,8 +453,15 @@ protected: QHash, float> _loadPriorities; QWeakPointer _self; QPointer _cache; + + qint64 _bytesReceived{ 0 }; + qint64 _bytesTotal{ 0 }; + qint64 _bytes{ 0 }; + + int _requestID; + ResourceRequest* _request{ nullptr }; -private slots: +public slots: void handleDownloadProgress(uint64_t bytesReceived, uint64_t bytesTotal); void handleReplyFinished(); @@ -460,20 +471,14 @@ private: void setLRUKey(int lruKey) { _lruKey = lruKey; } - void makeRequest(); void retry(); void reinsert(); bool isInScript() const { return _isInScript; } void setInScript(bool isInScript) { _isInScript = isInScript; } - int _requestID; - ResourceRequest* _request{ nullptr }; int _lruKey{ 0 }; QTimer* _replyTimer{ nullptr }; - qint64 _bytesReceived{ 0 }; - qint64 _bytesTotal{ 0 }; - qint64 _bytes{ 0 }; int _attempts{ 0 }; bool _isInScript{ false }; }; diff --git a/libraries/networking/src/ResourceRequest.h b/libraries/networking/src/ResourceRequest.h index 25db34ab0a..01ca62cf05 100644 --- a/libraries/networking/src/ResourceRequest.h +++ b/libraries/networking/src/ResourceRequest.h @@ -21,7 +21,7 @@ struct ByteRange { int64_t fromInclusive { 0 }; int64_t toExclusive { 0 }; - bool isSet() { return fromInclusive < -1 || fromInclusive < toExclusive; } + bool isSet() { return fromInclusive < 0 || fromInclusive < toExclusive; } }; class ResourceRequest : public QObject {