From 00cbfa0f70c0776a0e38739e7e9d692ecfa5e9ec Mon Sep 17 00:00:00 2001 From: Ryan Huffman Date: Fri, 7 Apr 2017 16:48:22 -0700 Subject: [PATCH] Add start of progressive ktx-loading --- interface/src/ui/overlays/Image3DOverlay.cpp | 1 - libraries/gpu-gl/src/gpu/gl/GLTexture.cpp | 8 ++- libraries/gpu/src/gpu/Texture.h | 10 ++- libraries/gpu/src/gpu/Texture_ktx.cpp | 11 +++ libraries/ktx/src/ktx/KTX.cpp | 23 +++++- libraries/ktx/src/ktx/KTX.h | 11 ++- libraries/ktx/src/ktx/Writer.cpp | 62 ++++++++++++++++ .../src/model-networking/TextureCache.cpp | 72 +++++++++++++++++++ .../src/model-networking/TextureCache.h | 8 +++ .../networking/src/HTTPResourceRequest.cpp | 53 +++++++++++++- .../networking/src/NetworkAccessManager.cpp | 10 ++- libraries/networking/src/ResourceRequest.h | 4 ++ libraries/shared/src/shared/Storage.cpp | 1 + libraries/shared/src/shared/Storage.h | 1 + 14 files changed, 264 insertions(+), 11 deletions(-) diff --git a/interface/src/ui/overlays/Image3DOverlay.cpp b/interface/src/ui/overlays/Image3DOverlay.cpp index f06efcef1c..45d63d9cf1 100644 --- a/interface/src/ui/overlays/Image3DOverlay.cpp +++ b/interface/src/ui/overlays/Image3DOverlay.cpp @@ -53,7 +53,6 @@ void Image3DOverlay::update(float deltatime) { } void Image3DOverlay::render(RenderArgs* args) { - qDebug() << _url; if (!_isLoaded) { _isLoaded = true; _texture = DependencyManager::get()->getTexture(_url); diff --git a/libraries/gpu-gl/src/gpu/gl/GLTexture.cpp b/libraries/gpu-gl/src/gpu/gl/GLTexture.cpp index a6e6bf4fa3..b7d2ee0b0f 100644 --- a/libraries/gpu-gl/src/gpu/gl/GLTexture.cpp +++ b/libraries/gpu-gl/src/gpu/gl/GLTexture.cpp @@ -217,8 +217,12 @@ TransferJob::TransferJob(const GLTexture& parent, uint16_t sourceMip, uint16_t t _transferSize = mipSize; _bufferingLambda = [=] { auto mipData = _parent._gpuObject.accessStoredMipFace(sourceMip, face); - _buffer.resize(_transferSize); - memcpy(&_buffer[0], mipData->readData(), _transferSize); + if (!mipData) { + qWarning() << "Mip not available: " << sourceMip; + } else { + _buffer.resize(_transferSize); + memcpy(&_buffer[0], mipData->readData(), _transferSize); + } _bufferingCompleted = true; }; diff --git a/libraries/gpu/src/gpu/Texture.h b/libraries/gpu/src/gpu/Texture.h index 2f63bd6719..756748497d 100755 --- a/libraries/gpu/src/gpu/Texture.h +++ b/libraries/gpu/src/gpu/Texture.h @@ -28,6 +28,8 @@ namespace ktx { struct KTXDescriptor; using KTXDescriptorPointer = std::unique_ptr; struct Header; + struct KeyValue; + using KeyValues = std::list; } namespace gpu { @@ -503,9 +505,15 @@ public: ExternalUpdates getUpdates() const; - // Textures can be serialized directly to ktx data file, here is how + // Serialize ktx header and keyvalues directly to a file, and return a Texture representing that file + static Texture* serializeHeader(const std::string& ktxfile, const ktx::Header& header, const ktx::KeyValues& keyValues); + + // Serialize a texture into a KTX file static ktx::KTXUniquePointer serialize(const Texture& texture); + static TexturePointer unserialize(const std::string& ktxFile, TextureUsageType usageType = TextureUsageType::RESOURCE, Usage usage = Usage(), const Sampler::Desc& sampler = Sampler::Desc()); + static TexturePointer unserialize(const std::string& ktxFile, const ktx::KTXDescriptor& descriptor, TextureUsageType usageType = TextureUsageType::RESOURCE, Usage usage = Usage(), const Sampler::Desc& sampler = Sampler::Desc()); + static bool evalKTXFormat(const Element& mipFormat, const Element& texelFormat, ktx::Header& header); static bool evalTextureFormat(const ktx::Header& header, Element& mipFormat, Element& texelFormat); diff --git a/libraries/gpu/src/gpu/Texture_ktx.cpp b/libraries/gpu/src/gpu/Texture_ktx.cpp index 50e9cb6d07..db6808a866 100644 --- a/libraries/gpu/src/gpu/Texture_ktx.cpp +++ b/libraries/gpu/src/gpu/Texture_ktx.cpp @@ -207,6 +207,10 @@ TexturePointer Texture::unserialize(const std::string& ktxfile, TextureUsageType } ktx::KTXDescriptor descriptor { ktxPointer->toDescriptor() }; + return unserialize(ktxfile, ktxPointer->toDescriptor(), usageType, usage, sampler); +} + +TexturePointer Texture::unserialize(const std::string& ktxfile, const ktx::KTXDescriptor& descriptor, TextureUsageType usageType, Usage usage, const Sampler::Desc& sampler) { const auto& header = descriptor.header; Format mipFormat = Format::COLOR_BGRA_32; @@ -256,6 +260,13 @@ TexturePointer Texture::unserialize(const std::string& ktxfile, TextureUsageType return tex; } +Texture* Texture::serializeHeader(const std::string& ktxfile, const ktx::Header& header, const ktx::KeyValues& keyValues) { + // Create a memory-backed KTX object + auto ktxBuffer = ktx::KTX::createBare(header, keyValues); + + return unserialize(ktxfile, ktxBuffer->toDescriptor()); +} + bool Texture::evalKTXFormat(const Element& mipFormat, const Element& texelFormat, ktx::Header& header) { if (texelFormat == Format::COLOR_RGBA_32 && mipFormat == Format::COLOR_BGRA_32) { header.setUncompressed(ktx::GLType::UNSIGNED_BYTE, 1, ktx::GLFormat::BGRA, ktx::GLInternalFormat_Uncompressed::RGBA8, ktx::GLBaseInternalFormat::RGBA); diff --git a/libraries/ktx/src/ktx/KTX.cpp b/libraries/ktx/src/ktx/KTX.cpp index 6fca39788b..0580d9a8c3 100644 --- a/libraries/ktx/src/ktx/KTX.cpp +++ b/libraries/ktx/src/ktx/KTX.cpp @@ -45,7 +45,7 @@ uint32_t Header::evalPixelDepth(uint32_t level) const { } size_t Header::evalPixelSize() const { - return glTypeSize; // Really we should generate the size from the FOrmat etc + return 4;//glTypeSize; // Really we should generate the size from the FOrmat etc } size_t Header::evalRowSize(uint32_t level) const { @@ -70,6 +70,25 @@ size_t Header::evalImageSize(uint32_t level) const { } } +ImageDescriptors Header::generateImageDescriptors() const { + ImageDescriptors descriptors; + + for (auto level = 0; level < numberOfMipmapLevels; ++level) { + ImageHeader header { + numberOfFaces == NUM_CUBEMAPFACES, + static_cast(evalImageSize(level)), + 0 + }; + ImageHeader::FaceOffsets offsets; + for (auto i = 0; i < numberOfFaces; ++i) { + offsets.push_back(0); + } + descriptors.push_back(ImageDescriptor(header, offsets)); + } + + return descriptors; +} + KeyValue::KeyValue(const std::string& key, uint32_t valueByteSize, const Byte* value) : _byteSize((uint32_t) key.size() + 1 + valueByteSize), // keyString size + '\0' ending char + the value size @@ -209,4 +228,4 @@ KTXDescriptor KTX::toDescriptor() const { KTX::KTX(const StoragePointer& storage, const Header& header, const KeyValues& keyValues, const Images& images) : _header(header), _storage(storage), _keyValues(keyValues), _images(images) { -} \ No newline at end of file +} diff --git a/libraries/ktx/src/ktx/KTX.h b/libraries/ktx/src/ktx/KTX.h index 7eab67c0db..fb8927eca0 100644 --- a/libraries/ktx/src/ktx/KTX.h +++ b/libraries/ktx/src/ktx/KTX.h @@ -292,6 +292,9 @@ namespace ktx { using Storage = storage::Storage; using StoragePointer = std::shared_ptr; + struct ImageDescriptor; + using ImageDescriptors = std::vector; + // Header struct Header { static const size_t IDENTIFIER_LENGTH = 12; @@ -378,6 +381,7 @@ namespace ktx { void setCube(uint32_t width, uint32_t height) { setDimensions(width, height, 0, 0, NUM_CUBEMAPFACES); } void setCubeArray(uint32_t width, uint32_t height, uint32_t numSlices) { setDimensions(width, height, 0, (numSlices > 0 ? numSlices : 1), NUM_CUBEMAPFACES); } + ImageDescriptors generateImageDescriptors() const; }; static const size_t KTX_HEADER_SIZE = 64; static_assert(sizeof(Header) == KTX_HEADER_SIZE, "KTX Header size is static"); @@ -421,14 +425,14 @@ namespace ktx { struct Image; + // Image without the image data itself struct ImageDescriptor : public ImageHeader { const FaceOffsets _faceOffsets; ImageDescriptor(const ImageHeader& header, const FaceOffsets& offsets) : ImageHeader(header), _faceOffsets(offsets) {} Image toImage(const ktx::StoragePointer& storage) const; }; - using ImageDescriptors = std::vector; - + // Image with the image data itself struct Image : public ImageHeader { FaceBytes _faceBytes; Image(const ImageHeader& header, const FaceBytes& faces) : ImageHeader(header), _faceBytes(faces) {} @@ -473,6 +477,7 @@ namespace ktx { // This path allocate the Storage where to store header, keyvalues and copy mips // Then COPY all the data static std::unique_ptr create(const Header& header, const Images& images, const KeyValues& keyValues = KeyValues()); + static std::unique_ptr createBare(const Header& header, const KeyValues& keyValues = KeyValues()); // Instead of creating a full Copy of the src data in a KTX object, the write serialization can be performed with the // following two functions @@ -486,7 +491,9 @@ namespace ktx { // // This is exactly what is done in the create function static size_t evalStorageSize(const Header& header, const Images& images, const KeyValues& keyValues = KeyValues()); + static size_t evalStorageSize(const Header& header, const ImageDescriptors& images, const KeyValues& keyValues = KeyValues()); static size_t write(Byte* destBytes, size_t destByteSize, const Header& header, const Images& images, const KeyValues& keyValues = KeyValues()); + static size_t writeWithoutImages(Byte* destBytes, size_t destByteSize, const Header& header, const ImageDescriptors& descriptors, const KeyValues& keyValues = KeyValues()); static size_t writeKeyValues(Byte* destBytes, size_t destByteSize, const KeyValues& keyValues); static Images writeImages(Byte* destBytes, size_t destByteSize, const Images& images); diff --git a/libraries/ktx/src/ktx/Writer.cpp b/libraries/ktx/src/ktx/Writer.cpp index 25b363d31b..396aa09f9a 100644 --- a/libraries/ktx/src/ktx/Writer.cpp +++ b/libraries/ktx/src/ktx/Writer.cpp @@ -40,6 +40,20 @@ namespace ktx { return create(storagePointer); } + std::unique_ptr KTX::createBare(const Header& header, const KeyValues& keyValues) { + auto descriptors = header.generateImageDescriptors(); + + StoragePointer storagePointer; + { + auto storageSize = ktx::KTX::evalStorageSize(header, descriptors, keyValues); + auto memoryStorage = new storage::MemoryStorage(storageSize); + qDebug() << "Memory storage size is: " << storageSize; + ktx::KTX::writeWithoutImages(memoryStorage->data(), memoryStorage->size(), header, descriptors, keyValues); + storagePointer.reset(memoryStorage); + } + return create(storagePointer); + } + size_t KTX::evalStorageSize(const Header& header, const Images& images, const KeyValues& keyValues) { size_t storageSize = sizeof(Header); @@ -59,6 +73,25 @@ namespace ktx { return storageSize; } + size_t KTX::evalStorageSize(const Header& header, const ImageDescriptors& imageDescriptors, const KeyValues& keyValues) { + size_t storageSize = sizeof(Header); + + if (!keyValues.empty()) { + size_t keyValuesSize = KeyValue::serializedKeyValuesByteSize(keyValues); + storageSize += keyValuesSize; + } + + auto numMips = header.getNumberOfLevels(); + for (uint32_t l = 0; l < numMips; l++) { + if (imageDescriptors.size() > l) { + storageSize += sizeof(uint32_t); + storageSize += imageDescriptors[l]._imageSize; + storageSize += Header::evalPadding(imageDescriptors[l]._imageSize); + } + } + return storageSize; + } + size_t KTX::write(Byte* destBytes, size_t destByteSize, const Header& header, const Images& srcImages, const KeyValues& keyValues) { // Check again that we have enough destination capacity if (!destBytes || (destByteSize < evalStorageSize(header, srcImages, keyValues))) { @@ -87,6 +120,35 @@ namespace ktx { return destByteSize; } + size_t KTX::writeWithoutImages(Byte* destBytes, size_t destByteSize, const Header& header, const ImageDescriptors& descriptors, const KeyValues& keyValues) { + // Check again that we have enough destination capacity + if (!destBytes || (destByteSize < evalStorageSize(header, descriptors, keyValues))) { + return 0; + } + + auto currentDestPtr = destBytes; + // Header + auto destHeader = reinterpret_cast(currentDestPtr); + memcpy(currentDestPtr, &header, sizeof(Header)); + currentDestPtr += sizeof(Header); + + // KeyValues + if (!keyValues.empty()) { + destHeader->bytesOfKeyValueData = (uint32_t) writeKeyValues(currentDestPtr, destByteSize - sizeof(Header), keyValues); + } else { + // Make sure the header contains the right bytesOfKeyValueData size + destHeader->bytesOfKeyValueData = 0; + } + currentDestPtr += destHeader->bytesOfKeyValueData; + + for (int i = 0; i < descriptors.size(); ++i) { + *currentDestPtr = descriptors[i]._imageSize; + currentDestPtr += descriptors[i]._imageSize + sizeof(uint32_t); + } + + return destByteSize; + } + uint32_t KeyValue::writeSerializedKeyAndValue(Byte* destBytes, uint32_t destByteSize, const KeyValue& keyval) { uint32_t keyvalSize = keyval.serializedByteSize(); if (keyvalSize > destByteSize) { diff --git a/libraries/model-networking/src/model-networking/TextureCache.cpp b/libraries/model-networking/src/model-networking/TextureCache.cpp index 45000b30a8..6b6fc09975 100644 --- a/libraries/model-networking/src/model-networking/TextureCache.cpp +++ b/libraries/model-networking/src/model-networking/TextureCache.cpp @@ -335,6 +335,77 @@ void NetworkTexture::downloadFinished(const QByteArray& data) { } void NetworkTexture::loadContent(const QByteArray& content) { + if (_sourceIsKTX) { + if (_ktxLoadState == LOADING_HEADER) { + // TODO Handle case where we already have the source hash texture on disk + // TODO Handle case where data isn't as large as the ktx header + _ktxLoadState = LOADING_LOWEST_SIX; + auto header = reinterpret_cast(content.data()); + qDebug() << "Identifier:" << QString(QByteArray((char*)header->identifier, 12)); + qDebug() << "Type:" << header->glType; + qDebug() << "TypeSize:" << header->glTypeSize; + qDebug() << "numberOfArrayElements:" << header->numberOfArrayElements; + qDebug() << "numberOfFaces:" << header->numberOfFaces; + qDebug() << "numberOfMipmapLevels:" << header->numberOfMipmapLevels; + auto kvSize = header->bytesOfKeyValueData; + if (kvSize > content.size() - ktx::KTX_HEADER_SIZE) { + qWarning() << "Cannot load " << _url << ", did not receive all kv data with initial request"; + return; + } + + auto keyValues = ktx::KTX::parseKeyValues(header->bytesOfKeyValueData, reinterpret_cast(content.data()) + ktx::KTX_HEADER_SIZE); + + // Create bare ktx in memory + std::string filename = "test"; + auto memKtx = ktx::KTX::createBare(*header, keyValues); + + + auto textureCache = DependencyManager::get(); + + // Move ktx to file + const char* data = reinterpret_cast(memKtx->_storage->data()); + size_t length = memKtx->_storage->size(); + KTXFilePointer file; + auto& ktxCache = textureCache->_ktxCache; + if (!memKtx || !(file = ktxCache.writeFile(data, KTXCache::Metadata(filename, length)))) { + qCWarning(modelnetworking) << _url << "file cache failed"; + } else { + _file = file; + } + + //auto texture = gpu::Texture::serializeHeader("test.ktx", *header, keyValues); + gpu::TexturePointer texture; + texture.reset(gpu::Texture::unserialize(_file->getFilepath(), memKtx->toDescriptor())); + texture->setKtxBacking(file->getFilepath()); + + // We replace the texture with the one stored in the cache. This deals with the possible race condition of two different + // images with the same hash being loaded concurrently. Only one of them will make it into the cache by hash first and will + // be the winner + if (textureCache) { + texture = textureCache->cacheTextureByHash(filename, texture); + } + + setImage(texture, header->getPixelWidth(), header->getPixelHeight()); + + + auto desc = memKtx->toDescriptor(); + int numMips = desc.images.size(); + auto numMipsToGet = glm::min(numMips, 6); + auto sizeOfTopMips = 0; + for (int i = 0; i < numMipsToGet; ++i) { + auto& img = desc.images[i]; + sizeOfTopMips += img._imageSize; + } + _requestByteRange.fromInclusive = length - sizeOfTopMips; + _requestByteRange.toExclusive = length; + attemptRequest(); + + } else { + qDebug() << "Got highest 6 mips"; + } + return; + } + QThreadPool::globalInstance()->start(new ImageReader(_self, _url, content, _maxNumPixels)); } @@ -457,6 +528,7 @@ void ImageReader::read() { if (texture && textureCache) { auto memKtx = gpu::Texture::serialize(*texture); + // Move the texture into a memory mapped file if (memKtx) { const char* data = reinterpret_cast(memKtx->_storage->data()); size_t length = memKtx->_storage->size(); diff --git a/libraries/model-networking/src/model-networking/TextureCache.h b/libraries/model-networking/src/model-networking/TextureCache.h index d0916d61eb..40eb29de35 100644 --- a/libraries/model-networking/src/model-networking/TextureCache.h +++ b/libraries/model-networking/src/model-networking/TextureCache.h @@ -72,6 +72,14 @@ private: friend class ImageReader; image::TextureUsage::Type _type; + + enum KTXLoadState { + LOADING_HEADER, + LOADING_LOWEST_SIX, + DONE_LOADING + }; + + KTXLoadState _ktxLoadState { LOADING_HEADER }; KTXFilePointer _file; bool _sourceIsKTX { false }; int _originalWidth { 0 }; diff --git a/libraries/networking/src/HTTPResourceRequest.cpp b/libraries/networking/src/HTTPResourceRequest.cpp index 28d0485383..5932193830 100644 --- a/libraries/networking/src/HTTPResourceRequest.cpp +++ b/libraries/networking/src/HTTPResourceRequest.cpp @@ -60,7 +60,7 @@ void HTTPResourceRequest::doSend() { } if (_byteRange.isSet()) { - auto byteRange = QString("bytes={}-{}").arg(_byteRange.fromInclusive).arg(_byteRange.toExclusive); + auto byteRange = QString("bytes=%1-%2").arg(_byteRange.fromInclusive).arg(_byteRange.toExclusive); networkRequest.setRawHeader("Range", byteRange.toLatin1()); } networkRequest.setAttribute(QNetworkRequest::HttpPipeliningAllowedAttribute, true); @@ -78,12 +78,61 @@ void HTTPResourceRequest::onRequestFinished() { Q_ASSERT(_reply); cleanupTimer(); - + + // Content-Range headers have the form: + // + // Content-Range: -/ + // Content-Range: -/* + // Content-Range: */ + // + auto parseContentRangeHeader = [](QString contentRangeHeader) -> std::pair { + auto unitRangeParts = contentRangeHeader.split(' '); + if (unitRangeParts.size() != 2) { + return { false, 0 }; + } + + auto rangeSizeParts = unitRangeParts[1].split('/'); + if (rangeSizeParts.size() != 2) { + return { false, 0 }; + } + + auto sizeStr = rangeSizeParts[1]; + if (sizeStr == "*") { + return { true, 0 }; + } else { + bool ok; + auto size = sizeStr.toLong(&ok); + return { ok, size }; + } + }; + switch(_reply->error()) { case QNetworkReply::NoError: _data = _reply->readAll(); _loadedFromCache = _reply->attribute(QNetworkRequest::SourceIsFromCacheAttribute).toBool(); _result = Success; + + if (_byteRange.isSet()) { + auto statusCode = _reply->attribute(QNetworkRequest::HttpStatusCodeAttribute).toInt(); + if (statusCode == 206) { + _rangeRequestSuccessful = true; + auto contentRangeHeader = _reply->rawHeader("Content-Range"); + bool success; + uint64_t size; + std::tie(success, size) = parseContentRangeHeader(contentRangeHeader); + if (success) { + qWarning(networking) << "Total http resource size is: " << size; + _totalSizeOfResource = size; + } else { + qWarning(networking) << "Error parsing content-range header: " << contentRangeHeader; + _totalSizeOfResource = 0; + } + } else { + _rangeRequestSuccessful = false; + _totalSizeOfResource = _data.size(); + } + } + break; case QNetworkReply::TimeoutError: diff --git a/libraries/networking/src/NetworkAccessManager.cpp b/libraries/networking/src/NetworkAccessManager.cpp index 73096825e0..6895118be5 100644 --- a/libraries/networking/src/NetworkAccessManager.cpp +++ b/libraries/networking/src/NetworkAccessManager.cpp @@ -13,12 +13,20 @@ #include "AtpReply.h" #include "NetworkAccessManager.h" +#include QThreadStorage networkAccessManagers; QNetworkAccessManager& NetworkAccessManager::getInstance() { if (!networkAccessManagers.hasLocalData()) { - networkAccessManagers.setLocalData(new QNetworkAccessManager()); + auto nm = new QNetworkAccessManager(); + networkAccessManagers.setLocalData(nm); + + QNetworkProxy proxy; + proxy.setType(QNetworkProxy::HttpProxy); + proxy.setHostName("127.0.0.1"); + proxy.setPort(8888); + nm->setProxy(proxy); } return *networkAccessManagers.localData(); diff --git a/libraries/networking/src/ResourceRequest.h b/libraries/networking/src/ResourceRequest.h index 03b46e715d..25db34ab0a 100644 --- a/libraries/networking/src/ResourceRequest.h +++ b/libraries/networking/src/ResourceRequest.h @@ -53,6 +53,8 @@ public: QString getResultString() const; QUrl getUrl() const { return _url; } bool loadedFromCache() const { return _loadedFromCache; } + bool getRangeRequestSuccessful() const { return _rangeRequestSuccessful; } + bool getTotalSizeOfResource() const { return _totalSizeOfResource; } void setCacheEnabled(bool value) { _cacheEnabled = value; } void setByteRange(ByteRange byteRange) { _byteRange = byteRange; } @@ -74,6 +76,8 @@ protected: bool _cacheEnabled { true }; bool _loadedFromCache { false }; ByteRange _byteRange; + bool _rangeRequestSuccessful { false }; + uint64_t _totalSizeOfResource { 0 }; }; #endif diff --git a/libraries/shared/src/shared/Storage.cpp b/libraries/shared/src/shared/Storage.cpp index 3c46347a49..8999caf1e8 100644 --- a/libraries/shared/src/shared/Storage.cpp +++ b/libraries/shared/src/shared/Storage.cpp @@ -67,6 +67,7 @@ StoragePointer FileStorage::create(const QString& filename, size_t size, const u return std::make_shared(filename); } +// Represents a memory mapped file FileStorage::FileStorage(const QString& filename) : _file(filename) { if (_file.open(QFile::ReadOnly)) { _mapped = _file.map(0, _file.size()); diff --git a/libraries/shared/src/shared/Storage.h b/libraries/shared/src/shared/Storage.h index 306984040f..46f45cfdc5 100644 --- a/libraries/shared/src/shared/Storage.h +++ b/libraries/shared/src/shared/Storage.h @@ -20,6 +20,7 @@ namespace storage { class Storage; using StoragePointer = std::shared_ptr; + // Abstract class to represent memory that stored _somewhere_ (in system memory or in a file, for example) class Storage : public std::enable_shared_from_this { public: virtual ~Storage() {}