Add start of progressive ktx-loading

interface/src/ui/overlays/Image3DOverlay.cpp

@@ -53,7 +53,6 @@ void Image3DOverlay::update(float deltatime) {
 }
 
 void Image3DOverlay::render(RenderArgs* args) {
-    qDebug() << _url;
     if (!_isLoaded) {
         _isLoaded = true;
         _texture = DependencyManager::get<TextureCache>()->getTexture(_url);

libraries/gpu-gl/src/gpu/gl/GLTexture.cpp

@@ -217,8 +217,12 @@ TransferJob::TransferJob(const GLTexture& parent, uint16_t sourceMip, uint16_t t
         _transferSize = mipSize;
         _bufferingLambda = [=] {
             auto mipData = _parent._gpuObject.accessStoredMipFace(sourceMip, face);
-            _buffer.resize(_transferSize);
-            memcpy(&_buffer[0], mipData->readData(), _transferSize);
+            if (!mipData) {
+                qWarning() << "Mip not available: " << sourceMip;
+            } else {
+                _buffer.resize(_transferSize);
+                memcpy(&_buffer[0], mipData->readData(), _transferSize);
+            }
             _bufferingCompleted = true;
         };
 

libraries/gpu/src/gpu/Texture.h

@@ -28,6 +28,8 @@ namespace ktx {
     struct KTXDescriptor;
     using KTXDescriptorPointer = std::unique_ptr<KTXDescriptor>;
     struct Header;
+    struct KeyValue;
+    using KeyValues = std::list<KeyValue>;
 }
 
 namespace gpu {
@@ -503,9 +505,15 @@ public:
 
     ExternalUpdates getUpdates() const;
 
-    // Textures can be serialized directly to ktx data file, here is how
+    // Serialize ktx header and keyvalues directly to a file, and return a Texture representing that file
+    static Texture* serializeHeader(const std::string& ktxfile, const ktx::Header& header, const ktx::KeyValues& keyValues);
+
+    // Serialize a texture into a KTX file
     static ktx::KTXUniquePointer serialize(const Texture& texture);
+
     static TexturePointer unserialize(const std::string& ktxFile, TextureUsageType usageType = TextureUsageType::RESOURCE, Usage usage = Usage(), const Sampler::Desc& sampler = Sampler::Desc());
+    static TexturePointer unserialize(const std::string& ktxFile, const ktx::KTXDescriptor& descriptor, TextureUsageType usageType = TextureUsageType::RESOURCE, Usage usage = Usage(), const Sampler::Desc& sampler = Sampler::Desc());
+
     static bool evalKTXFormat(const Element& mipFormat, const Element& texelFormat, ktx::Header& header);
     static bool evalTextureFormat(const ktx::Header& header, Element& mipFormat, Element& texelFormat);
 

libraries/gpu/src/gpu/Texture_ktx.cpp

@@ -207,6 +207,10 @@ TexturePointer Texture::unserialize(const std::string& ktxfile, TextureUsageType
     }
 
     ktx::KTXDescriptor descriptor { ktxPointer->toDescriptor() };
+    return unserialize(ktxfile, ktxPointer->toDescriptor(), usageType, usage, sampler);
+}
+
+TexturePointer Texture::unserialize(const std::string& ktxfile, const ktx::KTXDescriptor& descriptor, TextureUsageType usageType, Usage usage, const Sampler::Desc& sampler) {
     const auto& header = descriptor.header;
 
     Format mipFormat = Format::COLOR_BGRA_32;
@@ -256,6 +260,13 @@ TexturePointer Texture::unserialize(const std::string& ktxfile, TextureUsageType
     return tex;
 }
 
+Texture* Texture::serializeHeader(const std::string& ktxfile, const ktx::Header& header, const ktx::KeyValues& keyValues) {
+    // Create a memory-backed KTX object
+    auto ktxBuffer = ktx::KTX::createBare(header, keyValues);
+
+    return unserialize(ktxfile, ktxBuffer->toDescriptor());
+}
+
 bool Texture::evalKTXFormat(const Element& mipFormat, const Element& texelFormat, ktx::Header& header) {
     if (texelFormat == Format::COLOR_RGBA_32 && mipFormat == Format::COLOR_BGRA_32) {
         header.setUncompressed(ktx::GLType::UNSIGNED_BYTE, 1, ktx::GLFormat::BGRA, ktx::GLInternalFormat_Uncompressed::RGBA8, ktx::GLBaseInternalFormat::RGBA);

libraries/ktx/src/ktx/KTX.cpp

@@ -45,7 +45,7 @@ uint32_t Header::evalPixelDepth(uint32_t level) const {
 }
 
 size_t Header::evalPixelSize() const {
-    return glTypeSize; // Really we should generate the size from the FOrmat etc
+    return 4;//glTypeSize; // Really we should generate the size from the FOrmat etc
 }
 
 size_t Header::evalRowSize(uint32_t level) const {
@@ -70,6 +70,25 @@ size_t Header::evalImageSize(uint32_t level) const {
     }
 }
 
+ImageDescriptors Header::generateImageDescriptors() const {
+    ImageDescriptors descriptors;
+
+    for (auto level = 0; level < numberOfMipmapLevels; ++level) {
+        ImageHeader header {
+            numberOfFaces == NUM_CUBEMAPFACES,
+            static_cast<uint32_t>(evalImageSize(level)),
+            0
+        };
+        ImageHeader::FaceOffsets offsets;
+        for (auto i = 0; i < numberOfFaces; ++i) {
+            offsets.push_back(0);
+        }
+        descriptors.push_back(ImageDescriptor(header, offsets));
+    }
+
+    return descriptors;
+}
+
 
 KeyValue::KeyValue(const std::string& key, uint32_t valueByteSize, const Byte* value) :
     _byteSize((uint32_t) key.size() + 1 + valueByteSize), // keyString size + '\0' ending char + the value size
@@ -209,4 +228,4 @@ KTXDescriptor KTX::toDescriptor() const {
 
 KTX::KTX(const StoragePointer& storage, const Header& header, const KeyValues& keyValues, const Images& images)
     : _header(header), _storage(storage), _keyValues(keyValues), _images(images) {
-}
+}

libraries/ktx/src/ktx/KTX.h

@@ -292,6 +292,9 @@ namespace ktx {
     using Storage = storage::Storage;
     using StoragePointer = std::shared_ptr<Storage>;
 
+    struct ImageDescriptor;
+    using ImageDescriptors = std::vector<ImageDescriptor>;
+
     // Header
     struct Header {
         static const size_t IDENTIFIER_LENGTH = 12;
@@ -378,6 +381,7 @@ namespace ktx {
         void setCube(uint32_t width, uint32_t height) { setDimensions(width, height, 0, 0, NUM_CUBEMAPFACES); }
         void setCubeArray(uint32_t width, uint32_t height, uint32_t numSlices) { setDimensions(width, height, 0, (numSlices > 0 ? numSlices : 1), NUM_CUBEMAPFACES); }
 
+        ImageDescriptors generateImageDescriptors() const;
     };
     static const size_t KTX_HEADER_SIZE = 64;
     static_assert(sizeof(Header) == KTX_HEADER_SIZE, "KTX Header size is static");
@@ -421,14 +425,14 @@ namespace ktx {
 
     struct Image;
 
+    // Image without the image data itself
     struct ImageDescriptor : public ImageHeader {
         const FaceOffsets _faceOffsets;
         ImageDescriptor(const ImageHeader& header, const FaceOffsets& offsets) : ImageHeader(header), _faceOffsets(offsets) {}
         Image toImage(const ktx::StoragePointer& storage) const;
     };
 
-    using ImageDescriptors = std::vector<ImageDescriptor>;
-
+    // Image with the image data itself
     struct Image : public ImageHeader {
         FaceBytes _faceBytes;
         Image(const ImageHeader& header, const FaceBytes& faces) : ImageHeader(header), _faceBytes(faces) {}
@@ -473,6 +477,7 @@ namespace ktx {
         // This path allocate the Storage where to store header, keyvalues and copy mips
         // Then COPY all the data
         static std::unique_ptr<KTX> create(const Header& header, const Images& images, const KeyValues& keyValues = KeyValues());
+        static std::unique_ptr<KTX> createBare(const Header& header, const KeyValues& keyValues = KeyValues());
 
         // Instead of creating a full Copy of the src data in a KTX object, the write serialization can be performed with the
         // following two functions
@@ -486,7 +491,9 @@ namespace ktx {
         //
         // This is exactly what is done in the create function
         static size_t evalStorageSize(const Header& header, const Images& images, const KeyValues& keyValues = KeyValues());
+        static size_t evalStorageSize(const Header& header, const ImageDescriptors& images, const KeyValues& keyValues = KeyValues());
         static size_t write(Byte* destBytes, size_t destByteSize, const Header& header, const Images& images, const KeyValues& keyValues = KeyValues());
+        static size_t writeWithoutImages(Byte* destBytes, size_t destByteSize, const Header& header, const ImageDescriptors& descriptors, const KeyValues& keyValues = KeyValues());
         static size_t writeKeyValues(Byte* destBytes, size_t destByteSize, const KeyValues& keyValues);
         static Images writeImages(Byte* destBytes, size_t destByteSize, const Images& images);
 

libraries/ktx/src/ktx/Writer.cpp

@@ -40,6 +40,20 @@ namespace ktx {
         return create(storagePointer);
     }
 
+    std::unique_ptr<KTX> KTX::createBare(const Header& header, const KeyValues& keyValues) {
+        auto descriptors = header.generateImageDescriptors();
+
+        StoragePointer storagePointer;
+        {
+            auto storageSize = ktx::KTX::evalStorageSize(header, descriptors, keyValues);
+            auto memoryStorage = new storage::MemoryStorage(storageSize);
+            qDebug() << "Memory storage size is: " << storageSize;
+            ktx::KTX::writeWithoutImages(memoryStorage->data(), memoryStorage->size(), header, descriptors, keyValues);
+            storagePointer.reset(memoryStorage);
+        }
+        return create(storagePointer);
+    }
+
     size_t KTX::evalStorageSize(const Header& header, const Images& images, const KeyValues& keyValues) {
         size_t storageSize = sizeof(Header);
 
@@ -59,6 +73,25 @@ namespace ktx {
         return storageSize;
     }
 
+    size_t KTX::evalStorageSize(const Header& header, const ImageDescriptors& imageDescriptors, const KeyValues& keyValues) {
+        size_t storageSize = sizeof(Header);
+
+        if (!keyValues.empty()) {
+            size_t keyValuesSize = KeyValue::serializedKeyValuesByteSize(keyValues);
+            storageSize += keyValuesSize;
+        }
+
+        auto numMips = header.getNumberOfLevels();
+        for (uint32_t l = 0; l < numMips; l++) {
+            if (imageDescriptors.size() > l) {
+                storageSize += sizeof(uint32_t);
+                storageSize += imageDescriptors[l]._imageSize;
+                storageSize += Header::evalPadding(imageDescriptors[l]._imageSize);
+            }
+        }
+        return storageSize;
+    }
+
     size_t KTX::write(Byte* destBytes, size_t destByteSize, const Header& header, const Images& srcImages, const KeyValues& keyValues) {
         // Check again that we have enough destination capacity
         if (!destBytes || (destByteSize < evalStorageSize(header, srcImages, keyValues))) {
@@ -87,6 +120,35 @@ namespace ktx {
         return destByteSize;
     }
 
+    size_t KTX::writeWithoutImages(Byte* destBytes, size_t destByteSize, const Header& header, const ImageDescriptors& descriptors, const KeyValues& keyValues) {
+        // Check again that we have enough destination capacity
+        if (!destBytes || (destByteSize < evalStorageSize(header, descriptors, keyValues))) {
+            return 0;
+        }
+
+        auto currentDestPtr = destBytes;
+        // Header
+        auto destHeader = reinterpret_cast<Header*>(currentDestPtr);
+        memcpy(currentDestPtr, &header, sizeof(Header));
+        currentDestPtr += sizeof(Header);
+
+        // KeyValues
+        if (!keyValues.empty()) {
+            destHeader->bytesOfKeyValueData = (uint32_t) writeKeyValues(currentDestPtr, destByteSize - sizeof(Header), keyValues);
+        } else {
+            // Make sure the header contains the right bytesOfKeyValueData size
+            destHeader->bytesOfKeyValueData = 0;
+        }
+        currentDestPtr += destHeader->bytesOfKeyValueData;
+
+        for (int i = 0; i < descriptors.size(); ++i) {
+            *currentDestPtr = descriptors[i]._imageSize;
+            currentDestPtr += descriptors[i]._imageSize + sizeof(uint32_t);
+        }
+
+        return destByteSize;
+    }
+
     uint32_t KeyValue::writeSerializedKeyAndValue(Byte* destBytes, uint32_t destByteSize, const KeyValue& keyval) {
         uint32_t keyvalSize = keyval.serializedByteSize();
         if (keyvalSize > destByteSize) {

libraries/model-networking/src/model-networking/TextureCache.cpp

@@ -335,6 +335,77 @@ void NetworkTexture::downloadFinished(const QByteArray& data) {
 }
 
 void NetworkTexture::loadContent(const QByteArray& content) {
+    if (_sourceIsKTX) {
+        if (_ktxLoadState == LOADING_HEADER) {
+            // TODO Handle case where we already have the source hash texture on disk
+            // TODO Handle case where data isn't as large as the ktx header
+            _ktxLoadState = LOADING_LOWEST_SIX;
+            auto header = reinterpret_cast<const ktx::Header*>(content.data());
+            qDebug() << "Identifier:" << QString(QByteArray((char*)header->identifier, 12));
+            qDebug() << "Type:" << header->glType;
+            qDebug() << "TypeSize:" << header->glTypeSize;
+            qDebug() << "numberOfArrayElements:" << header->numberOfArrayElements;
+            qDebug() << "numberOfFaces:" << header->numberOfFaces;
+            qDebug() << "numberOfMipmapLevels:" << header->numberOfMipmapLevels;
+            auto kvSize = header->bytesOfKeyValueData;
+            if (kvSize > content.size() - ktx::KTX_HEADER_SIZE) {
+                qWarning() << "Cannot load " << _url << ", did not receive all kv data with initial request";
+                return;
+            }
+
+            auto keyValues = ktx::KTX::parseKeyValues(header->bytesOfKeyValueData, reinterpret_cast<const ktx::Byte*>(content.data()) + ktx::KTX_HEADER_SIZE);
+
+            // Create bare ktx in memory
+            std::string filename = "test";
+            auto memKtx = ktx::KTX::createBare(*header, keyValues);
+
+
+            auto textureCache = DependencyManager::get<TextureCache>();
+
+            // Move ktx to file
+            const char* data = reinterpret_cast<const char*>(memKtx->_storage->data());
+            size_t length = memKtx->_storage->size();
+            KTXFilePointer file;
+            auto& ktxCache = textureCache->_ktxCache;
+            if (!memKtx || !(file = ktxCache.writeFile(data, KTXCache::Metadata(filename, length)))) {
+                qCWarning(modelnetworking) << _url << "file cache failed";
+            } else {
+                _file = file;
+            }
+
+            //auto texture = gpu::Texture::serializeHeader("test.ktx", *header, keyValues);
+            gpu::TexturePointer texture;
+            texture.reset(gpu::Texture::unserialize(_file->getFilepath(), memKtx->toDescriptor()));
+            texture->setKtxBacking(file->getFilepath());
+
+            // We replace the texture with the one stored in the cache.  This deals with the possible race condition of two different 
+            // images with the same hash being loaded concurrently.  Only one of them will make it into the cache by hash first and will
+            // be the winner
+            if (textureCache) {
+                texture = textureCache->cacheTextureByHash(filename, texture);
+            }
+
+            setImage(texture, header->getPixelWidth(), header->getPixelHeight());
+
+
+            auto desc = memKtx->toDescriptor();
+            int numMips = desc.images.size();
+            auto numMipsToGet = glm::min(numMips, 6);
+            auto sizeOfTopMips = 0;
+            for (int i = 0; i < numMipsToGet; ++i) {
+                auto& img = desc.images[i];
+                sizeOfTopMips += img._imageSize;
+            }
+            _requestByteRange.fromInclusive = length - sizeOfTopMips;
+            _requestByteRange.toExclusive = length;
+            attemptRequest();
+
+        } else {
+            qDebug() << "Got highest 6 mips";
+        }
+        return;
+    }
+
     QThreadPool::globalInstance()->start(new ImageReader(_self, _url, content, _maxNumPixels));
 }
 
@@ -457,6 +528,7 @@ void ImageReader::read() {
     if (texture && textureCache) {
         auto memKtx = gpu::Texture::serialize(*texture);
 
+        // Move the texture into a memory mapped file
         if (memKtx) {
             const char* data = reinterpret_cast<const char*>(memKtx->_storage->data());
             size_t length = memKtx->_storage->size();

libraries/model-networking/src/model-networking/TextureCache.h

@@ -72,6 +72,14 @@ private:
     friend class ImageReader;
 
     image::TextureUsage::Type _type;
+
+    enum KTXLoadState {
+        LOADING_HEADER,
+        LOADING_LOWEST_SIX,
+        DONE_LOADING 
+    };
+
+    KTXLoadState _ktxLoadState { LOADING_HEADER };
     KTXFilePointer _file;
     bool _sourceIsKTX { false };
     int _originalWidth { 0 };

libraries/networking/src/HTTPResourceRequest.cpp

@@ -60,7 +60,7 @@ void HTTPResourceRequest::doSend() {
     }
 
     if (_byteRange.isSet()) {
-        auto byteRange = QString("bytes={}-{}").arg(_byteRange.fromInclusive).arg(_byteRange.toExclusive);
+        auto byteRange = QString("bytes=%1-%2").arg(_byteRange.fromInclusive).arg(_byteRange.toExclusive);
         networkRequest.setRawHeader("Range", byteRange.toLatin1());
     }
     networkRequest.setAttribute(QNetworkRequest::HttpPipeliningAllowedAttribute, true);
@@ -78,12 +78,61 @@ void HTTPResourceRequest::onRequestFinished() {
     Q_ASSERT(_reply);
 
     cleanupTimer();
-    
+
+    // Content-Range headers have the form: 
+    //
+    //   Content-Range: <unit> <range-start>-<range-end>/<size>
+    //   Content-Range: <unit> <range-start>-<range-end>/*
+    //   Content-Range: <unit> */<size>
+    //
+    auto parseContentRangeHeader = [](QString contentRangeHeader) -> std::pair<bool, uint64_t> {
+        auto unitRangeParts = contentRangeHeader.split(' ');
+        if (unitRangeParts.size() != 2) {
+            return { false, 0 };
+        }
+
+        auto rangeSizeParts = unitRangeParts[1].split('/');
+        if (rangeSizeParts.size() != 2) {
+            return { false, 0 };
+        }
+
+        auto sizeStr = rangeSizeParts[1];
+        if (sizeStr == "*") {
+            return { true, 0 };
+        } else {
+            bool ok;
+            auto size = sizeStr.toLong(&ok);
+            return { ok, size };
+        }
+    };
+
     switch(_reply->error()) {
         case QNetworkReply::NoError:
             _data = _reply->readAll();
             _loadedFromCache = _reply->attribute(QNetworkRequest::SourceIsFromCacheAttribute).toBool();
             _result = Success;
+
+            if (_byteRange.isSet()) {
+                auto statusCode = _reply->attribute(QNetworkRequest::HttpStatusCodeAttribute).toInt();
+                if (statusCode == 206) {
+                    _rangeRequestSuccessful = true;
+                    auto contentRangeHeader = _reply->rawHeader("Content-Range");
+                    bool success;
+                    uint64_t size;
+                    std::tie(success, size) = parseContentRangeHeader(contentRangeHeader);
+                    if (success) {
+                        qWarning(networking) << "Total http resource size is: " << size;
+                        _totalSizeOfResource = size;
+                    } else {
+                        qWarning(networking) << "Error parsing content-range header: " << contentRangeHeader;
+                        _totalSizeOfResource = 0;
+                    }
+                } else {
+                    _rangeRequestSuccessful = false;
+                    _totalSizeOfResource = _data.size();
+                }
+            }
+
             break;
 
         case QNetworkReply::TimeoutError:

libraries/networking/src/NetworkAccessManager.cpp

@@ -13,12 +13,20 @@
 
 #include "AtpReply.h"
 #include "NetworkAccessManager.h"
+#include <QtNetwork/QNetworkProxy>
 
 QThreadStorage<QNetworkAccessManager*> networkAccessManagers;
 
 QNetworkAccessManager& NetworkAccessManager::getInstance() {
     if (!networkAccessManagers.hasLocalData()) {
-        networkAccessManagers.setLocalData(new QNetworkAccessManager());
+        auto nm = new QNetworkAccessManager();
+        networkAccessManagers.setLocalData(nm);
+
+        QNetworkProxy proxy;
+        proxy.setType(QNetworkProxy::HttpProxy);
+        proxy.setHostName("127.0.0.1");
+        proxy.setPort(8888);
+        nm->setProxy(proxy);
     }
     
     return *networkAccessManagers.localData();

libraries/networking/src/ResourceRequest.h

@@ -53,6 +53,8 @@ public:
     QString getResultString() const;
     QUrl getUrl() const { return _url; }
     bool loadedFromCache() const { return _loadedFromCache; }
+    bool getRangeRequestSuccessful() const { return _rangeRequestSuccessful; }
+    bool getTotalSizeOfResource() const { return _totalSizeOfResource; }
 
     void setCacheEnabled(bool value) { _cacheEnabled = value; }
     void setByteRange(ByteRange byteRange) { _byteRange = byteRange; }
@@ -74,6 +76,8 @@ protected:
     bool _cacheEnabled { true };
     bool _loadedFromCache { false };
     ByteRange _byteRange;
+    bool _rangeRequestSuccessful { false };
+    uint64_t _totalSizeOfResource { 0 };
 };
 
 #endif

libraries/shared/src/shared/Storage.cpp

@@ -67,6 +67,7 @@ StoragePointer FileStorage::create(const QString& filename, size_t size, const u
     return std::make_shared<FileStorage>(filename);
 }
 
+// Represents a memory mapped file
 FileStorage::FileStorage(const QString& filename) : _file(filename) {
     if (_file.open(QFile::ReadOnly)) {
         _mapped = _file.map(0, _file.size());

libraries/shared/src/shared/Storage.h

@@ -20,6 +20,7 @@ namespace storage {
     class Storage;
     using StoragePointer = std::shared_ptr<const Storage>;
 
+    // Abstract class to represent memory that stored _somewhere_ (in system memory or in a file, for example)
     class Storage : public std::enable_shared_from_this<Storage> {
     public:
         virtual ~Storage() {}