Maybe saving the first ktx textures, testing the save pipeline

interface/CMakeLists.txt

@@ -178,7 +178,7 @@ endif()
 
 # link required hifi libraries
 link_hifi_libraries(
-  shared octree gpu gl gpu-gl procedural model render
+  shared octree ktx gpu gl gpu-gl procedural model render
   recording fbx networking model-networking entities avatars
   audio audio-client animation script-engine physics
   render-utils entities-renderer ui auto-updater

libraries/gpu/src/gpu/Texture.cpp

@@ -1001,3 +1001,28 @@ Texture::ExternalUpdates Texture::getUpdates() const {
     }
     return result;
 }
+
+#include <ktx/KTX.h>
+
+ktx::KTXUniquePointer Texture::serialize(const Texture& texture) {
+    
+    ktx::Header header;
+    header.setUncompressed(ktx::GLType::UNSIGNED_BYTE, 4, ktx::GLFormat::BGRA, ktx::GLInternalFormat_Uncompressed::RGBA8, ktx::GLBaseInternalFormat::RGBA);
+    header.pixelWidth = texture.getWidth();
+    header.pixelHeight = texture.getHeight();
+    header.numberOfMipmapLevels = texture.mipLevels();
+
+    ktx::Images images;
+    for (int level = 0; level < header.numberOfMipmapLevels; level++) {
+        auto mip = texture.accessStoredMipFace(level);
+        if (mip) {
+            images.emplace_back(ktx::Image(mip->getSize(), 0, mip->readData()));
+        }
+    }
+
+    auto ktxBuffer = ktx::KTX::create(header, ktx::KeyValues(), images);
+    return ktxBuffer;
+}
+TexturePointer Texture::unserialize(const ktx::KTXUniquePointer& srcData) {
+    return nullptr;
+}

libraries/gpu/src/gpu/Texture.h

@@ -20,6 +20,11 @@
 #include "Forward.h"
 #include "Resource.h"
 
+namespace ktx {
+    class KTX;
+    using KTXUniquePointer = std::unique_ptr<KTX>;
+}
+
 namespace gpu {
 
 // THe spherical harmonics is a nice tool for cubemap, so if required, the irradiance SH can be automatically generated
@@ -475,6 +480,9 @@ public:
 
     ExternalUpdates getUpdates() const;
 
+    static ktx::KTXUniquePointer serialize(const Texture& texture);
+    static TexturePointer unserialize(const ktx::KTXUniquePointer& srcData);
+
 protected:
     // Should only be accessed internally or by the backend sync function
     mutable Mutex _externalMutex;

libraries/ktx/src/ktx/KTX.cpp

@@ -15,11 +15,12 @@
 
 using namespace ktx;
 
-uint32_t evalPadding(size_t byteSize) {
+uint32_t Header::evalPadding(size_t byteSize) {
     auto padding = byteSize % PACKING_SIZE;
-    return (padding ? PACKING_SIZE - padding : 0);
+    return (uint32_t) (padding ? PACKING_SIZE - padding : 0);
 }
 
+
 const Header::Identifier ktx::Header::IDENTIFIER {{
     0xAB, 0x4B, 0x54, 0x58, 0x20, 0x31, 0x31, 0xBB, 0x0D, 0x0A, 0x1A, 0x0A
 }};
@@ -29,7 +30,7 @@ Header::Header() {
 }
 
 uint32_t Header::evalMaxDimension() const {
-    return std::max(pixelWidth, std::max(pixelHeight, pixelDepth));
+    return std::max(getPixelWidth(), std::max(getPixelHeight(), getPixelDepth()));
 }
 
 uint32_t Header::evalMaxLevel() const {
@@ -37,13 +38,13 @@ uint32_t Header::evalMaxLevel() const {
 }
 
 uint32_t Header::evalPixelWidth(uint32_t level) const {
-    return std::max(pixelWidth >> level, 1U);
+    return std::max(getPixelWidth() >> level, 1U);
 }
 uint32_t Header::evalPixelHeight(uint32_t level) const {
-    return std::max(pixelHeight >> level, 1U);
+    return std::max(getPixelHeight() >> level, 1U);
 }
 uint32_t Header::evalPixelDepth(uint32_t level) const {
-    return std::max(pixelDepth >> level, 1U);
+    return std::max(getPixelDepth() >> level, 1U);
 }
 
 size_t Header::evalPixelSize() const {
@@ -51,24 +52,24 @@ size_t Header::evalPixelSize() const {
 }
 
 size_t Header::evalRowSize(uint32_t level) const {
-    auto pixelWidth = evalPixelWidth(level);
+    auto pixWidth = evalPixelWidth(level);
     auto pixSize = evalPixelSize();
-    auto netSize = pixelWidth * pixSize;
+    auto netSize = pixWidth * pixSize;
     auto padding = evalPadding(netSize);
     return netSize + padding;
 }
 size_t Header::evalFaceSize(uint32_t level) const {
-    auto pixelHeight = evalPixelHeight(level);
+    auto pixHeight = evalPixelHeight(level);
-    auto pixelDepth = evalPixelDepth(level);
+    auto pixDepth = evalPixelDepth(level);
     auto rowSize = evalRowSize(level);
-    return pixelDepth * pixelHeight * rowSize;
+    return pixDepth * pixHeight * rowSize;
 }
 size_t Header::evalImageSize(uint32_t level) const {
     auto faceSize = evalFaceSize(level);
     if (numberOfFaces == 6 && numberOfArrayElements == 0) {
         return faceSize;
     } else {
-        return (numberOfArrayElements * numberOfFaces * faceSize);
+        return (getNumberOfSlices() * numberOfFaces * faceSize);
     }
 }
 
@@ -76,6 +77,9 @@ size_t Header::evalImageSize(uint32_t level) const {
 KTX::KTX() {
 }
 
+KTX::~KTX() {
+}
+
 void KTX::resetStorage(Storage* storage) {
     _storage.reset(storage);
 }
@@ -99,7 +103,8 @@ size_t KTX::getKeyValueDataSize() const {
 
 size_t KTX::getTexelsDataSize() const {
     if (_storage) {
-        return  _storage->size() - sizeof(Header) + getKeyValueDataSize();
+        //return  _storage->size() - (sizeof(Header) + getKeyValueDataSize());
+        return  (_storage->_bytes + _storage->_size) - getTexelsData();
     } else {
         return 0;
     }
@@ -121,3 +126,10 @@ const Byte* KTX::getTexelsData() const {
     }
 }
 
+Byte* KTX::getTexelsData() {
+    if (_storage) {
+        return (_storage->_bytes + sizeof(Header) + getKeyValueDataSize());
+    } else {
+        return nullptr;
+    }
+}

libraries/ktx/src/ktx/KTX.h

@@ -68,9 +68,8 @@ end
 namespace ktx {
     const uint32_t PACKING_SIZE { sizeof(uint32_t) };
     using Byte = uint8_t;
-    uint32_t evalPadding(size_t byteSize);
 
-    enum GLType : uint32_t {
+    enum class GLType : uint32_t {
         COMPRESSED_TYPE                 = 0,
 
         // GL 4.4 Table 8.2
@@ -102,7 +101,7 @@ namespace ktx {
         NUM_GLTYPES = 25,
     };
 
-    enum GLFormat : uint32_t {
+    enum class GLFormat : uint32_t {
         COMPRESSED_FORMAT               = 0,
 
         // GL 4.4 Table 8.3
@@ -131,7 +130,7 @@ namespace ktx {
         NUM_GLFORMATS = 20,
     };
 
-    enum GLInternalFormat_Uncompressed : uint32_t {
+    enum class GLInternalFormat_Uncompressed : uint32_t {
         // GL 4.4 Table 8.12
         R8                              = 0x8229,
         R8_SNORM                        = 0x8F94,
@@ -233,7 +232,7 @@ namespace ktx {
         NUM_UNCOMPRESSED_GLINTERNALFORMATS = 74,
     };
 
-    enum GLInternalFormat_Compressed : uint32_t {
+    enum class GLInternalFormat_Compressed : uint32_t {
         // GL 4.4 Table 8.14
         COMPRESSED_RED = 0x8225,
         COMPRESSED_RG = 0x8226,
@@ -268,15 +267,15 @@ namespace ktx {
          NUM_COMPRESSED_GLINTERNALFORMATS = 24,
     };
  
-    enum GLBaseInternalFormat : uint32_t {
+    enum class GLBaseInternalFormat : uint32_t {
         // GL 4.4 Table 8.11
-        BIF_DEPTH_COMPONENT = 0x1902,
+        DEPTH_COMPONENT = 0x1902,
-        BIF_DEPTH_STENCIL = 0x84F9,
+        DEPTH_STENCIL = 0x84F9,
-        BIF_RED = 0x1903,
+        RED = 0x1903,
-        BIF_RG = 0x8227,
+        RG = 0x8227,
-        BIF_RGB = 0x1907,
+        RGB = 0x1907,
-        BIF_RGBA = 0x1908,
+        RGBA = 0x1908,
-        BIF_STENCIL_INDEX = 0x1901,
+        STENCIL_INDEX = 0x1901,
 
         NUM_GLBASEINTERNALFORMATS = 7,
     };
@@ -341,22 +340,33 @@ namespace ktx {
         static const uint32_t ENDIAN_TEST = 0x04030201;
         static const uint32_t REVERSE_ENDIAN_TEST = 0x01020304;
 
+        static uint32_t evalPadding(size_t byteSize);
+
         Header();
 
         Byte identifier[IDENTIFIER_LENGTH];
         uint32_t endianness { ENDIAN_TEST };
+
         uint32_t glType;
-        uint32_t glTypeSize;
+        uint32_t glTypeSize { 0 };
         uint32_t glFormat;
         uint32_t glInternalFormat;
         uint32_t glBaseInternalFormat;
-        uint32_t pixelWidth;
+
-        uint32_t pixelHeight;
+        uint32_t pixelWidth { 0 };
-        uint32_t pixelDepth;
+        uint32_t pixelHeight { 0 };
-        uint32_t numberOfArrayElements;
+        uint32_t pixelDepth { 0 };
-        uint32_t numberOfFaces;
+        uint32_t numberOfArrayElements { 0 };
-        uint32_t numberOfMipmapLevels;
+        uint32_t numberOfFaces { 1 };
-        uint32_t bytesOfKeyValueData;
+        uint32_t numberOfMipmapLevels { 1 };
+
+        uint32_t bytesOfKeyValueData { 0 };
+
+        uint32_t getPixelWidth() const { return (pixelWidth ? pixelWidth : 1); }
+        uint32_t getPixelHeight() const { return (pixelHeight ? pixelHeight : 1); }
+        uint32_t getPixelDepth() const { return (pixelDepth ? pixelDepth : 1); }
+        uint32_t getNumberOfSlices() const { return (numberOfArrayElements ? numberOfArrayElements : 1); }
+        uint32_t getNumberOfLevels() const { return (numberOfMipmapLevels ? numberOfMipmapLevels : 1); }
 
         uint32_t evalMaxDimension() const;
         uint32_t evalMaxLevel() const;
@@ -369,6 +379,20 @@ namespace ktx {
         size_t evalFaceSize(uint32_t level) const;
         size_t evalImageSize(uint32_t level) const;
 
+        void setUncompressed(GLType type, uint32_t typeSize, GLFormat format, GLInternalFormat_Uncompressed internalFormat, GLBaseInternalFormat baseInternalFormat) {
+            glType = (uint32_t) type;
+            glTypeSize = 0;
+            glFormat = (uint32_t) format;
+            glInternalFormat = (uint32_t) internalFormat;
+            glBaseInternalFormat = (uint32_t) baseInternalFormat;
+        }
+        void setCompressed(GLInternalFormat_Compressed internalFormat, GLBaseInternalFormat baseInternalFormat) {
+            glType = (uint32_t) GLType::COMPRESSED_TYPE;
+            glTypeSize = 1;
+            glFormat = (uint32_t) GLFormat::COMPRESSED_FORMAT;
+            glInternalFormat = (uint32_t) internalFormat;
+            glBaseInternalFormat = (uint32_t) baseInternalFormat;
+        }
     };
 
     // Key Values
@@ -392,6 +416,9 @@ namespace ktx {
     class KTX {
         void resetStorage(Storage* src);
 
+        void resetHeader(const Header& header);
+        void resetImages(const Images& images);
+
         KTX();
     public:
 
@@ -412,11 +439,11 @@ namespace ktx {
         const Header* getHeader() const;
         const Byte* getKeyValueData() const;
         const Byte* getTexelsData() const;
+        Byte* getTexelsData();
 
         size_t getKeyValueDataSize() const;
         size_t getTexelsDataSize() const;
 
-
         std::unique_ptr<Storage> _storage;
         KeyValues _keyValues;
         Images _images;

libraries/ktx/src/ktx/Reader.cpp

@@ -84,18 +84,18 @@ namespace ktx {
     Images getImagesTable(const Header& header, size_t mipsDataSize, const Byte* mipsData) {
         Images images;
         auto currentPtr = mipsData;
-        auto numMips = header.numberOfMipmapLevels + 1;
+        auto numMips = header.getNumberOfLevels();
 
         // Keep identifying new mip as long as we can at list query the next imageSize
         while ((currentPtr - mipsData) + sizeof(uint32_t) <= (mipsDataSize)) {
 
             // Grab the imageSize coming up
-            auto imageSize = *reinterpret_cast<const uint32_t*>(currentPtr);
+            size_t imageSize = *reinterpret_cast<const uint32_t*>(currentPtr);
             currentPtr += sizeof(uint32_t);
 
             // If enough data ahead then capture the pointer
             if ((currentPtr - mipsData) + imageSize <= (mipsDataSize)) {
-                auto padding = evalPadding(imageSize);
+                auto padding = Header::evalPadding(imageSize);
 
                 images.emplace_back(Image(imageSize, padding, currentPtr));
 

libraries/ktx/src/ktx/Writer.cpp

@@ -25,17 +25,60 @@ namespace ktx {
 
     std::unique_ptr<Storage> generateStorage(const Header& header, const KeyValues& keyValues, const Images& images) {
         size_t storageSize = sizeof(Header);
-        auto numMips = header.numberOfMipmapLevels + 1;
+        auto numMips = header.getNumberOfLevels();
 
         for (uint32_t l = 0; l < numMips; l++) {
             if (images.size() > l) {
-
+                storageSize += sizeof(uint32_t);
-                storageSize += images[l]._imageSize;
                 storageSize += images[l]._imageSize;
+                storageSize += Header::evalPadding(images[l]._imageSize);
             }
-           
         }
 
+        std::unique_ptr<Storage> storage(new Storage(storageSize));
+        return storage;
+    }
+
+
+    void KTX::resetHeader(const Header& header) {
+         if (!_storage) {
+            return;
+        }
+        memcpy(_storage->_bytes, &header, sizeof(Header));
+    }
+    void KTX::resetImages(const Images& srcImages) {
+        auto imagesDataPtr = getTexelsData();
+        if (!imagesDataPtr) {
+            return;
+        }
+        auto allocatedImagesDataSize = getTexelsDataSize();
+        size_t currentDataSize = 0;
+        auto currentPtr = imagesDataPtr;
+
+        _images.clear();
+
+
+        for (uint32_t l = 0; l < srcImages.size(); l++) {
+            if (currentDataSize + sizeof(uint32_t) < allocatedImagesDataSize) {
+                size_t imageSize = srcImages[l]._imageSize;
+                *(reinterpret_cast<uint32_t*> (currentPtr)) = imageSize;
+                currentPtr += sizeof(uint32_t);
+                currentDataSize += sizeof(uint32_t);
+
+                // If enough data ahead then capture the copy source pointer
+                if (currentDataSize + imageSize <= (allocatedImagesDataSize)) {
+
+                    auto copied = memcpy(currentPtr, srcImages[l]._bytes, imageSize);
+
+                    auto padding = Header::evalPadding(imageSize);
+
+                    _images.emplace_back(Image(imageSize, padding, currentPtr));
+
+                    currentPtr += imageSize + padding;
+                    currentDataSize += imageSize + padding;
+                }
+            }
+        }
     }
 
     std::unique_ptr<KTX> KTX::create(const Header& header, const KeyValues& keyValues, const Images& images) {
@@ -43,11 +86,13 @@ namespace ktx {
         std::unique_ptr<KTX> result(new KTX());
         result->resetStorage(generateStorage(header, keyValues, images).release());
 
+        result->resetHeader(header);
+
         // read metadata
         result->_keyValues = keyValues;
 
         // populate image table
-        result->_images = images;
+        result->resetImages(images);
 
         return result;
     }

libraries/model/CMakeLists.txt

@@ -1,5 +1,5 @@
 set(TARGET_NAME model)
 AUTOSCRIBE_SHADER_LIB(gpu model)
 setup_hifi_library()
-link_hifi_libraries(shared gpu)
+link_hifi_libraries(shared ktx gpu)
 

libraries/model/src/model/TextureMap.cpp

@@ -10,6 +10,8 @@
 //
 #include "TextureMap.h"
 
+#include <ktx/KTX.h>
+
 #include <QImage>
 #include <QPainter>
 #include <QDebug>
@@ -264,8 +266,23 @@ gpu::Texture* TextureUsage::process2DTextureColorFromImage(const QImage& srcImag
         if (generateMips) {
             ::generateMips(theTexture, image, formatMip, false);
         }
+        auto theKTX = Texture::serialize(*theTexture);
+        if (theKTX) {
+            // save that!
+            std::string filename("C://temp//ktxCache//texmex");
+                filename += std::to_string((size_t) theTexture);
+                filename += ".ktx";
+
+            FILE* file = fopen (filename.c_str(),"wb");
+            if (file != nullptr) {
+                fwrite(theKTX->_storage->data(), 1, theKTX->_storage->size(), file);
+                fclose (file);
+            }
+
+        }
     }
 
+
     return theTexture;
 }
 

libraries/render-utils/CMakeLists.txt

@@ -3,7 +3,7 @@ AUTOSCRIBE_SHADER_LIB(gpu model render)
 # pull in the resources.qrc file
 qt5_add_resources(QT_RESOURCES_FILE "${CMAKE_CURRENT_SOURCE_DIR}/res/fonts/fonts.qrc")
 setup_hifi_library(Widgets OpenGL Network Qml Quick Script)
-link_hifi_libraries(shared gpu model model-networking render animation fbx entities)
+link_hifi_libraries(shared ktx gpu model model-networking render animation fbx entities)
 
 if (NOT ANDROID)
   target_nsight()

libraries/render/CMakeLists.txt

@@ -3,6 +3,6 @@ AUTOSCRIBE_SHADER_LIB(gpu model)
 setup_hifi_library()
 
 # render needs octree only for getAccuracyAngle(float, int)
-link_hifi_libraries(shared gpu model octree)
+link_hifi_libraries(shared ktx gpu model octree)
 
 target_nsight()