Fix KTX issues with the serializer.

* Stop trying to be compatible with the old format, and just bump the version number.
* Add uint64_t support to serializer
* A bit improved debug output from serializer
* Add lastAdvance() function to ask the serializer how much data was added/read in the last operation.

libraries/gpu/src/gpu/Texture_ktx.cpp

@@ -32,11 +32,8 @@ struct GPUKTXPayload {
     using Version = uint8;
 
     static const std::string KEY;
-    static const Version CURRENT_VERSION { 2 };
+    static const Version CURRENT_VERSION { 3 };
-    static const size_t PADDING { 2 };
+    static const size_t SIZE { sizeof(Version) + sizeof(Sampler::Desc) + sizeof(uint64_t) + sizeof(TextureUsageType) + sizeof(glm::ivec2) };
-    static const size_t SIZE { sizeof(Version) + sizeof(Sampler::Desc) + sizeof(uint32) + sizeof(TextureUsageType) + sizeof(glm::ivec2) + PADDING };
-    static_assert(GPUKTXPayload::SIZE == 44, "Packing size may differ between platforms");
-    static_assert(GPUKTXPayload::SIZE % 4 == 0, "GPUKTXPayload is not 4 bytes aligned");
 
     Sampler::Desc _samplerDesc;
     Texture::Usage _usage;
@@ -44,38 +41,43 @@ struct GPUKTXPayload {
     glm::ivec2 _originalSize { 0, 0 };
 
     void serialize(DataSerializer &ser) {
+
         ser << CURRENT_VERSION;
+
+
         ser << _samplerDesc;
 
-        qCWarning(gpulogging) << "Offsets: " << offsetof(struct GPUKTXPayload, _samplerDesc) << offsetof(struct GPUKTXPayload, _usage) << offsetof(struct GPUKTXPayload, _usageType) << offsetof(struct GPUKTXPayload, _originalSize);
+        uint64_t usageData = _usage._flags.to_ulong();
-        uint32 usageData = _usage._flags.to_ulong();
         ser << usageData;
-
+        ser << ((uint8_t)_usageType);
-        ser << (char)_usageType;
         ser << _originalSize;
-        ser.addPadding(PADDING);
 
+
+        // The +1 is here because we're adding the CURRENT_VERSION at the top, but since it's declared as a static
+        // const, it's not actually part of the class' size.
         assert(ser.length() == GPUKTXPayload::SIZE);
     }
 
     bool unserialize(DataDeserializer &dsr) {
         Version version = 0;
-        uint32 usageData;
+        uint64_t usageData = 0;
         uint8_t usagetype = 0;
 
         dsr >> version;
 
-        if (version > CURRENT_VERSION) {
+        if (version != CURRENT_VERSION) {
             // If we try to load a version that we don't know how to parse,
             // it will render incorrectly
-            qCWarning(gpulogging) << "KTX version" << version << "is newer than our own," << CURRENT_VERSION;
+            qCWarning(gpulogging) << "KTX version" << version << "is different than our own," << CURRENT_VERSION;
-            qCWarning(gpulogging) << "Offsets: " << offsetof(struct GPUKTXPayload, _samplerDesc) << offsetof(struct GPUKTXPayload, _usage) << offsetof(struct GPUKTXPayload, _usageType) << offsetof(struct GPUKTXPayload, _originalSize);
             qCWarning(gpulogging) << dsr;
             return false;
         }
 
         dsr >> _samplerDesc;
+
         dsr >> usageData;
+        _usage._flags = gpu::Texture::Usage::Flags(usageData);
+
         dsr >> usagetype;
         _usageType = (TextureUsageType)usagetype;
 

libraries/shared/src/SerDes.h

@@ -238,6 +238,33 @@ class DataSerializer {
             return *this;
         }
 
+        ///////////////////////////////////////////////////////////
+
+        /**
+         * @brief Add an uint64_t to the output
+         *
+         * @param val  Value to add
+         * @return SerDes& This object
+         */
+        DataSerializer &operator<<(uint64_t val) {
+            return *this << int64_t(val);
+        }
+
+        /**
+         * @brief Add an int64_t to the output
+         *
+         * @param val  Value to add
+         * @return SerDes& This object
+         */
+        DataSerializer &operator<<(int64_t val) {
+            if (!extendBy(sizeof(val), "int64_t")) {
+                return *this;
+            }
+
+            memcpy(&_store[_pos], (char*)&val, sizeof(val));
+            _pos += sizeof(val);
+            return *this;
+        }
 
         ///////////////////////////////////////////////////////////
 
@@ -410,7 +437,18 @@ class DataSerializer {
          * @brief Reset the serializer to the start, clear overflow bit.
          *
          */
-        void rewind() { _pos = 0; _overflow = false; }
+        void rewind() { _pos = 0; _overflow = false; _lastAdvance = 0; }
+
+
+        /**
+         * @brief Size of the last advance
+         *
+         * This can be used to get how many bytes were added in the last operation.
+         * It is reset on rewind()
+         *
+         * @return size_t
+         */
+        size_t lastAdvance() const { return _lastAdvance; }
 
         /**
          * @brief Dump the contents of this object into QDebug
@@ -430,7 +468,7 @@ class DataSerializer {
 
             if ( _capacity < _length + bytes) {
                 if ( _storeIsExternal ) {
-                    qCritical() << "Serializer trying to write past end of output buffer, writing" << bytes << "bytes for" << type_name << " from position " << _pos << ", length " << _length;
+                    qCritical() << "Serializer trying to write past end of output buffer of" << _capacity << "bytes. Error writing" << bytes << "bytes for" << type_name << " from position " << _pos << ", length " << _length;
                     _overflow = true;
                     return false;
                 }
@@ -439,6 +477,7 @@ class DataSerializer {
             }
 
             _length += bytes;
+            _lastAdvance = bytes;
             return true;
         }
 
@@ -451,6 +490,7 @@ class DataSerializer {
         size_t _capacity = 0;
         size_t _length = 0;
         size_t _pos = 0;
+        size_t _lastAdvance = 0;
 };
 
 /**
@@ -502,6 +542,7 @@ class DataDeserializer {
             _length = storeLength;
             _pos = 0;
             _store = externalStore;
+            _lastAdvance = 0;
         }
 
         /**
@@ -536,6 +577,7 @@ class DataDeserializer {
             }
 
             _pos += bytes;
+            _lastAdvance = bytes;
         }
 
 
@@ -558,6 +600,7 @@ class DataDeserializer {
         DataDeserializer &operator>>(int8_t &c) {
             if ( canAdvanceBy(1, "int8_t") ) {
                 c = _store[_pos++];
+                _lastAdvance = sizeof(c);
             }
 
             return *this;
@@ -585,6 +628,7 @@ class DataDeserializer {
             if ( canAdvanceBy(sizeof(val), "int16_t") ) {
                 memcpy((char*)&val, &_store[_pos], sizeof(val));
                 _pos += sizeof(val);
+                _lastAdvance = sizeof(val);
             }
 
             return *this;
@@ -612,10 +656,37 @@ class DataDeserializer {
             if ( canAdvanceBy(sizeof(val), "int32_t") ) {
                 memcpy((char*)&val, &_store[_pos], sizeof(val));
                 _pos += sizeof(val);
+                _lastAdvance = sizeof(val);
             }
             return *this;
         }
 
+        ///////////////////////////////////////////////////////////
+
+        /**
+         * @brief Read an uint64_t from the buffer
+         *
+         * @param val Value to read
+         * @return SerDes& This object
+         */
+        DataDeserializer &operator>>(uint64_t &val) {
+            return *this >> reinterpret_cast<int64_t&>(val);
+        }
+
+        /**
+         * @brief Read an int64_t from the buffer
+         *
+         * @param val Value to read
+         * @return SerDes& This object
+         */
+        DataDeserializer &operator>>(int64_t &val) {
+            if ( canAdvanceBy(sizeof(val), "int64_t") ) {
+                memcpy((char*)&val, &_store[_pos], sizeof(val));
+                _pos += sizeof(val);
+                _lastAdvance = sizeof(val);
+            }
+            return *this;
+        }
 
         ///////////////////////////////////////////////////////////
 
@@ -630,6 +701,7 @@ class DataDeserializer {
             if ( canAdvanceBy(sizeof(val), "float") ) {
                 memcpy((char*)&val, &_store[_pos], sizeof(val));
                 _pos += sizeof(val);
+                _lastAdvance = sizeof(val);
             }
             return *this;
         }
@@ -654,6 +726,7 @@ class DataDeserializer {
                 memcpy((char*)&val.z, &_store[_pos + sz*2], sz);
 
                 _pos += sz*3;
+                _lastAdvance = sz * 3;
             }
 
             return *this;
@@ -678,6 +751,7 @@ class DataDeserializer {
                 memcpy((char*)&val.w, &_store[_pos + sz*3], sz);
 
                 _pos += sz*4;
+                _lastAdvance = sz*4;
             }
             return *this;
         }
@@ -699,6 +773,7 @@ class DataDeserializer {
                 memcpy((char*)&val.y, &_store[_pos + sz  ], sz);
 
                 _pos += sz*2;
+                _lastAdvance = sz * 2;
             }
 
             return *this;
@@ -753,7 +828,17 @@ class DataDeserializer {
          * @brief Reset the serializer to the start, clear overflow bit.
          *
          */
-        void rewind() { _pos = 0; _overflow = false; }
+        void rewind() { _pos = 0; _overflow = false; _lastAdvance = 0; }
+
+        /**
+         * @brief Size of the last advance
+         *
+         * This can be used to get how many bytes were added in the last operation.
+         * It is reset on rewind()
+         *
+         * @return size_t
+         */
+        size_t lastAdvance() const { return _lastAdvance; }
 
         /**
          * @brief Dump the contents of this object into QDebug
@@ -772,7 +857,7 @@ class DataDeserializer {
             //qDebug() << "Checking advance by" << bytes;
 
             if ( _length < _pos + bytes) {
-                qCritical() << "Deserializer trying to read past end of input, reading" << bytes << "bytes for" << type_name << "from position " << _pos << ". Max length " << _length;
+                qCritical() << "Deserializer trying to read past end of input buffer of" << _length << "bytes, reading" << bytes << "bytes for" << type_name << "from position " << _pos;
                 _overflow = true;
                 return false;
             }
@@ -784,4 +869,5 @@ class DataDeserializer {
         bool _overflow = false;
         size_t _length = 0;
         size_t _pos = 0;
+        size_t _lastAdvance = 0;
 };

tests/shared/src/SerializerTests.cpp

@@ -124,6 +124,35 @@ void SerializerTests::testReadPastEnd() {
     QCOMPARE(d.pos(), 0);
 }
 
+void SerializerTests::testWritePastEnd() {
+    qint8 i8 = 255;
+    qint16 i16 = 65535;
+
+
+    char buf[16];
+
+
+    // 1 byte buffer, we can write 1 byte
+    memset(buf, 0, sizeof(buf));
+    DataSerializer s1(buf, 1);
+    s1 << i8;
+    QCOMPARE(s1.pos(), 1);
+    QCOMPARE(s1.isOverflow(), false);
+    QCOMPARE(buf[0], i8);
+
+    // 1 byte buffer, we can't write 2 bytes
+    memset(buf, 0, sizeof(buf));
+    DataSerializer s2(buf, 1);
+    s2 << i16;
+    QCOMPARE(s2.pos(), 0);
+    QCOMPARE(s2.isOverflow(), true);
+    QCOMPARE(buf[0], 0); // We didn't write
+    QCOMPARE(buf[1], 0);
+}
+
+
+
+
 void SerializerTests::benchmarkEncodingDynamicAlloc() {
     QBENCHMARK {
         DataSerializer s;

tests/shared/src/SerializerTests.h

@@ -21,6 +21,7 @@ private slots:
     void testAdd();
     void testAddAndRead();
     void testReadPastEnd();
+    void testWritePastEnd();
     void benchmarkEncodingDynamicAlloc();
     void benchmarkEncodingStaticAlloc();
     void benchmarkDecoding();