Initial version

libraries/gpu/src/gpu/Texture.h

@@ -24,6 +24,7 @@
 #include "Forward.h"
 #include "Resource.h"
 #include "Metric.h"
+#include "SerDes.h"
 
 const int ABSOLUTE_MAX_TEXTURE_NUM_PIXELS = 8192 * 8192;
 
@@ -136,7 +137,7 @@ public:
         uint8 _wrapModeU = WRAP_REPEAT;
         uint8 _wrapModeV = WRAP_REPEAT;
         uint8 _wrapModeW = WRAP_REPEAT;
-            
+
         uint8 _mipOffset = 0;
         uint8 _minMip = 0;
         uint8 _maxMip = MAX_MIP_LEVEL;
@@ -156,8 +157,24 @@ public:
                 _minMip == other._minMip &&
                 _maxMip == other._maxMip;
         }
+
+        SerDes &operator<<(SerDes &dsd) {
+            dsd << _borderColor;
+            dsd << _maxAnisotropy;
+            dsd << _filter;
+            dsd << _comparisonFunc;
+            dsd << _wrapModeU;
+            dsd << _wrapModeV;
+            dsd << _wrapModeW;
+            dsd << _mipOffset;
+            dsd << _minMip;
+            dsd << _maxMip;
+            return dsd;
+        }
     };
 
+
+
     Sampler() {}
     Sampler(const Filter filter, const WrapMode wrap = WRAP_REPEAT) : _desc(filter, wrap) {}
     Sampler(const Desc& desc) : _desc(desc) {}
@@ -193,6 +210,33 @@ protected:
     friend class Deserializer;
 };
 
+inline SerDes &operator<<(SerDes &ser, const Sampler::Desc &d) {
+    ser << d._borderColor;
+    ser << d._maxAnisotropy;
+    ser << d._filter;
+    ser << d._comparisonFunc;
+    ser << d._wrapModeU;
+    ser << d._wrapModeV;
+    ser << d._wrapModeW;
+    ser << d._mipOffset;
+    ser << d._minMip;
+    ser << d._maxMip;
+    return ser;
+}
+
+inline SerDes &operator>>(SerDes &dsr, Sampler::Desc &d) {
+    dsr >> d._borderColor;
+    dsr >> d._maxAnisotropy;
+    dsr >> d._filter;
+    dsr >> d._comparisonFunc;
+    dsr >> d._wrapModeU;
+    dsr >> d._wrapModeV;
+    dsr >> d._wrapModeW;
+    dsr >> d._mipOffset;
+    dsr >> d._minMip;
+    dsr >> d._maxMip;
+    return dsr;
+}
 enum class TextureUsageType : uint8 {
     RENDERBUFFER,       // Used as attachments to a framebuffer
     RESOURCE,           // Resource textures, like materials... subject to memory manipulation
@@ -230,7 +274,7 @@ public:
             NORMAL,      // Texture is a normal map
             ALPHA,      // Texture has an alpha channel
             ALPHA_MASK,       // Texture alpha channel is a Mask 0/1
-            NUM_FLAGS,  
+            NUM_FLAGS,
         };
 
         typedef std::bitset<NUM_FLAGS> Flags;
@@ -478,7 +522,7 @@ public:
     uint16 evalMipDepth(uint16 level) const { return std::max(_depth >> level, 1); }
 
     // The true size of an image line or surface depends on the format, tiling and padding rules
-    // 
+    //
     // Here are the static function to compute the different sizes from parametered dimensions and format
     // Tile size must be a power of 2
     static uint16 evalTiledPadding(uint16 length, int tile) { int tileMinusOne = (tile - 1); return (tileMinusOne - (length + tileMinusOne) % tile); }
@@ -507,7 +551,7 @@ public:
     uint32 evalMipFaceNumTexels(uint16 level) const { return evalMipWidth(level) * evalMipHeight(level) * evalMipDepth(level); }
     uint32 evalMipNumTexels(uint16 level) const { return evalMipFaceNumTexels(level) * getNumFaces(); }
 
-    // For convenience assign a source name 
+    // For convenience assign a source name
     const std::string& source() const { return _source; }
     void setSource(const std::string& source) { _source = source; }
     const std::string& sourceHash() const { return _sourceHash; }
@@ -633,7 +677,7 @@ protected:
     uint16 _maxMipLevel { 0 };
 
     uint16 _minMip { 0 };
- 
+
     Type _type { TEX_1D };
 
     Usage _usage;
@@ -643,7 +687,7 @@ protected:
     bool _isIrradianceValid = false;
     bool _defined = false;
     bool _important = false;
-   
+
     static TexturePointer create(TextureUsageType usageType, Type type, const Element& texelFormat, uint16 width, uint16 height, uint16 depth, uint16 numSamples, uint16 numSlices, uint16 numMips, const Sampler& sampler);
 
     Size resize(Type type, const Element& texelFormat, uint16 width, uint16 height, uint16 depth, uint16 numSamples, uint16 numSlices, uint16 numMips);

libraries/gpu/src/gpu/Texture_ktx.cpp

@@ -18,6 +18,7 @@
 #include <ktx/KTX.h>
 
 #include "GPULogging.h"
+#include "SerDes.h"
 
 using namespace gpu;
 
@@ -42,6 +43,44 @@ struct GPUKTXPayload {
     TextureUsageType _usageType;
     glm::ivec2 _originalSize { 0, 0 };
 
+    void serialize2(SerDes &ser) {
+        ser << CURRENT_VERSION;
+        ser << _samplerDesc;
+
+        uint32 usageData = _usage._flags.to_ulong();
+        ser << usageData;
+
+        ser << (char)_usageType;
+        ser << _originalSize;
+        ser.addPadding(PADDING);
+    }
+
+    bool unserialize2(SerDes &dsr) {
+        Version version = 0;
+        uint32 usageData;
+        uint8_t usagetype = 0;
+
+        dsr >> version;
+
+        if (version > CURRENT_VERSION) {
+            // If we try to load a version that we don't know how to parse,
+            // it will render incorrectly
+            return false;
+        }
+
+        dsr >> _samplerDesc;
+        dsr >> usageData;
+        dsr >> usagetype;
+        _usageType = (TextureUsageType)usagetype;
+
+        if (version >= 2) {
+            dsr >> _originalSize;
+        }
+
+        return true;
+    }
+
+
     Byte* serialize(Byte* data) const {
         *(Version*)data = CURRENT_VERSION;
         data += sizeof(Version);
@@ -103,7 +142,9 @@ struct GPUKTXPayload {
         auto found = std::find_if(keyValues.begin(), keyValues.end(), isGPUKTX);
         if (found != keyValues.end()) {
             auto value = found->_value;
-            return payload.unserialize(value.data(), value.size());
+            SerDes dsr(value.data(), value.size());
+            return payload.unserialize2(dsr);
+            //return payload.unserialize(value.data(), value.size());
         }
         return false;
     }
@@ -467,7 +508,9 @@ ktx::KTXUniquePointer Texture::serialize(const Texture& texture, const glm::ivec
     gpuKeyval._originalSize = originalSize;
 
     Byte keyvalPayload[GPUKTXPayload::SIZE];
-    gpuKeyval.serialize(keyvalPayload);
+    SerDes ser(keyvalPayload, sizeof(keyvalPayload));
+
+    gpuKeyval.serialize2(ser);
 
     ktx::KeyValues keyValues;
     keyValues.emplace_back(GPUKTXPayload::KEY, (uint32)GPUKTXPayload::SIZE, (ktx::Byte*) &keyvalPayload);

libraries/shared/src/SerDes.cpp

@@ -0,0 +1,70 @@
+//
+//  SerDes.h
+//
+//
+//  Created by Dale Glass on 5/6/2022
+//  Copyright 2022 Dale Glass
+//
+//  Distributed under the Apache License, Version 2.0.
+//  See the accompanying file LICENSE or http://www.apache.org/licenses/LICENSE-2.0.html
+//
+
+#include <cctype>
+
+#include "SerDes.h"
+const int  SerDes::DEFAULT_SIZE;
+const char SerDes::PADDING_CHAR;
+
+QDebug operator<<(QDebug debug, const SerDes &ds) {
+    debug << "{ capacity =" << ds.capacity() << "; length = " << ds.length() << "; pos = " << ds.pos() << "}";
+    debug << "\n";
+
+    QString literal;
+    QString hex;
+
+    for(size_t i=0;i<ds.length();i++) {
+        char c = ds._store[i];
+
+        if (std::isalnum(c)) {
+            literal.append(c);
+        } else {
+            literal.append(".");
+        }
+
+        QString hnum = QString::number( static_cast<unsigned char>(c), 16 );
+        if ( hnum.length() == 1 ) {
+            hnum.prepend("0");
+        }
+
+        hex.append(hnum  + " ");
+
+        if ( literal.length() == 16 || (i+1 == ds.length()) ) {
+            while( literal.length() < 16 ) {
+                literal.append(" ");
+                hex.append("   ");
+            }
+
+            debug << literal << "  " << hex << "\n";
+            literal.clear();
+            hex.clear();
+        }
+    }
+
+    return debug;
+}
+
+
+void SerDes::changeAllocation(size_t new_size) {
+    while ( _capacity < new_size) {
+        _capacity *= 2;
+    }
+
+    char *new_buf = new char[_capacity];
+    assert( *new_buf );
+
+    memcpy(new_buf, _store, _length);
+    char *prev_buf = _store;
+    _store = new_buf;
+
+    delete []prev_buf;
+}

libraries/shared/src/SerDes.h

@@ -0,0 +1,382 @@
+//
+//  SerDes.h
+//
+//
+//  Created by Dale Glass on 5/6/2022
+//  Copyright 2022 Dale Glass
+//
+//  Distributed under the Apache License, Version 2.0.
+//  See the accompanying file LICENSE or http://www.apache.org/licenses/LICENSE-2.0.html
+//
+
+#pragma once
+#include <string>
+#include <cstring>
+#include <QtCore/QByteArray>
+#include <QDebug>
+#include <glm/glm.hpp>
+
+/**
+ * @brief Data serializer/deserializer
+ *
+ * When encoding, this class takes in data and encodes it into a buffer. No attempt is made to store version numbers, lengths,
+ * or any other metadata. It's entirely up to the user to use the class in such a way that the process can be
+ * correctly reversed if variable-length or optional fields are used.
+ *
+ * It can operate both on an internal, dynamically-allocated buffer, or an externally provided, fixed-size one.
+ *
+ * If an external store is used, the class will refuse to add data once capacity is reached and set the overflow flag.
+ *
+ * When decoding, this class operates on a fixed size buffer. If an attempt to read past the end is made, the read fails,
+ * and the overflow flag is set.
+ *
+ * The class was written for the maximum simplicity possible and inline friendliness.
+ */
+class SerDes {
+    public:
+        // This class is aimed at network serialization, so we assume we're going to deal
+        // with something MTU-sized by default.
+        static const int  DEFAULT_SIZE = 1500;
+        static const char PADDING_CHAR = 0xAA;
+
+        /**
+         * @brief Construct a dynamically allocated serializer
+         *
+         * If constructed this way, an internal buffer will be dynamically allocated and grown as needed.
+         *
+         * The default buffer size is 1500 bytes, based on the assumption that it will be used to construct
+         * network packets.
+         */
+        SerDes() {
+            _capacity = DEFAULT_SIZE;
+            _pos = 0;
+            _length = 0;
+            _store = new char[_capacity];
+        }
+
+        /**
+         * @brief Construct a statically allocated serializer
+         *
+         * If constructed this way, the external buffer will be used to store data. The class will refuse to
+         * keep adding data if the maximum length is reached, and set the overflow flag.
+         *
+         * The flag can be read with isOverflow()
+         *
+         * @param externalStore External data store
+         * @param storeLength Length of the data store
+         */
+        SerDes(char *externalStore, size_t storeLength) {
+            _capacity = storeLength;
+            _length = storeLength;
+            _pos = 0;
+            _storeIsExternal = true;
+            _store = externalStore;
+        }
+
+        SerDes(uint8_t *externalStore, size_t storeLength) : SerDes((char*)externalStore, storeLength) {
+
+        }
+
+        SerDes(const SerDes &) = delete;
+        SerDes &operator=(const SerDes &) = delete;
+
+
+
+        ~SerDes() {
+            if (!_storeIsExternal) {
+                delete[] _store;
+            }
+        }
+
+        void addPadding(size_t bytes) {
+            if (!extendBy(bytes)) {
+                return;
+            }
+
+            // Fill padding with something recognizable. Will keep valgrind happier.
+            memset(&_store[_pos], PADDING_CHAR, bytes);
+            _pos += bytes;
+        }
+
+        SerDes &operator<<(uint8_t c) {
+            return *this << int8_t(c);
+        }
+
+        SerDes &operator<<(int8_t c) {
+            if (!extendBy(1)) {
+                return *this;
+            }
+
+            _store[_pos++] = c;
+            return *this;
+        }
+
+        SerDes &operator>>(uint8_t &c) {
+            return *this >> reinterpret_cast<int8_t&>(c);
+        }
+
+        SerDes &operator>>(int8_t &c) {
+            if ( _pos < _length ) {
+                c = _store[_pos++];
+            } else {
+                _overflow = true;
+                qCritical() << "Deserializer trying to read past end of input, reading 8 bits from position " << _pos << ", length " << _length;
+            }
+
+            return *this;
+        }
+
+        ///////////////////////////////////////////////////////////
+
+        SerDes &operator<<(uint16_t val) {
+            return *this << int16_t(val);
+        }
+
+        SerDes &operator<<(int16_t val) {
+            if (!extendBy(sizeof(val))) {
+                return *this;
+            }
+
+            memcpy(&_store[_pos], (char*)&val, sizeof(val));
+            _pos += sizeof(val);
+            return *this;
+        }
+
+        SerDes &operator>>(uint16_t &val) {
+            return *this >> reinterpret_cast<int16_t&>(val);
+        }
+
+        SerDes &operator>>(int16_t &val) {
+            if ( _pos + sizeof(val) <= _length ) {
+                memcpy((char*)&val, &_store[_pos], sizeof(val));
+                _pos += sizeof(val);
+            } else {
+                _overflow = true;
+                qCritical() << "Deserializer trying to read past end of input, reading 16 bits from position " << _pos << ", length " << _length;
+            }
+
+            return *this;
+        }
+
+        ///////////////////////////////////////////////////////////
+
+        SerDes &operator<<(uint32_t val) {
+            return *this << int32_t(val);
+        }
+
+        SerDes &operator<<(int32_t val) {
+            if (!extendBy(sizeof(val))) {
+                return *this;
+            }
+
+            memcpy(&_store[_pos], (char*)&val, sizeof(val));
+            _pos += sizeof(val);
+            return *this;
+        }
+
+        SerDes &operator>>(uint32_t &val) {
+            return *this >> reinterpret_cast<int32_t&>(val);
+        }
+
+        SerDes &operator>>(int32_t &val) {
+            if ( _pos + sizeof(val) <= _length ) {
+                memcpy((char*)&val, &_store[_pos], sizeof(val));
+                _pos += sizeof(val);
+            } else {
+                _overflow = true;
+                qCritical() << "Deserializer trying to read past end of input, reading 32 bits from position " << _pos << ", length " << _length;
+            }
+            return *this;
+        }
+
+
+        ///////////////////////////////////////////////////////////
+
+        SerDes &operator<<(glm::vec3 val) {
+            size_t sz = sizeof(val.x);
+            if (!extendBy(sz*3)) {
+                return *this;
+            }
+
+            memcpy(&_store[_pos       ], (char*)&val.x, sz);
+            memcpy(&_store[_pos + sz  ], (char*)&val.y, sz);
+            memcpy(&_store[_pos + sz*2], (char*)&val.z, sz);
+
+            _pos += sz*3;
+            return *this;
+        }
+
+        SerDes &operator>>(glm::vec3 &val) {
+            size_t sz = sizeof(val.x);
+
+            if ( _pos + sz*3 <= _length ) {
+                memcpy((char*)&val.x, &_store[_pos       ], sz);
+                memcpy((char*)&val.y, &_store[_pos + sz  ], sz);
+                memcpy((char*)&val.z, &_store[_pos + sz*2], sz);
+
+                _pos += sz*3;
+            } else {
+                _overflow = true;
+                qCritical() << "Deserializer trying to read past end of input, reading glm::vec3 from position " << _pos << ", length " << _length;
+            }
+            return *this;
+        }
+
+        ///////////////////////////////////////////////////////////
+
+        SerDes &operator<<(glm::vec4 val) {
+            size_t sz = sizeof(val.x);
+            if (!extendBy(sz*4)) {
+                return *this;
+            }
+
+            memcpy(&_store[_pos       ], (char*)&val.x, sz);
+            memcpy(&_store[_pos + sz  ], (char*)&val.y, sz);
+            memcpy(&_store[_pos + sz*2], (char*)&val.z, sz);
+            memcpy(&_store[_pos + sz*3], (char*)&val.w, sz);
+
+            _pos += sz*3;
+            return *this;
+        }
+
+        SerDes &operator>>(glm::vec4 &val) {
+            size_t sz = sizeof(val.x);
+
+            if ( _pos + sz*4 <= _length ) {
+                memcpy((char*)&val.x, &_store[_pos       ], sz);
+                memcpy((char*)&val.y, &_store[_pos + sz  ], sz);
+                memcpy((char*)&val.z, &_store[_pos + sz*2], sz);
+                memcpy((char*)&val.w, &_store[_pos + sz*3], sz);
+
+                _pos += sz*3;
+            } else {
+                _overflow = true;
+                qCritical() << "Deserializer trying to read past end of input, reading glm::vec3 from position " << _pos << ", length " << _length;
+            }
+            return *this;
+        }
+
+        ///////////////////////////////////////////////////////////
+
+        SerDes &operator<<(glm::ivec2 val) {
+            size_t sz = sizeof(val.x);
+            if (!extendBy(sz*2)) {
+                return *this;
+            }
+
+            memcpy(&_store[_pos       ], (char*)&val.x, sz);
+            memcpy(&_store[_pos + sz  ], (char*)&val.y, sz);
+
+            _pos += sz*2;
+            return *this;
+        }
+
+        SerDes &operator>>(glm::ivec2 &val) {
+            size_t sz = sizeof(val.x);
+
+            if ( _pos + sz*2 <= _length ) {
+                memcpy((char*)&val.x, &_store[_pos       ], sz);
+                memcpy((char*)&val.y, &_store[_pos + sz  ], sz);
+
+                _pos += sz*2;
+            } else {
+                _overflow = true;
+                qCritical() << "Deserializer trying to read past end of input, reading glm::ivec2 from position " << _pos << ", length " << _length;
+            }
+            return *this;
+        }
+        ///////////////////////////////////////////////////////////
+
+        SerDes &operator<<(const char *val) {
+            size_t len = strlen(val)+1;
+            extendBy(len);
+            memcpy(&_store[_pos], val, len);
+            _pos += len;
+            return *this;
+        }
+
+        SerDes &operator<<(const QString &val) {
+            return *this << val.toUtf8().constData();
+        }
+
+
+        ///////////////////////////////////////////////////////////
+
+        /**
+         * @brief Current position in the buffer. Starts at 0.
+         *
+         * @return size_t
+         */
+        size_t pos() const { return _pos; }
+
+        /**
+         * @brief Last position that was written to in the buffer. Starts at 0.
+         *
+         * @return size_t
+         */
+        size_t length() const { return _length; }
+
+        /**
+         * @brief Current capacity of the buffer.
+         *
+         * If the buffer is dynamically allocated, it can grow.
+         *
+         * If the buffer is static, this is a fixed limit.
+         *
+         * @return size_t
+         */
+        size_t capacity() const { return _capacity; }
+
+        /**
+         * @brief Whether there's any data in the buffer
+         *
+         * @return true Something has been written
+         * @return false The buffer is empty
+         */
+        bool isEmpty() const { return _length == 0; }
+
+        /**
+         * @brief The buffer size limit has been reached
+         *
+         * This can only return true for a statically allocated buffer.
+         *
+         * @return true Limit reached
+         * @return false There is still room
+         */
+        bool isOverflow() const { return _overflow; }
+
+        /**
+         * @brief Reset the serializer to the start, clear overflow bit.
+         *
+         */
+        void rewind() { _pos = 0; _overflow = false; }
+
+        friend QDebug operator<<(QDebug debug, const SerDes &ds);
+
+    private:
+        bool extendBy(size_t bytes) {
+            //qDebug() << "Extend by" << bytes;
+
+            if ( _capacity < _length + bytes) {
+                if ( _storeIsExternal ) {
+                    _overflow = true;
+                    return false;
+                }
+
+                changeAllocation(_length + bytes);
+            }
+
+            _length += bytes;
+            return true;
+        }
+
+        // This is split up here to try to make the class as inline-friendly as possible.
+        void changeAllocation(size_t new_size);
+
+        char *_store;
+        bool _storeIsExternal = false;
+        bool _overflow = false;
+        size_t _capacity = 0;
+        size_t _length = 0;
+        size_t _pos = 0;
+};

tests/shared/src/SerializerTests.cpp

@@ -0,0 +1,107 @@
+//
+//  SerializerTests.cpp
+//
+//  Copyright 2022 Dale Glass
+//
+//  Distributed under the Apache License, Version 2.0.
+//  See the accompanying file LICENSE or http://www.apache.org/licenses/LICENSE-2.0.html
+//
+
+
+#include "SerializerTests.h"
+#include <SerDes.h>
+#include <QDebug>
+#include <glm/glm.hpp>
+
+QTEST_GUILESS_MAIN(SerializerTests)
+
+
+void SerializerTests::initTestCase() {
+}
+
+void SerializerTests::testCreate() {
+    SerDes s;
+    QCOMPARE(s.length(), 0);
+    QCOMPARE(s.capacity(), SerDes::DEFAULT_SIZE);
+    QCOMPARE(s.isEmpty(), true);
+}
+
+void SerializerTests::testAdd() {
+    SerDes s;
+    s << (qint8)1;
+    QCOMPARE(s.length(), 1);
+    QCOMPARE(s.isEmpty(), false);
+
+    s << (quint8)-1;
+    QCOMPARE(s.length(), 2);
+
+    s << (qint16)0xaabb;
+    QCOMPARE(s.length(), 4);
+
+    s << (quint16)-18000;
+    QCOMPARE(s.length(), 6);
+
+    s << (qint32)0xCCDDEEFF;
+    QCOMPARE(s.length(), 10);
+
+    s << (quint32)-1818000000;
+    QCOMPARE(s.length(), 14);
+
+    s << "Hello, world!";
+    QCOMPARE(s.length(), 28);
+
+    glm::vec3 v{1.f,2.f,3.f};
+    s << v;
+    QCOMPARE(s.length(), 40);
+
+
+    qDebug() << s;
+}
+
+void SerializerTests::testAddAndRead() {
+    SerDes s;
+    glm::vec3 v{1.f, 3.1415f, 2.71828f};
+    glm::vec3 v2;
+
+    s << (qint8)1;
+    s << (qint16)0xaabb;
+    s << (qint32)0xccddeeff;
+    s << v;
+
+    qint8 i8;
+    qint16 i16;
+    qint32 i32;
+
+    s.rewind();
+
+    s >> i8;
+    s >> i16;
+    s >> i32;
+    s >> v2;
+
+    qDebug() << s;
+
+    QCOMPARE(i8, (qint8)1);
+    QCOMPARE(i16, (qint16)0xaabb);
+    QCOMPARE(i32, (qint32)0xccddeeff);
+    QCOMPARE(v, v2);
+}
+
+void SerializerTests::testReadPastEnd() {
+    SerDes s;
+    qint8 i8;
+    qint16 i16;
+    s << (qint8)1;
+    s.rewind();
+    s >> i8;
+    QCOMPARE(s.pos(), 1);
+
+    s.rewind();
+    s >> i16;
+    QCOMPARE(s.pos(), 0);
+}
+
+
+void SerializerTests::cleanupTestCase() {
+}
+

tests/shared/src/SerializerTests.h

@@ -0,0 +1,29 @@
+//
+//  ResourceTests.h
+//
+//  Copyright 2015 High Fidelity, Inc.
+//
+//  Distributed under the Apache License, Version 2.0.
+//  See the accompanying file LICENSE or http://www.apache.org/licenses/LICENSE-2.0.html
+//
+
+#ifndef overte_SerializerTests_h
+#define overte_SerializerTests_h
+
+#include <QtTest/QtTest>
+#include <QtCore/QTemporaryDir>
+
+class SerializerTests : public QObject {
+    Q_OBJECT
+private slots:
+    void initTestCase();
+    void testCreate();
+    void testAdd();
+    void testAddAndRead();
+    void testReadPastEnd();
+    void cleanupTestCase();
+private:
+
+};
+
+#endif // overte_SerializerTests_h