Merge pull request #1293 from HifiExperiments/implement-serializer

Implement network serializer (rebased)

libraries/gpu/src/gpu/Texture.h

@@ -24,6 +24,7 @@
 #include "Forward.h"
 #include "Resource.h"
 #include "Metric.h"
+#include "SerDes.h"
 
 const int ABSOLUTE_MAX_TEXTURE_NUM_PIXELS = 8192 * 8192;
 
@@ -91,6 +92,37 @@ public:
 };
 typedef std::shared_ptr< SphericalHarmonics > SHPointer;
 
+
+inline DataSerializer &operator<<(DataSerializer &ser, const SphericalHarmonics &h) {
+    DataSerializer::SizeTracker tracker(ser);
+
+    ser << h.L00  << h.spare0;
+    ser << h.L1m1 << h.spare1;
+    ser << h.L10  << h.spare2;
+    ser << h.L11  << h.spare3;
+    ser << h.L2m2 << h.spare4;
+    ser << h.L2m1 << h.spare5;
+    ser << h.L20  << h.spare6;
+    ser << h.L21  << h.spare7;
+    ser << h.L22  << h.spare8;
+    return ser;
+}
+
+inline DataDeserializer &operator>>(DataDeserializer &des, SphericalHarmonics &h) {
+    DataDeserializer::SizeTracker tracker(des);
+
+    des >> h.L00  >> h.spare0;
+    des >> h.L1m1 >> h.spare1;
+    des >> h.L10  >> h.spare2;
+    des >> h.L11  >> h.spare3;
+    des >> h.L2m2 >> h.spare4;
+    des >> h.L2m1 >> h.spare5;
+    des >> h.L20  >> h.spare6;
+    des >> h.L21  >> h.spare7;
+    des >> h.L22  >> h.spare8;
+    return des;
+}
+
 class Sampler {
 public:
 
@@ -136,7 +168,7 @@ public:
         uint8 _wrapModeU = WRAP_REPEAT;
         uint8 _wrapModeV = WRAP_REPEAT;
         uint8 _wrapModeW = WRAP_REPEAT;
-            
+
         uint8 _mipOffset = 0;
         uint8 _minMip = 0;
         uint8 _maxMip = MAX_MIP_LEVEL;
@@ -193,6 +225,35 @@ protected:
     friend class Deserializer;
 };
 
+inline DataSerializer &operator<<(DataSerializer &ser, const Sampler::Desc &d) {
+    DataSerializer::SizeTracker tracker(ser);
+    ser << d._borderColor;
+    ser << d._maxAnisotropy;
+    ser << d._filter;
+    ser << d._comparisonFunc;
+    ser << d._wrapModeU;
+    ser << d._wrapModeV;
+    ser << d._wrapModeW;
+    ser << d._mipOffset;
+    ser << d._minMip;
+    ser << d._maxMip;
+    return ser;
+}
+
+inline DataDeserializer &operator>>(DataDeserializer &dsr, Sampler::Desc &d) {
+    DataDeserializer::SizeTracker tracker(dsr);
+    dsr >> d._borderColor;
+    dsr >> d._maxAnisotropy;
+    dsr >> d._filter;
+    dsr >> d._comparisonFunc;
+    dsr >> d._wrapModeU;
+    dsr >> d._wrapModeV;
+    dsr >> d._wrapModeW;
+    dsr >> d._mipOffset;
+    dsr >> d._minMip;
+    dsr >> d._maxMip;
+    return dsr;
+}
 enum class TextureUsageType : uint8 {
     RENDERBUFFER,       // Used as attachments to a framebuffer
     RESOURCE,           // Resource textures, like materials... subject to memory manipulation
@@ -230,7 +291,7 @@ public:
             NORMAL,      // Texture is a normal map
             ALPHA,      // Texture has an alpha channel
             ALPHA_MASK,       // Texture alpha channel is a Mask 0/1
-            NUM_FLAGS,  
+            NUM_FLAGS,
         };
 
         typedef std::bitset<NUM_FLAGS> Flags;
@@ -478,7 +539,7 @@ public:
     uint16 evalMipDepth(uint16 level) const { return std::max(_depth >> level, 1); }
 
     // The true size of an image line or surface depends on the format, tiling and padding rules
-    // 
+    //
     // Here are the static function to compute the different sizes from parametered dimensions and format
     // Tile size must be a power of 2
     static uint16 evalTiledPadding(uint16 length, int tile) { int tileMinusOne = (tile - 1); return (tileMinusOne - (length + tileMinusOne) % tile); }
@@ -507,7 +568,7 @@ public:
     uint32 evalMipFaceNumTexels(uint16 level) const { return evalMipWidth(level) * evalMipHeight(level) * evalMipDepth(level); }
     uint32 evalMipNumTexels(uint16 level) const { return evalMipFaceNumTexels(level) * getNumFaces(); }
 
-    // For convenience assign a source name 
+    // For convenience assign a source name
     const std::string& source() const { return _source; }
     void setSource(const std::string& source) { _source = source; }
     const std::string& sourceHash() const { return _sourceHash; }
@@ -633,7 +694,7 @@ protected:
     uint16 _maxMipLevel { 0 };
 
     uint16 _minMip { 0 };
- 
+
     Type _type { TEX_1D };
 
     Usage _usage;
@@ -643,7 +704,7 @@ protected:
     bool _isIrradianceValid = false;
     bool _defined = false;
     bool _important = false;
-   
+
     static TexturePointer create(TextureUsageType usageType, Type type, const Element& texelFormat, uint16 width, uint16 height, uint16 depth, uint16 numSamples, uint16 numSlices, uint16 numMips, const Sampler& sampler);
 
     Size resize(Type type, const Element& texelFormat, uint16 width, uint16 height, uint16 depth, uint16 numSamples, uint16 numSlices, uint16 numMips);

libraries/gpu/src/gpu/Texture_ktx.cpp

@@ -18,6 +18,7 @@
 #include <ktx/KTX.h>
 
 #include "GPULogging.h"
+#include "SerDes.h"
 
 using namespace gpu;
 
@@ -27,71 +28,94 @@ using KtxStorage = Texture::KtxStorage;
 std::vector<std::pair<std::shared_ptr<storage::FileStorage>, std::shared_ptr<std::mutex>>> KtxStorage::_cachedKtxFiles;
 std::mutex KtxStorage::_cachedKtxFilesMutex;
 
+
+/**
+ * @brief Payload for a KTX (texture)
+ *
+ * This contains a ready to use texture. This is both used for the local cache, and for baked textures.
+ *
+ * @note The usage for textures means breaking compatibility is a bad idea, and that the implementation
+ * should just keep on adding extra data at the bottom of the structure, and remain able to read old
+ * formats. In fact, version 1 KTX can be found in older baked assets.
+ */
 struct GPUKTXPayload {
     using Version = uint8;
 
     static const std::string KEY;
     static const Version CURRENT_VERSION { 2 };
     static const size_t PADDING { 2 };
-    static const size_t SIZE { sizeof(Version) + sizeof(Sampler::Desc) + sizeof(uint32) + sizeof(TextureUsageType) + sizeof(glm::ivec2) + PADDING };
+    static const size_t SIZE { sizeof(Version) + sizeof(Sampler::Desc) + sizeof(uint32_t) + sizeof(TextureUsageType) + sizeof(glm::ivec2) + PADDING };
+
     static_assert(GPUKTXPayload::SIZE == 44, "Packing size may differ between platforms");
-    static_assert(GPUKTXPayload::SIZE % 4 == 0, "GPUKTXPayload is not 4 bytes aligned");
 
     Sampler::Desc _samplerDesc;
     Texture::Usage _usage;
     TextureUsageType _usageType;
     glm::ivec2 _originalSize { 0, 0 };
 
-    Byte* serialize(Byte* data) const {
+    /**
-        *(Version*)data = CURRENT_VERSION;
+     * @brief Serialize the KTX payload
-        data += sizeof(Version);
+     *
+     * @warning Be careful modifying this code, as it influences baked assets.
+     * Backwards compatibility must be maintained.
+     *
+     * @param ser Destination serializer
+     */
+    void serialize(DataSerializer &ser) {
 
-        memcpy(data, &_samplerDesc, sizeof(Sampler::Desc));
+        ser << CURRENT_VERSION;
-        data += sizeof(Sampler::Desc);
 
-        // We can't copy the bitset in Texture::Usage in a crossplateform manner
+        ser << _samplerDesc;
-        // So serialize it manually
-        uint32 usageData = _usage._flags.to_ulong();
-        memcpy(data, &usageData, sizeof(uint32));
-        data += sizeof(uint32);
 
-        memcpy(data, &_usageType, sizeof(TextureUsageType));
+        uint32_t usageData = (uint32_t)_usage._flags.to_ulong();
-        data += sizeof(TextureUsageType);
+        ser << usageData;
+        ser << ((uint8_t)_usageType);
+        ser << _originalSize;
 
-        memcpy(data, glm::value_ptr(_originalSize), sizeof(glm::ivec2));
+        ser.addPadding(PADDING);
-        data += sizeof(glm::ivec2);
 
-        return data + PADDING;
+        assert(ser.length() == GPUKTXPayload::SIZE);
     }
 
-    bool unserialize(const Byte* data, size_t size) {
+    /**
-        Version version = *(const Version*)data;
+     * @brief Deserialize the KTX payload
-        data += sizeof(Version);
+     *
+     * @warning Be careful modifying this code, as it influences baked assets.
+     * Backwards compatibility must be maintained.
+     *
+     * @param dsr Deserializer object
+     * @return true Successful
+     * @return false Version check failed
+     */
+    bool unserialize(DataDeserializer &dsr) {
+        Version version = 0;
+        uint32_t usageData = 0;
+        uint8_t usagetype = 0;
+
+        dsr >> version;
 
         if (version > CURRENT_VERSION) {
             // If we try to load a version that we don't know how to parse,
             // it will render incorrectly
+            qCWarning(gpulogging) << "KTX version" << version << "is newer than our own," << CURRENT_VERSION;
+            qCWarning(gpulogging) << dsr;
             return false;
         }
 
-        memcpy(&_samplerDesc, data, sizeof(Sampler::Desc));
+        dsr >> _samplerDesc;
-        data += sizeof(Sampler::Desc);
 
-        // We can't copy the bitset in Texture::Usage in a crossplateform manner
+        dsr >> usageData;
-        // So unserialize it manually
+        _usage = gpu::Texture::Usage(usageData);
-        uint32 usageData;
-        memcpy(&usageData, data, sizeof(uint32));
-        _usage = Texture::Usage(usageData);
-        data += sizeof(uint32);
 
-        memcpy(&_usageType, data, sizeof(TextureUsageType));
+        dsr >> usagetype;
-        data += sizeof(TextureUsageType);
+        _usageType = (TextureUsageType)usagetype;
 
         if (version >= 2) {
-            memcpy(&_originalSize, data, sizeof(glm::ivec2));
+            dsr >> _originalSize;
-            data += sizeof(glm::ivec2);
         }
 
+        dsr.skipPadding(PADDING);
+
         return true;
     }
 
@@ -103,7 +127,8 @@ struct GPUKTXPayload {
         auto found = std::find_if(keyValues.begin(), keyValues.end(), isGPUKTX);
         if (found != keyValues.end()) {
             auto value = found->_value;
-            return payload.unserialize(value.data(), value.size());
+            DataDeserializer dsr(value.data(), value.size());
+            return payload.unserialize(dsr);
         }
         return false;
     }
@@ -123,29 +148,24 @@ struct IrradianceKTXPayload {
 
     SphericalHarmonics _irradianceSH;
 
-    Byte* serialize(Byte* data) const {
+    void serialize(DataSerializer &ser) const {
-        *(Version*)data = CURRENT_VERSION;
+        ser << CURRENT_VERSION;
-        data += sizeof(Version);
+        ser << _irradianceSH;
-
+        ser.addPadding(PADDING);
-        memcpy(data, &_irradianceSH, sizeof(SphericalHarmonics));
-        data += sizeof(SphericalHarmonics);
-
-        return data + PADDING;
     }
 
-    bool unserialize(const Byte* data, size_t size) {
+    bool unserialize(DataDeserializer &des) {
-        if (size != SIZE) {
+        Version version;
+        if (des.length() != SIZE) {
             return false;
         }
 
-        Version version = *(const Version*)data;
+        des >> version;
         if (version != CURRENT_VERSION) {
             return false;
         }
-        data += sizeof(Version);
-
-        memcpy(&_irradianceSH, data, sizeof(SphericalHarmonics));
 
+        des >> _irradianceSH;
         return true;
     }
 
@@ -157,7 +177,8 @@ struct IrradianceKTXPayload {
         auto found = std::find_if(keyValues.begin(), keyValues.end(), isIrradianceKTX);
         if (found != keyValues.end()) {
             auto value = found->_value;
-            return payload.unserialize(value.data(), value.size());
+            DataDeserializer des(value.data(), value.size());
+            return payload.unserialize(des);
         }
         return false;
     }
@@ -467,7 +488,9 @@ ktx::KTXUniquePointer Texture::serialize(const Texture& texture, const glm::ivec
     gpuKeyval._originalSize = originalSize;
 
     Byte keyvalPayload[GPUKTXPayload::SIZE];
-    gpuKeyval.serialize(keyvalPayload);
+    DataSerializer ser(keyvalPayload, sizeof(keyvalPayload));
+
+    gpuKeyval.serialize(ser);
 
     ktx::KeyValues keyValues;
     keyValues.emplace_back(GPUKTXPayload::KEY, (uint32)GPUKTXPayload::SIZE, (ktx::Byte*) &keyvalPayload);
@@ -477,7 +500,8 @@ ktx::KTXUniquePointer Texture::serialize(const Texture& texture, const glm::ivec
         irradianceKeyval._irradianceSH = *texture.getIrradiance();
 
         Byte irradianceKeyvalPayload[IrradianceKTXPayload::SIZE];
-        irradianceKeyval.serialize(irradianceKeyvalPayload);
+        DataSerializer ser(irradianceKeyvalPayload, sizeof(irradianceKeyvalPayload));
+        irradianceKeyval.serialize(ser);
 
         keyValues.emplace_back(IrradianceKTXPayload::KEY, (uint32)IrradianceKTXPayload::SIZE, (ktx::Byte*) &irradianceKeyvalPayload);
     }

libraries/octree/src/OctreePacketData.cpp

@@ -17,6 +17,7 @@
 #include "OctreeLogging.h"
 #include "NumericalConstants.h"
 #include <glm/gtc/type_ptr.hpp>
+#include "SerDes.h"
 
 bool OctreePacketData::_debug = false;
 AtomicUIntStat OctreePacketData::_totalBytesOfOctalCodes { 0 };
@@ -847,10 +848,10 @@ int OctreePacketData::unpackDataFromBytes(const unsigned char* dataBytes, QByteA
 }
 
 int OctreePacketData::unpackDataFromBytes(const unsigned char* dataBytes, AACube& result) {
-    aaCubeData cube;
+    DataDeserializer des(dataBytes, sizeof(aaCubeData));
-    memcpy(&cube, dataBytes, sizeof(aaCubeData));
+    des >> result;
-    result = AACube(cube.corner, cube.scale);
+
-    return sizeof(aaCubeData);
+    return des.length();
 }
 
 int OctreePacketData::unpackDataFromBytes(const unsigned char* dataBytes, QRect& result) {

libraries/shared/src/AACube.h

@@ -20,6 +20,7 @@
 #include <QDebug>
 
 #include "BoxBase.h"
+#include "SerDes.h"
 
 class AABox;
 class Extents;
@@ -80,6 +81,10 @@ private:
 
     glm::vec3 _corner;
     float _scale;
+
+    friend DataSerializer& operator<<(DataSerializer &ser, const AACube &cube);
+    friend DataDeserializer& operator>>(DataDeserializer &des, AACube &cube);
+
 };
 
 inline bool operator==(const AACube& a, const AACube& b) {
@@ -99,5 +104,16 @@ inline QDebug operator<<(QDebug debug, const AACube& cube) {
     return debug;
 }
 
+inline DataSerializer& operator<<(DataSerializer &ser, const AACube &cube) {
+    ser << cube._corner;
+    ser << cube._scale;
+    return ser;
+}
+
+inline DataDeserializer& operator>>(DataDeserializer &des, AACube &cube) {
+    des >> cube._corner;
+    des >> cube._scale;
+    return des;
+}
 
 #endif // hifi_AACube_h

libraries/shared/src/BlendshapeConstants.h

@@ -122,6 +122,25 @@ struct BlendshapeOffsetUnpacked {
     float positionOffsetX, positionOffsetY, positionOffsetZ;
     float normalOffsetX, normalOffsetY, normalOffsetZ;
     float tangentOffsetX, tangentOffsetY, tangentOffsetZ;
+
+    /**
+     * @brief Set all components of all the offsets to zero
+     *
+     * @note glm::vec3 is not trivially copyable, so it's not correct to clear it with memset.
+     */
+    void clear() {
+        positionOffsetX = 0.0f;
+        positionOffsetY = 0.0f;
+        positionOffsetZ = 0.0f;
+
+        normalOffsetX = 0.0f;
+        normalOffsetY = 0.0f;
+        normalOffsetZ = 0.0f;
+
+        tangentOffsetX = 0.0f;
+        tangentOffsetY = 0.0f;
+        tangentOffsetZ = 0.0f;
+    }
 };
 
 using BlendshapeOffset = BlendshapeOffsetPacked;

libraries/shared/src/SerDes.cpp

@@ -0,0 +1,85 @@
+//
+//  SerDes.h
+//
+//
+//  Created by Dale Glass on 5/6/2022
+//  Copyright 2024 Overte e.V.
+//
+//  Distributed under the Apache License, Version 2.0.
+//  See the accompanying file LICENSE or http://www.apache.org/licenses/LICENSE-2.0.html
+//
+
+#include <cctype>
+
+#include "SerDes.h"
+const int  DataSerializer::DEFAULT_SIZE;
+const char DataSerializer::PADDING_CHAR;
+
+
+static void dumpHex(QDebug &debug, const char*buf, size_t len) {
+    QString literal;
+    QString hex;
+
+    for(size_t i=0;i<len;i++) {
+        char c = buf[i];
+
+        if (std::isalnum(c)) {
+            literal.append(c);
+        } else {
+            literal.append(".");
+        }
+
+        QString hnum = QString::number( static_cast<unsigned char>(c), 16 );
+        if ( hnum.length() == 1 ) {
+            hnum.prepend("0");
+        }
+
+        hex.append(hnum  + " ");
+
+        if ( literal.length() == 16 || (i+1 == len) ) {
+            while( literal.length() < 16 ) {
+                literal.append(" ");
+                hex.append("   ");
+            }
+
+            debug << literal << "  " << hex << "\n";
+            literal.clear();
+            hex.clear();
+        }
+    }
+}
+
+
+QDebug operator<<(QDebug debug, const DataSerializer &ser) {
+    debug << "{ capacity =" << ser.capacity() << "; length = " << ser.length() << "; pos = " << ser.pos() << "}";
+    debug << "\n";
+
+    dumpHex(debug, ser.buffer(), ser.length());
+    return debug;
+}
+
+
+QDebug operator<<(QDebug debug, const DataDeserializer &des) {
+    debug << "{ length = " << des.length() << "; pos = " << des.pos() << "}";
+    debug << "\n";
+
+
+    dumpHex(debug, des.buffer(), des.length());
+    return debug;
+}
+
+
+void DataSerializer::changeAllocation(size_t new_size) {
+    while ( _capacity < new_size) {
+        _capacity *= 2;
+    }
+
+    char *new_buf = new char[_capacity];
+    assert( *new_buf );
+
+    memcpy(new_buf, _store, _length);
+    char *prev_buf = _store;
+    _store = new_buf;
+
+    delete []prev_buf;
+}

libraries/shared/src/SerDes.h

@@ -0,0 +1,953 @@
+//
+//  SerDes.h
+//
+//
+//  Created by Dale Glass on 5/6/2022
+//  Copyright 2024 Overte e.V.
+//
+//  Distributed under the Apache License, Version 2.0.
+//  See the accompanying file LICENSE or http://www.apache.org/licenses/LICENSE-2.0.html
+//
+
+#pragma once
+#include <string>
+#include <cstring>
+#include <QtCore/QByteArray>
+#include <QDebug>
+#include <glm/glm.hpp>
+
+/**
+ * @brief Data serializer
+ *
+ * When encoding, this class takes in data and encodes it into a buffer. No attempt is made to store version numbers, lengths,
+ * or any other metadata. It's entirely up to the user to use the class in such a way that the process can be
+ * correctly reversed if variable-length or optional fields are used.
+ *
+ * It can operate both on an internal, dynamically-allocated buffer, or an externally provided, fixed-size one.
+ * If an external store is used, the class will refuse to add data once capacity is reached and set the overflow flag.
+ * When decoding, this class operates on a fixed size buffer. If an attempt to read past the end is made, the read fails,
+ * and the overflow flag is set.
+ *
+ * The class was written for the maximum simplicity possible and inline friendliness.
+ *
+ * Example of encoding:
+ *
+ * @code {.cpp}
+ * uint8_t version = 1;
+ * uint16_t count = 1;
+ * glm::vec3 pos{1.5, 2.0, 9.0};
+ *
+ * Serializer ser;
+ * ser << version;
+ * ser << count;
+ * ser << pos;
+ *
+ * // Serialized data is in ser.buffer(), ser.length() long.
+ * @endcode
+ *
+ * This object should be modified directly to add support for any primitive and common datatypes in the code. To support serializing/deserializing
+ * classes and structures, implement a `operator<<` and `operator>>` functions for that object, eg:
+ *
+ * @code {.cpp}
+ * DataSerializer &operator<<(DataSerializer &ser, const Object &o) {
+ *  ser << o._borderColor;
+ *  ser << o._maxAnisotropy;
+ *  ser << o._filter;
+ *  return ser;
+ * }
+ * @endcode
+ *
+ */
+class DataSerializer {
+    public:
+
+        /**
+         * @brief RAII tracker of advance position
+         *
+         * When a custom operator<< is implemented for DataSserializer,
+         * this class allows to easily keep track of how much data has been added and
+         * adjust the parent's lastAdvance() count on this class' destruction.
+         *
+         * @code {.cpp}
+         * DataSerializer &operator<<(DataSerializer &ser, const Object &o) {
+         *  DataSerializer::SizeTracker tracker(ser);
+         *
+         *  ser << o._borderColor;
+         *  ser << o._maxAnisotropy;
+         *  ser << o._filter;
+         *  return ser;
+         * }
+         * @endcode
+         */
+        class SizeTracker {
+            public:
+            SizeTracker(DataSerializer &parent) : _parent(parent) {
+                _start_pos = _parent.pos();
+            }
+
+            ~SizeTracker() {
+                size_t cur_pos = _parent.pos();
+
+                if ( cur_pos >= _start_pos ) {
+                    _parent._lastAdvance = cur_pos - _start_pos;
+                } else {
+                    _parent._lastAdvance = 0;
+                }
+            }
+
+            private:
+            DataSerializer &_parent;
+            size_t _start_pos = 0;
+        };
+
+        /**
+         * @brief Default size for a dynamically allocated buffer.
+         *
+         * Since this is mostly intended to be used for networking, we default to the largest probable MTU here.
+         */
+        static const int DEFAULT_SIZE = 1500;
+
+        /**
+         * @brief Character to use for padding.
+         *
+         * Padding should be ignored, so it doesn't matter what we go with here, but it can be useful to set it
+         * to something that would be distinctive in a dump.
+         */
+        static const char PADDING_CHAR = (char)0xAA;
+
+        /**
+         * @brief Construct a dynamically allocated serializer
+         *
+         * If constructed this way, an internal buffer will be dynamically allocated and grown as needed.
+         *
+         * The buffer is SerDes::DEFAULT_SIZE bytes by default, and doubles in size every time the limit is reached.
+         */
+        DataSerializer() {
+            _capacity = DEFAULT_SIZE;
+            _pos = 0;
+            _length = 0;
+            _store = new char[_capacity];
+        }
+
+        /**
+         * @brief Construct a statically allocated serializer
+         *
+         * If constructed this way, the external buffer will be used to store data. The class will refuse to
+         * keep adding data if the maximum length is reached, write a critical message to the log, and set
+         * the overflow flag.
+         *
+         * The flag can be read with isOverflow()
+         *
+         * @param externalStore External data store
+         * @param storeLength Length of the data store
+         */
+        DataSerializer(char *externalStore, size_t storeLength) {
+            _capacity = storeLength;
+            _length = 0;
+            _pos = 0;
+            _storeIsExternal = true;
+            _store = externalStore;
+        }
+
+        /**
+         * @brief Construct a statically allocated serializer
+         *
+         * If constructed this way, the external buffer will be used to store data. The class will refuse to
+         * keep adding data if the maximum length is reached, and set the overflow flag.
+         *
+         * The flag can be read with isOverflow()
+         *
+         * @param externalStore External data store
+         * @param storeLength Length of the data store
+         */
+        DataSerializer(uint8_t *externalStore, size_t storeLength) : DataSerializer((char*)externalStore, storeLength) {
+
+        }
+
+        DataSerializer(const DataSerializer &) = delete;
+        DataSerializer &operator=(const DataSerializer &) = delete;
+
+
+
+        ~DataSerializer() {
+            if (!_storeIsExternal) {
+                delete[] _store;
+            }
+        }
+
+        /**
+         * @brief Adds padding to the output
+         *
+         * The bytes will be set to SerDes::PADDING_CHAR, which is a constant in the source code.
+         * Since padding isn't supposed to be read, it can be any value and is intended to
+         * be set to something that can be easily recognized in a dump.
+         *
+         * @param bytes Number of bytes to add
+         */
+        void addPadding(size_t bytes) {
+            if (!extendBy(bytes, "padding")) {
+                return;
+            }
+
+            // Fill padding with something recognizable. Will keep valgrind happier.
+            memset(&_store[_pos], PADDING_CHAR, bytes);
+            _pos += bytes;
+        }
+
+        /**
+         * @brief Add an uint8_t to the output
+         *
+         * @param c  Character to add
+         * @return SerDes& This object
+         */
+        DataSerializer &operator<<(uint8_t c) {
+            return *this << int8_t(c);
+        }
+
+        /**
+         * @brief Add an int8_t to the output
+         *
+         * @param c  Character to add
+         * @return SerDes& This object
+         */
+        DataSerializer &operator<<(int8_t c) {
+            if (!extendBy(1, "int8_t")) {
+                return *this;
+            }
+
+            _store[_pos++] = c;
+            return *this;
+        }
+
+
+        ///////////////////////////////////////////////////////////
+
+        /**
+         * @brief Add an uint16_t to the output
+         *
+         * @param val  Value to add
+         * @return SerDes& This object
+         */
+        DataSerializer &operator<<(uint16_t val) {
+            return *this << int16_t(val);
+        }
+
+        /**
+         * @brief Add an int16_t to the output
+         *
+         * @param val  Value to add
+         * @return SerDes& This object
+         */
+        DataSerializer &operator<<(int16_t val) {
+            if (!extendBy(sizeof(val), "int16_t")) {
+                return *this;
+            }
+
+            memcpy(&_store[_pos], (char*)&val, sizeof(val));
+            _pos += sizeof(val);
+            return *this;
+        }
+
+
+
+        ///////////////////////////////////////////////////////////
+
+        /**
+         * @brief Add an uint32_t to the output
+         *
+         * @param val  Value to add
+         * @return SerDes& This object
+         */
+        DataSerializer &operator<<(uint32_t val) {
+            return *this << int32_t(val);
+        }
+
+        /**
+         * @brief Add an int32_t to the output
+         *
+         * @param val  Value to add
+         * @return SerDes& This object
+         */
+        DataSerializer &operator<<(int32_t val) {
+            if (!extendBy(sizeof(val), "int32_t")) {
+                return *this;
+            }
+
+            memcpy(&_store[_pos], (char*)&val, sizeof(val));
+            _pos += sizeof(val);
+            return *this;
+        }
+
+        ///////////////////////////////////////////////////////////
+
+        /**
+         * @brief Add an uint64_t to the output
+         *
+         * @param val  Value to add
+         * @return SerDes& This object
+         */
+        DataSerializer &operator<<(uint64_t val) {
+            return *this << int64_t(val);
+        }
+
+        /**
+         * @brief Add an int64_t to the output
+         *
+         * @param val  Value to add
+         * @return SerDes& This object
+         */
+        DataSerializer &operator<<(int64_t val) {
+            if (!extendBy(sizeof(val), "int64_t")) {
+                return *this;
+            }
+
+            memcpy(&_store[_pos], (char*)&val, sizeof(val));
+            _pos += sizeof(val);
+            return *this;
+        }
+
+        ///////////////////////////////////////////////////////////
+
+        /**
+         * @brief Add an float to the output
+         *
+         * @param val  Value to add
+         * @return SerDes& This object
+         */
+        DataSerializer &operator<<(float val) {
+            if (extendBy(sizeof(val), "float")) {
+                memcpy(&_store[_pos], (char*)&val, sizeof(val));
+                _pos += sizeof(val);
+            }
+            return *this;
+        }
+
+
+        ///////////////////////////////////////////////////////////
+
+
+        /**
+         * @brief Add an glm::vec3 to the output
+         *
+         * @param val  Value to add
+         * @return SerDes& This object
+         */
+        DataSerializer &operator<<(glm::vec3 val) {
+            size_t sz = sizeof(val.x);
+            if (extendBy(sz*3, "glm::vec3")) {
+                memcpy(&_store[_pos       ], (char*)&val.x, sz);
+                memcpy(&_store[_pos + sz  ], (char*)&val.y, sz);
+                memcpy(&_store[_pos + sz*2], (char*)&val.z, sz);
+
+                _pos += sz*3;
+            }
+            return *this;
+        }
+
+        ///////////////////////////////////////////////////////////
+
+        /**
+         * @brief Add a glm::vec4 to the output
+         *
+         * @param val  Value to add
+         * @return SerDes& This object
+         */
+        DataSerializer &operator<<(glm::vec4 val) {
+            size_t sz = sizeof(val.x);
+            if (extendBy(sz*4, "glm::vec4")) {
+                memcpy(&_store[_pos       ], (char*)&val.x, sz);
+                memcpy(&_store[_pos + sz  ], (char*)&val.y, sz);
+                memcpy(&_store[_pos + sz*2], (char*)&val.z, sz);
+                memcpy(&_store[_pos + sz*3], (char*)&val.w, sz);
+
+                _pos += sz*4;
+            }
+            return *this;
+        }
+
+        ///////////////////////////////////////////////////////////
+
+        /**
+         * @brief Add a glm::ivec2 to the output
+         *
+         * @param val  Value to add
+         * @return SerDes& This object
+         */
+        DataSerializer &operator<<(glm::ivec2 val) {
+            size_t sz = sizeof(val.x);
+            if (extendBy(sz*2, "glm::ivec2")) {
+                memcpy(&_store[_pos       ], (char*)&val.x, sz);
+                memcpy(&_store[_pos + sz  ], (char*)&val.y, sz);
+
+                _pos += sz*2;
+            }
+            return *this;
+        }
+
+
+        ///////////////////////////////////////////////////////////
+
+        /**
+         * @brief Write a null-terminated string into the buffer
+         *
+         * The `\0` at the end of the string is also written.
+         *
+         * @param val Value to write
+         * @return SerDes& This object
+         */
+        DataSerializer &operator<<(const char *val) {
+            size_t len = strlen(val)+1;
+            if (extendBy(len, "string")) {
+                memcpy(&_store[_pos], val, len);
+                _pos += len;
+            }
+            return *this;
+        }
+
+        /**
+         * @brief Write a QString into the buffer
+         *
+         * The string is encoded in UTF-8 and the `\0` at the end of the string is also written.
+         *
+         * @param val Value to write
+         * @return SerDes& This object
+         */
+        DataSerializer &operator<<(const QString &val) {
+            return *this << val.toUtf8().constData();
+        }
+
+
+        ///////////////////////////////////////////////////////////
+
+        /**
+         * @brief Pointer to the start of the internal buffer.
+         *
+         * The allocated amount can be found with capacity().
+         *
+         * The end of the stored data can be found with length().
+         *
+         * @return Pointer to buffer
+         */
+        char *buffer() const { return _store; }
+
+        /**
+         * @brief Current position in the buffer. Starts at 0.
+         *
+         * @return size_t
+         */
+        size_t pos() const { return _pos; }
+
+        /**
+         * @brief Last position that was written to in the buffer. Starts at 0.
+         *
+         * @return size_t
+         */
+        size_t length() const { return _length; }
+
+        /**
+         * @brief Current capacity of the buffer.
+         *
+         * If the buffer is dynamically allocated, it can grow.
+         *
+         * If the buffer is static, this is a fixed limit.
+         *
+         * @return size_t
+         */
+        size_t capacity() const { return _capacity; }
+
+        /**
+         * @brief Whether there's any data in the buffer
+         *
+         * @return true Something has been written
+         * @return false The buffer is empty
+         */
+        bool isEmpty() const { return _length == 0; }
+
+        /**
+         * @brief The buffer size limit has been reached
+         *
+         * This can only return true for a statically allocated buffer.
+         *
+         * @return true Limit reached
+         * @return false There is still room
+         */
+        bool isOverflow() const { return _overflow; }
+
+        /**
+         * @brief Reset the serializer to the start, clear overflow bit.
+         *
+         */
+        void rewind() { _pos = 0; _overflow = false; _lastAdvance = 0; }
+
+
+        /**
+         * @brief Size of the last advance
+         *
+         * This can be used to get how many bytes were added in the last operation.
+         * It is reset on rewind()
+         *
+         * @return size_t
+         */
+        size_t lastAdvance() const { return _lastAdvance; }
+
+        /**
+         * @brief Dump the contents of this object into QDebug
+         *
+         * This produces a dump of the internal state, and an ASCII/hex dump of
+         * the contents, for debugging.
+         *
+         * @param debug Qt QDebug stream
+         * @param ds  This object
+         * @return QDebug
+         */
+        friend QDebug operator<<(QDebug debug, const DataSerializer &ds);
+
+    private:
+        bool extendBy(size_t bytes, const QString &type_name) {
+            //qDebug() << "Extend by" << bytes;
+
+            if ( _capacity < _length + bytes) {
+                if ( _storeIsExternal ) {
+                    qCritical() << "Serializer trying to write past end of output buffer of" << _capacity << "bytes. Error writing" << bytes << "bytes for" << type_name << " from position " << _pos << ", length " << _length;
+                    _overflow = true;
+                    return false;
+                }
+
+                changeAllocation(_length + bytes);
+            }
+
+            _length += bytes;
+            _lastAdvance = bytes;
+            return true;
+        }
+
+        // This is split up here to try to make the class as inline-friendly as possible.
+        void changeAllocation(size_t new_size);
+
+        char *_store;
+        bool _storeIsExternal = false;
+        bool _overflow = false;
+        size_t _capacity = 0;
+        size_t _length = 0;
+        size_t _pos = 0;
+        size_t _lastAdvance = 0;
+};
+
+/**
+ * @brief Data deserializer
+ *
+ * This class operates on a fixed size buffer. If an attempt to read past the end is made, the read fails,
+ * and the overflow flag is set.
+ *
+ * The class was written for the maximum simplicity possible and inline friendliness.
+ *
+ * Example of decoding:
+ *
+ * @code {.cpp}
+ * // Incoming data has been placed in:
+ * // char buffer[1024];
+ *
+ * uint8_t version;
+ * uint16_t count;
+ * glm::vec3 pos;
+ *
+ * DataDeserializer des(buffer, sizeof(buffer));
+ * des >> version;
+ * des >> count;
+ * des >> pos;
+ * @endcode
+ *
+ * This object should be modified directly to add support for any primitive and common datatypes in the code. To support deserializing
+ * classes and structures, implement an `operator>>` function for that object, eg:
+ *
+ * @code {.cpp}
+ * DataDeserializer &operator>>(DataDeserializer &des, Object &o) {
+ *  des >> o._borderColor;
+ *  des >> o._maxAnisotropy;
+ *  des >> o._filter;
+ *  return des;
+ * }
+ * @endcode
+ *
+ */
+class DataDeserializer {
+    public:
+
+        /**
+         * @brief RAII tracker of advance position
+         *
+         * When a custom operator>> is implemented for DataDeserializer,
+         * this class allows to easily keep track of how much data has been added and
+         * adjust the parent's lastAdvance() count on this class' destruction.
+         *
+         * @code {.cpp}
+         * DataDeserializer &operator>>(Deserializer &des, Object &o) {
+         *  DataDeserializer::SizeTracker tracker(des);
+         *
+         *  des >> o._borderColor;
+         *  des >> o._maxAnisotropy;
+         *  des >> o._filter;
+         *  return des;
+         * }
+         * @endcode
+         */
+        class SizeTracker {
+            public:
+            SizeTracker(DataDeserializer &parent) : _parent(parent) {
+                _start_pos = _parent.pos();
+            }
+
+            ~SizeTracker() {
+                size_t cur_pos = _parent.pos();
+
+                if ( cur_pos >= _start_pos ) {
+                    _parent._lastAdvance = cur_pos - _start_pos;
+                } else {
+                    _parent._lastAdvance = 0;
+                }
+            }
+
+            private:
+            DataDeserializer &_parent;
+            size_t _start_pos = 0;
+        };
+
+        /**
+         * @brief Construct a Deserializer
+         *         *
+         * @param externalStore External data store
+         * @param storeLength Length of the data store
+         */
+        DataDeserializer(const char *externalStore, size_t storeLength) {
+            _length = storeLength;
+            _pos = 0;
+            _store = externalStore;
+            _lastAdvance = 0;
+        }
+
+        /**
+         * @brief Construct a Deserializer
+         *
+         * @param externalStore External data store
+         * @param storeLength Length of the data store
+         */
+        DataDeserializer(const uint8_t *externalStore, size_t storeLength) : DataDeserializer((const char*)externalStore, storeLength) {
+
+        }
+
+        /**
+         * @brief Construct a new Deserializer reading data from a Serializer
+         *
+         * This is a convenience function for testing.
+         *
+         * @param serializer Serializer with data
+         */
+        DataDeserializer(const DataSerializer &serializer) : DataDeserializer(serializer.buffer(), serializer.length()) {
+
+        }
+
+        /**
+         * @brief Skips padding in the input
+         *
+         * @param bytes Number of bytes to skip
+         */
+        void skipPadding(size_t bytes) {
+            if (!canAdvanceBy(bytes, "padding")) {
+                return;
+            }
+
+            _pos += bytes;
+            _lastAdvance = bytes;
+        }
+
+
+        /**
+         * @brief Read an uint8_t from the buffer
+         *
+         * @param c Character to read
+         * @return SerDes& This object
+         */
+        DataDeserializer &operator>>(uint8_t &c) {
+            return *this >> reinterpret_cast<int8_t&>(c);
+        }
+
+        /**
+         * @brief Read an int8_t from the buffer
+         *
+         * @param c Character to read
+         * @return SerDes& This object
+         */
+        DataDeserializer &operator>>(int8_t &c) {
+            if ( canAdvanceBy(1, "int8_t") ) {
+                c = _store[_pos++];
+                _lastAdvance = sizeof(c);
+            }
+
+            return *this;
+        }
+
+        ///////////////////////////////////////////////////////////
+
+        /**
+         * @brief Read an uint16_t from the buffer
+         *
+         * @param val Value to read
+         * @return SerDes& This object
+         */
+        DataDeserializer &operator>>(uint16_t &val) {
+            return *this >> reinterpret_cast<int16_t&>(val);
+        }
+
+        /**
+         * @brief Read an int16_t from the buffer
+         *
+         * @param val Value to read
+         * @return SerDes& This object
+         */
+        DataDeserializer &operator>>(int16_t &val) {
+            if ( canAdvanceBy(sizeof(val), "int16_t") ) {
+                memcpy((char*)&val, &_store[_pos], sizeof(val));
+                _pos += sizeof(val);
+                _lastAdvance = sizeof(val);
+            }
+
+            return *this;
+        }
+
+        ///////////////////////////////////////////////////////////
+
+        /**
+         * @brief Read an uint32_t from the buffer
+         *
+         * @param val Value to read
+         * @return SerDes& This object
+         */
+        DataDeserializer &operator>>(uint32_t &val) {
+            return *this >> reinterpret_cast<int32_t&>(val);
+        }
+
+        /**
+         * @brief Read an int32_t from the buffer
+         *
+         * @param val Value to read
+         * @return SerDes& This object
+         */
+        DataDeserializer &operator>>(int32_t &val) {
+            if ( canAdvanceBy(sizeof(val), "int32_t") ) {
+                memcpy((char*)&val, &_store[_pos], sizeof(val));
+                _pos += sizeof(val);
+                _lastAdvance = sizeof(val);
+            }
+            return *this;
+        }
+
+        ///////////////////////////////////////////////////////////
+
+        /**
+         * @brief Read an uint64_t from the buffer
+         *
+         * @param val Value to read
+         * @return SerDes& This object
+         */
+        DataDeserializer &operator>>(uint64_t &val) {
+            return *this >> reinterpret_cast<int64_t&>(val);
+        }
+
+        /**
+         * @brief Read an int64_t from the buffer
+         *
+         * @param val Value to read
+         * @return SerDes& This object
+         */
+        DataDeserializer &operator>>(int64_t &val) {
+            if ( canAdvanceBy(sizeof(val), "int64_t") ) {
+                memcpy((char*)&val, &_store[_pos], sizeof(val));
+                _pos += sizeof(val);
+                _lastAdvance = sizeof(val);
+            }
+            return *this;
+        }
+
+        ///////////////////////////////////////////////////////////
+
+
+        /**
+         * @brief Read an float from the buffer
+         *
+         * @param val Value to read
+         * @return SerDes& This object
+         */
+        DataDeserializer &operator>>(float &val) {
+            if ( canAdvanceBy(sizeof(val), "float") ) {
+                memcpy((char*)&val, &_store[_pos], sizeof(val));
+                _pos += sizeof(val);
+                _lastAdvance = sizeof(val);
+            }
+            return *this;
+        }
+
+        ///////////////////////////////////////////////////////////
+
+
+
+
+        /**
+         * @brief Read a glm::vec3 from the buffer
+         *
+         * @param val Value to read
+         * @return SerDes& This object
+         */
+        DataDeserializer &operator>>(glm::vec3 &val) {
+            size_t sz = sizeof(val.x);
+
+            if ( canAdvanceBy(sz*3, "glm::vec3") ) {
+                memcpy((char*)&val.x, &_store[_pos       ], sz);
+                memcpy((char*)&val.y, &_store[_pos + sz  ], sz);
+                memcpy((char*)&val.z, &_store[_pos + sz*2], sz);
+
+                _pos += sz*3;
+                _lastAdvance = sz * 3;
+            }
+
+            return *this;
+        }
+
+        ///////////////////////////////////////////////////////////
+
+
+        /**
+         * @brief Read a glm::vec4 from the buffer
+         *
+         * @param val Value to read
+         * @return SerDes& This object
+         */
+        DataDeserializer &operator>>(glm::vec4 &val) {
+            size_t sz = sizeof(val.x);
+
+            if ( canAdvanceBy(sz*4, "glm::vec4")) {
+                memcpy((char*)&val.x, &_store[_pos       ], sz);
+                memcpy((char*)&val.y, &_store[_pos + sz  ], sz);
+                memcpy((char*)&val.z, &_store[_pos + sz*2], sz);
+                memcpy((char*)&val.w, &_store[_pos + sz*3], sz);
+
+                _pos += sz*4;
+                _lastAdvance = sz*4;
+            }
+            return *this;
+        }
+
+        ///////////////////////////////////////////////////////////
+
+
+        /**
+         * @brief Read a glm::ivec2 from the buffer
+         *
+         * @param val Value to read
+         * @return SerDes& This object
+         */
+        DataDeserializer &operator>>(glm::ivec2 &val) {
+            size_t sz = sizeof(val.x);
+
+            if ( canAdvanceBy(sz*2, "glm::ivec2") ) {
+                memcpy((char*)&val.x, &_store[_pos       ], sz);
+                memcpy((char*)&val.y, &_store[_pos + sz  ], sz);
+
+                _pos += sz*2;
+                _lastAdvance = sz * 2;
+            }
+
+            return *this;
+        }
+
+        ///////////////////////////////////////////////////////////
+
+        /**
+         * @brief Pointer to the start of the internal buffer.
+         *
+         * The allocated amount can be found with capacity().
+         *
+         * The end of the stored data can be found with length().
+         *
+         * @return Pointer to buffer
+         */
+        const char *buffer() const { return _store; }
+
+        /**
+         * @brief Current position in the buffer. Starts at 0.
+         *
+         * @return size_t
+         */
+        size_t pos() const { return _pos; }
+
+        /**
+         * @brief Last position that was written to in the buffer. Starts at 0.
+         *
+         * @return size_t
+         */
+        size_t length() const { return _length; }
+
+        /**
+         * @brief Whether there's any data in the buffer
+         *
+         * @return true Something has been written
+         * @return false The buffer is empty
+         */
+        bool isEmpty() const { return _length == 0; }
+
+        /**
+         * @brief The buffer size limit has been reached
+         *
+         * This can only return true for a statically allocated buffer.
+         *
+         * @return true Limit reached
+         * @return false There is still room
+         */
+        bool isOverflow() const { return _overflow; }
+
+        /**
+         * @brief Reset the serializer to the start, clear overflow bit.
+         *
+         */
+        void rewind() { _pos = 0; _overflow = false; _lastAdvance = 0; }
+
+        /**
+         * @brief Size of the last advance
+         *
+         * This can be used to get how many bytes were added in the last operation.
+         * It is reset on rewind()
+         *
+         * @return size_t
+         */
+        size_t lastAdvance() const { return _lastAdvance; }
+
+        /**
+         * @brief Dump the contents of this object into QDebug
+         *
+         * This produces a dump of the internal state, and an ASCII/hex dump of
+         * the contents, for debugging.
+         *
+         * @param debug Qt QDebug stream
+         * @param ds  This object
+         * @return QDebug
+         */
+        friend QDebug operator<<(QDebug debug, const DataDeserializer &ds);
+
+    private:
+        bool canAdvanceBy(size_t bytes, const QString &type_name) {
+            //qDebug() << "Checking advance by" << bytes;
+
+            if ( _length < _pos + bytes) {
+                qCritical() << "Deserializer trying to read past end of input buffer of" << _length << "bytes, reading" << bytes << "bytes for" << type_name << "from position " << _pos;
+                _overflow = true;
+                return false;
+            }
+
+            return true;
+        }
+
+        const char *_store;
+        bool _overflow = false;
+        size_t _length = 0;
+        size_t _pos = 0;
+        size_t _lastAdvance = 0;
+};

tests/ktx/src/KtxTests.cpp

@@ -16,6 +16,7 @@
 #include <gpu/Texture.h>
 #include <image/Image.h>
 #include <image/TextureProcessing.h>
+#include "SerDes.h"
 
 
 QTEST_GUILESS_MAIN(KtxTests)
@@ -31,6 +32,19 @@ QString getRootPath() {
     return result;
 }
 
+#if 0
+ktx::Byte* serializeSPH(ktx::Byte* data, const gpu::IrradianceKTXPayload &payload) const {
+    *(ktx::IrradianceKTXPayload::Version*)data = IrradianceKTXPayload::CURRENT_VERSION;
+    data += sizeof(ktx::IrradianceKTXPayload::Version);
+
+    memcpy(data, &payload._irradianceSH, sizeof(ktx::SphericalHarmonics));
+    data += sizeof(SphericalHarmonics);
+
+    return data + PADDING;
+}
+#endif
+
+
 void KtxTests::initTestCase() {
 }
 
@@ -147,6 +161,14 @@ void KtxTests::testKtxSerialization() {
     testTexture->setKtxBacking(TEST_IMAGE_KTX.fileName().toStdString());
 }
 
+
+void KtxTests::testKtxNewSerializationSphericalHarmonics() {
+    DataSerializer ser;
+
+
+}
+
+
 #if 0
 
 static const QString TEST_FOLDER { "H:/ktx_cacheold" };

tests/ktx/src/KtxTests.h

@@ -16,6 +16,7 @@ private slots:
     void testKtxEvalFunctions();
     void testKhronosCompressionFunctions();
     void testKtxSerialization();
+    void testKtxNewSerializationSphericalHarmonics();
 };
 
 

tests/shared/src/SerializerTests.cpp

@@ -0,0 +1,232 @@
+//
+//  SerializerTests.cpp
+//
+//  Copyright 2022 Dale Glass
+//
+//  Distributed under the Apache License, Version 2.0.
+//  See the accompanying file LICENSE or http://www.apache.org/licenses/LICENSE-2.0.html
+//
+
+
+#include "SerializerTests.h"
+#include <SerDes.h>
+#include <QDebug>
+#include <glm/glm.hpp>
+
+QTEST_GUILESS_MAIN(SerializerTests)
+
+
+void SerializerTests::initTestCase() {
+}
+
+void SerializerTests::testCreate() {
+    DataSerializer s;
+    QCOMPARE(s.length(), 0);
+    QCOMPARE(s.capacity(), DataSerializer::DEFAULT_SIZE);
+    QCOMPARE(s.isEmpty(), true);
+
+
+    DataDeserializer d(s);
+    QCOMPARE(d.length(), 0);
+}
+
+void SerializerTests::testAdd() {
+    DataSerializer s;
+    s << (qint8)1;
+    QCOMPARE(s.length(), 1);
+    QCOMPARE(s.isEmpty(), false);
+
+    s << (quint8)-1;
+    QCOMPARE(s.length(), 2);
+
+    s << (qint16)0xaabb;
+    QCOMPARE(s.length(), 4);
+
+    s << (quint16)-18000;
+    QCOMPARE(s.length(), 6);
+
+    s << (qint32)0xCCDDEEFF;
+    QCOMPARE(s.length(), 10);
+
+    s << (quint32)-1818000000;
+    QCOMPARE(s.length(), 14);
+
+    s << "Hello, world!";
+    QCOMPARE(s.length(), 28);
+
+    glm::vec3 v{1.f,2.f,3.f};
+    s << v;
+    QCOMPARE(s.length(), 40);
+
+    s << 1.2345f;
+    QCOMPARE(s.length(), 44);
+
+
+    qDebug() << s;
+}
+
+void SerializerTests::testAddAndRead() {
+    DataSerializer s;
+    glm::vec3 v3_a{1.f, 3.1415f, 2.71828f};
+    glm::vec3 v3_b;
+    glm::vec4 v4_a{3.1415f, 2.71828f, 1.4142f, 1.6180f};
+    glm::vec4 v4_b;
+    glm::ivec2 iv2_a{10, 24};
+    glm::ivec2 iv2_b;
+    float f_a = 1.2345f;
+    float f_b;
+
+    s << (qint8)1;
+    s << (qint16)0xaabb;
+    s << (qint32)0xccddeeff;
+    s << v3_a;
+    s << v4_a;
+    s << iv2_a;
+    s << f_a;
+
+    qint8 i8;
+    qint16 i16;
+    qint32 i32;
+
+    DataDeserializer d(s);
+
+    d >> i8;
+    d >> i16;
+    d >> i32;
+    d >> v3_b;
+    d >> v4_b;
+    d >> iv2_b;
+    d >> f_b;
+
+    qDebug() << d;
+
+    QCOMPARE(i8, (qint8)1);
+    QCOMPARE(i16, (qint16)0xaabb);
+    QCOMPARE(i32, (qint32)0xccddeeff);
+    QCOMPARE(v3_a, v3_b);
+    QCOMPARE(v4_a, v4_b);
+    QCOMPARE(iv2_a, iv2_b);
+    QCOMPARE(f_a, f_b);
+}
+
+void SerializerTests::testReadPastEnd() {
+    DataSerializer s;
+    qint8 i8;
+    qint16 i16;
+    s << (qint8)1;
+
+    DataDeserializer d(s);
+    d >> i8;
+    QCOMPARE(d.pos(), 1);
+
+    d.rewind();
+    d >> i16;
+    QCOMPARE(d.pos(), 0);
+}
+
+void SerializerTests::testWritePastEnd() {
+    qint8 i8 = 255;
+    qint16 i16 = 65535;
+
+
+    char buf[16];
+
+
+    // 1 byte buffer, we can write 1 byte
+    memset(buf, 0, sizeof(buf));
+    DataSerializer s1(buf, 1);
+    s1 << i8;
+    QCOMPARE(s1.pos(), 1);
+    QCOMPARE(s1.isOverflow(), false);
+    QCOMPARE(buf[0], i8);
+
+    // 1 byte buffer, we can't write 2 bytes
+    memset(buf, 0, sizeof(buf));
+    DataSerializer s2(buf, 1);
+    s2 << i16;
+    QCOMPARE(s2.pos(), 0);
+    QCOMPARE(s2.isOverflow(), true);
+    QCOMPARE(buf[0], 0); // We didn't write
+    QCOMPARE(buf[1], 0);
+}
+
+
+
+
+void SerializerTests::benchmarkEncodingDynamicAlloc() {
+    QBENCHMARK {
+        DataSerializer s;
+        glm::vec3 v3_a{1.f, 3.1415f, 2.71828f};
+        glm::vec3 v3_b;
+        glm::vec4 v4_a{3.1415f, 2.71828f, 1.4142f, 1.6180f};
+        glm::vec4 v4_b;
+        glm::ivec2 iv2_a{10, 24};
+        glm::ivec2 iv2_b;
+
+        s << (qint8)1;
+        s << (qint16)0xaabb;
+        s << (qint32)0xccddeeff;
+        s << v3_a;
+        s << v4_a;
+        s << iv2_a;
+    }
+}
+
+void SerializerTests::benchmarkEncodingStaticAlloc() {
+    char buf[1024];
+
+    QBENCHMARK {
+        DataSerializer s(buf, sizeof(buf));
+        glm::vec3 v3_a{1.f, 3.1415f, 2.71828f};
+        glm::vec3 v3_b;
+        glm::vec4 v4_a{3.1415f, 2.71828f, 1.4142f, 1.6180f};
+        glm::vec4 v4_b;
+        glm::ivec2 iv2_a{10, 24};
+        glm::ivec2 iv2_b;
+
+        s << (qint8)1;
+        s << (qint16)0xaabb;
+        s << (qint32)0xccddeeff;
+        s << v3_a;
+        s << v4_a;
+        s << iv2_a;
+    }
+}
+
+
+void SerializerTests::benchmarkDecoding() {
+    DataSerializer s;
+    qint8 q8 = 1;
+    qint16 q16 = 0xaabb;
+    qint32 q32 = 0xccddeeff;
+
+    glm::vec3 v3_a{1.f, 3.1415f, 2.71828f};
+    glm::vec3 v3_b;
+    glm::vec4 v4_a{3.1415f, 2.71828f, 1.4142f, 1.6180f};
+    glm::vec4 v4_b;
+    glm::ivec2 iv2_a{10, 24};
+    glm::ivec2 iv2_b;
+
+    s << q8;
+    s << q16;
+    s << q32;
+    s << v3_a;
+    s << v4_a;
+    s << iv2_a;
+
+
+    QBENCHMARK {
+        DataDeserializer d(s);
+        d >> q8;
+        d >> q16;
+        d >> q32;
+        d >> v3_a;
+        d >> v4_a;
+        d >> iv2_a;
+    }
+}
+
+
+void SerializerTests::cleanupTestCase() {
+}
+

tests/shared/src/SerializerTests.h

@@ -0,0 +1,33 @@
+//
+//  ResourceTests.h
+//
+//  Copyright 2015 High Fidelity, Inc.
+//
+//  Distributed under the Apache License, Version 2.0.
+//  See the accompanying file LICENSE or http://www.apache.org/licenses/LICENSE-2.0.html
+//
+
+#ifndef overte_SerializerTests_h
+#define overte_SerializerTests_h
+
+#include <QtTest/QtTest>
+#include <QtCore/QTemporaryDir>
+
+class SerializerTests : public QObject {
+    Q_OBJECT
+private slots:
+    void initTestCase();
+    void testCreate();
+    void testAdd();
+    void testAddAndRead();
+    void testReadPastEnd();
+    void testWritePastEnd();
+    void benchmarkEncodingDynamicAlloc();
+    void benchmarkEncodingStaticAlloc();
+    void benchmarkDecoding();
+    void cleanupTestCase();
+private:
+
+};
+
+#endif // overte_SerializerTests_h