Mirrors/overte-AleziaKurdis
3
0
Fork 0
mirror of https://github.com/AleziaKurdis/overte.git synced 2025-04-05 21:12:40 +02:00
Code Issues Projects Releases Packages Wiki Activity Actions

Merge pull request #1293 from HifiExperiments/implement-serializer
Some checks failed
Master API-docs CI Build and Deploy / Build and deploy API-docs (push) Has been cancelled
Details
Master Doxygen CI Build and Deploy / Build and deploy Doxygen documentation (push) Has been cancelled
Details

Browse source 
Implement network serializer (rebased)
This commit is contained in:
HifiExperiments 2025-01-13 16:45:25 -08:00 committed by GitHub
commit a7805e7592
No known key found for this signature in database
GPG key ID: B5690EEEBB952194
11 changed files with 1507 additions and 60 deletions
Show stats Download patch file Download diff file Expand all files Collapse all files

61
libraries/gpu/src/gpu/Texture.h
View file

@ -24,6 +24,7 @@
#include "Forward.h"
#include "Resource.h"
#include "Metric.h"
#include "SerDes.h"
const int ABSOLUTE_MAX_TEXTURE_NUM_PIXELS = 8192 * 8192;
@ -91,6 +92,37 @@ public:
};
typedef std::shared_ptr< SphericalHarmonics > SHPointer;
inline DataSerializer &operator<<(DataSerializer &ser, const SphericalHarmonics &h) {
DataSerializer::SizeTracker tracker(ser);
ser << h.L00 << h.spare0;
ser << h.L1m1 << h.spare1;
ser << h.L10 << h.spare2;
ser << h.L11 << h.spare3;
ser << h.L2m2 << h.spare4;
ser << h.L2m1 << h.spare5;
ser << h.L20 << h.spare6;
ser << h.L21 << h.spare7;
ser << h.L22 << h.spare8;
return ser;
}
inline DataDeserializer &operator>>(DataDeserializer &des, SphericalHarmonics &h) {
DataDeserializer::SizeTracker tracker(des);
des >> h.L00 >> h.spare0;
des >> h.L1m1 >> h.spare1;
des >> h.L10 >> h.spare2;
des >> h.L11 >> h.spare3;
des >> h.L2m2 >> h.spare4;
des >> h.L2m1 >> h.spare5;
des >> h.L20 >> h.spare6;
des >> h.L21 >> h.spare7;
des >> h.L22 >> h.spare8;
return des;
}
class Sampler {
public:
@ -193,6 +225,35 @@ protected:
friend class Deserializer;
};
inline DataSerializer &operator<<(DataSerializer &ser, const Sampler::Desc &d) {
DataSerializer::SizeTracker tracker(ser);
ser << d._borderColor;
ser << d._maxAnisotropy;
ser << d._filter;
ser << d._comparisonFunc;
ser << d._wrapModeU;
ser << d._wrapModeV;
ser << d._wrapModeW;
ser << d._mipOffset;
ser << d._minMip;
ser << d._maxMip;
return ser;
}
inline DataDeserializer &operator>>(DataDeserializer &dsr, Sampler::Desc &d) {
DataDeserializer::SizeTracker tracker(dsr);
dsr >> d._borderColor;
dsr >> d._maxAnisotropy;
dsr >> d._filter;
dsr >> d._comparisonFunc;
dsr >> d._wrapModeU;
dsr >> d._wrapModeV;
dsr >> d._wrapModeW;
dsr >> d._mipOffset;
dsr >> d._minMip;
dsr >> d._maxMip;
return dsr;
}
enum class TextureUsageType : uint8 {
RENDERBUFFER, // Used as attachments to a framebuffer
RESOURCE, // Resource textures, like materials... subject to memory manipulation

124
libraries/gpu/src/gpu/Texture_ktx.cpp
View file

@ -18,6 +18,7 @@
#include <ktx/KTX.h>
#include "GPULogging.h"
#include "SerDes.h"
using namespace gpu;
@ -27,71 +28,94 @@ using KtxStorage = Texture::KtxStorage;
std::vector<std::pair<std::shared_ptr<storage::FileStorage>, std::shared_ptr<std::mutex>>> KtxStorage::_cachedKtxFiles;
std::mutex KtxStorage::_cachedKtxFilesMutex;
/**
* @brief Payload for a KTX (texture)
*
* This contains a ready to use texture. This is both used for the local cache, and for baked textures.
*
* @note The usage for textures means breaking compatibility is a bad idea, and that the implementation
* should just keep on adding extra data at the bottom of the structure, and remain able to read old
* formats. In fact, version 1 KTX can be found in older baked assets.
*/
struct GPUKTXPayload {
using Version = uint8;
static const std::string KEY;
static const Version CURRENT_VERSION { 2 };
static const size_t PADDING { 2 };
static const size_t SIZE { sizeof(Version) + sizeof(Sampler::Desc) + sizeof(uint32) + sizeof(TextureUsageType) + sizeof(glm::ivec2) + PADDING };
static const size_t SIZE { sizeof(Version) + sizeof(Sampler::Desc) + sizeof(uint32_t) + sizeof(TextureUsageType) + sizeof(glm::ivec2) + PADDING };
static_assert(GPUKTXPayload::SIZE == 44, "Packing size may differ between platforms");
static_assert(GPUKTXPayload::SIZE % 4 == 0, "GPUKTXPayload is not 4 bytes aligned");
Sampler::Desc _samplerDesc;
Texture::Usage _usage;
TextureUsageType _usageType;
glm::ivec2 _originalSize { 0, 0 };
Byte* serialize(Byte* data) const {
*(Version*)data = CURRENT_VERSION;
data += sizeof(Version);
/**
* @brief Serialize the KTX payload
*
* @warning Be careful modifying this code, as it influences baked assets.
* Backwards compatibility must be maintained.
*
* @param ser Destination serializer
*/
void serialize(DataSerializer &ser) {
memcpy(data, &_samplerDesc, sizeof(Sampler::Desc));
data += sizeof(Sampler::Desc);
ser << CURRENT_VERSION;
// We can't copy the bitset in Texture::Usage in a crossplateform manner
// So serialize it manually
uint32 usageData = _usage._flags.to_ulong();
memcpy(data, &usageData, sizeof(uint32));
data += sizeof(uint32);
ser << _samplerDesc;
memcpy(data, &_usageType, sizeof(TextureUsageType));
data += sizeof(TextureUsageType);
uint32_t usageData = (uint32_t)_usage._flags.to_ulong();
ser << usageData;
ser << ((uint8_t)_usageType);
ser << _originalSize;
memcpy(data, glm::value_ptr(_originalSize), sizeof(glm::ivec2));
data += sizeof(glm::ivec2);
ser.addPadding(PADDING);
return data + PADDING;
assert(ser.length() == GPUKTXPayload::SIZE);
}
bool unserialize(const Byte* data, size_t size) {
Version version = *(const Version*)data;
data += sizeof(Version);
/**
* @brief Deserialize the KTX payload
*
* @warning Be careful modifying this code, as it influences baked assets.
* Backwards compatibility must be maintained.
*
* @param dsr Deserializer object
* @return true Successful
* @return false Version check failed
*/
bool unserialize(DataDeserializer &dsr) {
Version version = 0;
uint32_t usageData = 0;
uint8_t usagetype = 0;
dsr >> version;
if (version > CURRENT_VERSION) {
// If we try to load a version that we don't know how to parse,
// it will render incorrectly
qCWarning(gpulogging) << "KTX version" << version << "is newer than our own," << CURRENT_VERSION;
qCWarning(gpulogging) << dsr;
return false;
}
memcpy(&_samplerDesc, data, sizeof(Sampler::Desc));
data += sizeof(Sampler::Desc);
dsr >> _samplerDesc;
// We can't copy the bitset in Texture::Usage in a crossplateform manner
// So unserialize it manually
uint32 usageData;
memcpy(&usageData, data, sizeof(uint32));
_usage = Texture::Usage(usageData);
data += sizeof(uint32);
dsr >> usageData;
_usage = gpu::Texture::Usage(usageData);
memcpy(&_usageType, data, sizeof(TextureUsageType));
data += sizeof(TextureUsageType);
dsr >> usagetype;
_usageType = (TextureUsageType)usagetype;
if (version >= 2) {
memcpy(&_originalSize, data, sizeof(glm::ivec2));
data += sizeof(glm::ivec2);
dsr >> _originalSize;
}
dsr.skipPadding(PADDING);
return true;
}
@ -103,7 +127,8 @@ struct GPUKTXPayload {
auto found = std::find_if(keyValues.begin(), keyValues.end(), isGPUKTX);
if (found != keyValues.end()) {
auto value = found->_value;
return payload.unserialize(value.data(), value.size());
DataDeserializer dsr(value.data(), value.size());
return payload.unserialize(dsr);
}
return false;
}
@ -123,29 +148,24 @@ struct IrradianceKTXPayload {
SphericalHarmonics _irradianceSH;
Byte* serialize(Byte* data) const {
*(Version*)data = CURRENT_VERSION;
data += sizeof(Version);
memcpy(data, &_irradianceSH, sizeof(SphericalHarmonics));
data += sizeof(SphericalHarmonics);
return data + PADDING;
void serialize(DataSerializer &ser) const {
ser << CURRENT_VERSION;
ser << _irradianceSH;
ser.addPadding(PADDING);
}
bool unserialize(const Byte* data, size_t size) {
if (size != SIZE) {
bool unserialize(DataDeserializer &des) {
Version version;
if (des.length() != SIZE) {
return false;
}
Version version = *(const Version*)data;
des >> version;
if (version != CURRENT_VERSION) {
return false;
}
data += sizeof(Version);
memcpy(&_irradianceSH, data, sizeof(SphericalHarmonics));
des >> _irradianceSH;
return true;
}
@ -157,7 +177,8 @@ struct IrradianceKTXPayload {
auto found = std::find_if(keyValues.begin(), keyValues.end(), isIrradianceKTX);
if (found != keyValues.end()) {
auto value = found->_value;
return payload.unserialize(value.data(), value.size());
DataDeserializer des(value.data(), value.size());
return payload.unserialize(des);
}
return false;
}
@ -467,7 +488,9 @@ ktx::KTXUniquePointer Texture::serialize(const Texture& texture, const glm::ivec
gpuKeyval._originalSize = originalSize;
Byte keyvalPayload[GPUKTXPayload::SIZE];
gpuKeyval.serialize(keyvalPayload);
DataSerializer ser(keyvalPayload, sizeof(keyvalPayload));
gpuKeyval.serialize(ser);
ktx::KeyValues keyValues;
keyValues.emplace_back(GPUKTXPayload::KEY, (uint32)GPUKTXPayload::SIZE, (ktx::Byte*) &keyvalPayload);
@ -477,7 +500,8 @@ ktx::KTXUniquePointer Texture::serialize(const Texture& texture, const glm::ivec
irradianceKeyval._irradianceSH = *texture.getIrradiance();
Byte irradianceKeyvalPayload[IrradianceKTXPayload::SIZE];
irradianceKeyval.serialize(irradianceKeyvalPayload);
DataSerializer ser(irradianceKeyvalPayload, sizeof(irradianceKeyvalPayload));
irradianceKeyval.serialize(ser);
keyValues.emplace_back(IrradianceKTXPayload::KEY, (uint32)IrradianceKTXPayload::SIZE, (ktx::Byte*) &irradianceKeyvalPayload);
}

9
libraries/octree/src/OctreePacketData.cpp
View file

@ -17,6 +17,7 @@
#include "OctreeLogging.h"
#include "NumericalConstants.h"
#include <glm/gtc/type_ptr.hpp>
#include "SerDes.h"
bool OctreePacketData::_debug = false;
AtomicUIntStat OctreePacketData::_totalBytesOfOctalCodes { 0 };
@ -847,10 +848,10 @@ int OctreePacketData::unpackDataFromBytes(const unsigned char* dataBytes, QByteA
}
int OctreePacketData::unpackDataFromBytes(const unsigned char* dataBytes, AACube& result) {
aaCubeData cube;
memcpy(&cube, dataBytes, sizeof(aaCubeData));
result = AACube(cube.corner, cube.scale);
return sizeof(aaCubeData);
DataDeserializer des(dataBytes, sizeof(aaCubeData));
des >> result;
return des.length();
}
int OctreePacketData::unpackDataFromBytes(const unsigned char* dataBytes, QRect& result) {

16
libraries/shared/src/AACube.h
View file

@ -20,6 +20,7 @@
#include <QDebug>
#include "BoxBase.h"
#include "SerDes.h"
class AABox;
class Extents;
@ -80,6 +81,10 @@ private:
glm::vec3 _corner;
float _scale;
friend DataSerializer& operator<<(DataSerializer &ser, const AACube &cube);
friend DataDeserializer& operator>>(DataDeserializer &des, AACube &cube);
};
inline bool operator==(const AACube& a, const AACube& b) {
@ -99,5 +104,16 @@ inline QDebug operator<<(QDebug debug, const AACube& cube) {
return debug;
}
inline DataSerializer& operator<<(DataSerializer &ser, const AACube &cube) {
ser << cube._corner;
ser << cube._scale;
return ser;
}
inline DataDeserializer& operator>>(DataDeserializer &des, AACube &cube) {
des >> cube._corner;
des >> cube._scale;
return des;
}
#endif // hifi_AACube_h

19
libraries/shared/src/BlendshapeConstants.h
View file

@ -122,6 +122,25 @@ struct BlendshapeOffsetUnpacked {
float positionOffsetX, positionOffsetY, positionOffsetZ;
float normalOffsetX, normalOffsetY, normalOffsetZ;
float tangentOffsetX, tangentOffsetY, tangentOffsetZ;
/**
* @brief Set all components of all the offsets to zero
*
* @note glm::vec3 is not trivially copyable, so it's not correct to clear it with memset.
*/
void clear() {
positionOffsetX = 0.0f;
positionOffsetY = 0.0f;
positionOffsetZ = 0.0f;
normalOffsetX = 0.0f;
normalOffsetY = 0.0f;
normalOffsetZ = 0.0f;
tangentOffsetX = 0.0f;
tangentOffsetY = 0.0f;
tangentOffsetZ = 0.0f;
}
};
using BlendshapeOffset = BlendshapeOffsetPacked;

85
libraries/shared/src/SerDes.cpp Normal file
View file

@ -0,0 +1,85 @@
//
// SerDes.h
//
//
// Created by Dale Glass on 5/6/2022
// Copyright 2024 Overte e.V.
//
// Distributed under the Apache License, Version 2.0.
// See the accompanying file LICENSE or http://www.apache.org/licenses/LICENSE-2.0.html
//
#include <cctype>
#include "SerDes.h"
const int DataSerializer::DEFAULT_SIZE;
const char DataSerializer::PADDING_CHAR;
static void dumpHex(QDebug &debug, const char*buf, size_t len) {
QString literal;
QString hex;
for(size_t i=0;i<len;i++) {
char c = buf[i];
if (std::isalnum(c)) {
literal.append(c);
} else {
literal.append(".");
}
QString hnum = QString::number( static_cast<unsigned char>(c), 16 );
if ( hnum.length() == 1 ) {
hnum.prepend("0");
}
hex.append(hnum + " ");
if ( literal.length() == 16 || (i+1 == len) ) {
while( literal.length() < 16 ) {
literal.append(" ");
hex.append(" ");
}
debug << literal << " " << hex << "\n";
literal.clear();
hex.clear();
}
}
}
QDebug operator<<(QDebug debug, const DataSerializer &ser) {
debug << "{ capacity =" << ser.capacity() << "; length = " << ser.length() << "; pos = " << ser.pos() << "}";
debug << "\n";
dumpHex(debug, ser.buffer(), ser.length());
return debug;
}
QDebug operator<<(QDebug debug, const DataDeserializer &des) {
debug << "{ length = " << des.length() << "; pos = " << des.pos() << "}";
debug << "\n";
dumpHex(debug, des.buffer(), des.length());
return debug;
}
void DataSerializer::changeAllocation(size_t new_size) {
while ( _capacity < new_size) {
_capacity *= 2;
}
char *new_buf = new char[_capacity];
assert( *new_buf );
memcpy(new_buf, _store, _length);
char *prev_buf = _store;
_store = new_buf;
delete []prev_buf;
}

953
libraries/shared/src/SerDes.h Normal file
View file

@ -0,0 +1,953 @@
//
// SerDes.h
//
//
// Created by Dale Glass on 5/6/2022
// Copyright 2024 Overte e.V.
//
// Distributed under the Apache License, Version 2.0.
// See the accompanying file LICENSE or http://www.apache.org/licenses/LICENSE-2.0.html
//
#pragma once
#include <string>
#include <cstring>
#include <QtCore/QByteArray>
#include <QDebug>
#include <glm/glm.hpp>
/**
* @brief Data serializer
*
* When encoding, this class takes in data and encodes it into a buffer. No attempt is made to store version numbers, lengths,
* or any other metadata. It's entirely up to the user to use the class in such a way that the process can be
* correctly reversed if variable-length or optional fields are used.
*
* It can operate both on an internal, dynamically-allocated buffer, or an externally provided, fixed-size one.
* If an external store is used, the class will refuse to add data once capacity is reached and set the overflow flag.
* When decoding, this class operates on a fixed size buffer. If an attempt to read past the end is made, the read fails,
* and the overflow flag is set.
*
* The class was written for the maximum simplicity possible and inline friendliness.
*
* Example of encoding:
*
* @code {.cpp}
* uint8_t version = 1;
* uint16_t count = 1;
* glm::vec3 pos{1.5, 2.0, 9.0};
*
* Serializer ser;
* ser << version;
* ser << count;
* ser << pos;
*
* // Serialized data is in ser.buffer(), ser.length() long.
* @endcode
*
* This object should be modified directly to add support for any primitive and common datatypes in the code. To support serializing/deserializing
* classes and structures, implement a `operator<<` and `operator>>` functions for that object, eg:
*
* @code {.cpp}
* DataSerializer &operator<<(DataSerializer &ser, const Object &o) {
* ser << o._borderColor;
* ser << o._maxAnisotropy;
* ser << o._filter;
* return ser;
* }
* @endcode
*
*/
class DataSerializer {
public:
/**
* @brief RAII tracker of advance position
*
* When a custom operator<< is implemented for DataSserializer,
* this class allows to easily keep track of how much data has been added and
* adjust the parent's lastAdvance() count on this class' destruction.
*
* @code {.cpp}
* DataSerializer &operator<<(DataSerializer &ser, const Object &o) {
* DataSerializer::SizeTracker tracker(ser);
*
* ser << o._borderColor;
* ser << o._maxAnisotropy;
* ser << o._filter;
* return ser;
* }
* @endcode
*/
class SizeTracker {
public:
SizeTracker(DataSerializer &parent) : _parent(parent) {
_start_pos = _parent.pos();
}
~SizeTracker() {
size_t cur_pos = _parent.pos();
if ( cur_pos >= _start_pos ) {
_parent._lastAdvance = cur_pos - _start_pos;
} else {
_parent._lastAdvance = 0;
}
}
private:
DataSerializer &_parent;
size_t _start_pos = 0;
};
/**
* @brief Default size for a dynamically allocated buffer.
*
* Since this is mostly intended to be used for networking, we default to the largest probable MTU here.
*/
static const int DEFAULT_SIZE = 1500;
/**
* @brief Character to use for padding.
*
* Padding should be ignored, so it doesn't matter what we go with here, but it can be useful to set it
* to something that would be distinctive in a dump.
*/
static const char PADDING_CHAR = (char)0xAA;
/**
* @brief Construct a dynamically allocated serializer
*
* If constructed this way, an internal buffer will be dynamically allocated and grown as needed.
*
* The buffer is SerDes::DEFAULT_SIZE bytes by default, and doubles in size every time the limit is reached.
*/
DataSerializer() {
_capacity = DEFAULT_SIZE;
_pos = 0;
_length = 0;
_store = new char[_capacity];
}
/**
* @brief Construct a statically allocated serializer
*
* If constructed this way, the external buffer will be used to store data. The class will refuse to
* keep adding data if the maximum length is reached, write a critical message to the log, and set
* the overflow flag.
*
* The flag can be read with isOverflow()
*
* @param externalStore External data store
* @param storeLength Length of the data store
*/
DataSerializer(char *externalStore, size_t storeLength) {
_capacity = storeLength;
_length = 0;
_pos = 0;
_storeIsExternal = true;
_store = externalStore;
}
/**
* @brief Construct a statically allocated serializer
*
* If constructed this way, the external buffer will be used to store data. The class will refuse to
* keep adding data if the maximum length is reached, and set the overflow flag.
*
* The flag can be read with isOverflow()
*
* @param externalStore External data store
* @param storeLength Length of the data store
*/
DataSerializer(uint8_t *externalStore, size_t storeLength) : DataSerializer((char*)externalStore, storeLength) {
}
DataSerializer(const DataSerializer &) = delete;
DataSerializer &operator=(const DataSerializer &) = delete;
~DataSerializer() {
if (!_storeIsExternal) {
delete[] _store;
}
}
/**
* @brief Adds padding to the output
*
* The bytes will be set to SerDes::PADDING_CHAR, which is a constant in the source code.
* Since padding isn't supposed to be read, it can be any value and is intended to
* be set to something that can be easily recognized in a dump.
*
* @param bytes Number of bytes to add
*/
void addPadding(size_t bytes) {
if (!extendBy(bytes, "padding")) {
return;
}
// Fill padding with something recognizable. Will keep valgrind happier.
memset(&_store[_pos], PADDING_CHAR, bytes);
_pos += bytes;
}
/**
* @brief Add an uint8_t to the output
*
* @param c Character to add
* @return SerDes& This object
*/
DataSerializer &operator<<(uint8_t c) {
return *this << int8_t(c);
}
/**
* @brief Add an int8_t to the output
*
* @param c Character to add
* @return SerDes& This object
*/
DataSerializer &operator<<(int8_t c) {
if (!extendBy(1, "int8_t")) {
return *this;
}
_store[_pos++] = c;
return *this;
}
///////////////////////////////////////////////////////////
/**
* @brief Add an uint16_t to the output
*
* @param val Value to add
* @return SerDes& This object
*/
DataSerializer &operator<<(uint16_t val) {
return *this << int16_t(val);
}
/**
* @brief Add an int16_t to the output
*
* @param val Value to add
* @return SerDes& This object
*/
DataSerializer &operator<<(int16_t val) {
if (!extendBy(sizeof(val), "int16_t")) {
return *this;
}
memcpy(&_store[_pos], (char*)&val, sizeof(val));
_pos += sizeof(val);
return *this;
}
///////////////////////////////////////////////////////////
/**
* @brief Add an uint32_t to the output
*
* @param val Value to add
* @return SerDes& This object
*/
DataSerializer &operator<<(uint32_t val) {
return *this << int32_t(val);
}
/**
* @brief Add an int32_t to the output
*
* @param val Value to add
* @return SerDes& This object
*/
DataSerializer &operator<<(int32_t val) {
if (!extendBy(sizeof(val), "int32_t")) {
return *this;
}
memcpy(&_store[_pos], (char*)&val, sizeof(val));
_pos += sizeof(val);
return *this;
}
///////////////////////////////////////////////////////////
/**
* @brief Add an uint64_t to the output
*
* @param val Value to add
* @return SerDes& This object
*/
DataSerializer &operator<<(uint64_t val) {
return *this << int64_t(val);
}
/**
* @brief Add an int64_t to the output
*
* @param val Value to add
* @return SerDes& This object
*/
DataSerializer &operator<<(int64_t val) {
if (!extendBy(sizeof(val), "int64_t")) {
return *this;
}
memcpy(&_store[_pos], (char*)&val, sizeof(val));
_pos += sizeof(val);
return *this;
}
///////////////////////////////////////////////////////////
/**
* @brief Add an float to the output
*
* @param val Value to add
* @return SerDes& This object
*/
DataSerializer &operator<<(float val) {
if (extendBy(sizeof(val), "float")) {
memcpy(&_store[_pos], (char*)&val, sizeof(val));
_pos += sizeof(val);
}
return *this;
}
///////////////////////////////////////////////////////////
/**
* @brief Add an glm::vec3 to the output
*
* @param val Value to add
* @return SerDes& This object
*/
DataSerializer &operator<<(glm::vec3 val) {
size_t sz = sizeof(val.x);
if (extendBy(sz*3, "glm::vec3")) {
memcpy(&_store[_pos ], (char*)&val.x, sz);
memcpy(&_store[_pos + sz ], (char*)&val.y, sz);
memcpy(&_store[_pos + sz*2], (char*)&val.z, sz);
_pos += sz*3;
}
return *this;
}
///////////////////////////////////////////////////////////
/**
* @brief Add a glm::vec4 to the output
*
* @param val Value to add
* @return SerDes& This object
*/
DataSerializer &operator<<(glm::vec4 val) {
size_t sz = sizeof(val.x);
if (extendBy(sz*4, "glm::vec4")) {
memcpy(&_store[_pos ], (char*)&val.x, sz);
memcpy(&_store[_pos + sz ], (char*)&val.y, sz);
memcpy(&_store[_pos + sz*2], (char*)&val.z, sz);
memcpy(&_store[_pos + sz*3], (char*)&val.w, sz);
_pos += sz*4;
}
return *this;
}
///////////////////////////////////////////////////////////
/**
* @brief Add a glm::ivec2 to the output
*
* @param val Value to add
* @return SerDes& This object
*/
DataSerializer &operator<<(glm::ivec2 val) {
size_t sz = sizeof(val.x);
if (extendBy(sz*2, "glm::ivec2")) {
memcpy(&_store[_pos ], (char*)&val.x, sz);
memcpy(&_store[_pos + sz ], (char*)&val.y, sz);
_pos += sz*2;
}
return *this;
}
///////////////////////////////////////////////////////////
/**
* @brief Write a null-terminated string into the buffer
*
* The `\0` at the end of the string is also written.
*
* @param val Value to write
* @return SerDes& This object
*/
DataSerializer &operator<<(const char *val) {
size_t len = strlen(val)+1;
if (extendBy(len, "string")) {
memcpy(&_store[_pos], val, len);
_pos += len;
}
return *this;
}
/**
* @brief Write a QString into the buffer
*
* The string is encoded in UTF-8 and the `\0` at the end of the string is also written.
*
* @param val Value to write
* @return SerDes& This object
*/
DataSerializer &operator<<(const QString &val) {
return *this << val.toUtf8().constData();
}
///////////////////////////////////////////////////////////
/**
* @brief Pointer to the start of the internal buffer.
*
* The allocated amount can be found with capacity().
*
* The end of the stored data can be found with length().
*
* @return Pointer to buffer
*/
char *buffer() const { return _store; }
/**
* @brief Current position in the buffer. Starts at 0.
*
* @return size_t
*/
size_t pos() const { return _pos; }
/**
* @brief Last position that was written to in the buffer. Starts at 0.
*
* @return size_t
*/
size_t length() const { return _length; }
/**
* @brief Current capacity of the buffer.
*
* If the buffer is dynamically allocated, it can grow.
*
* If the buffer is static, this is a fixed limit.
*
* @return size_t
*/
size_t capacity() const { return _capacity; }
/**
* @brief Whether there's any data in the buffer
*
* @return true Something has been written
* @return false The buffer is empty
*/
bool isEmpty() const { return _length == 0; }
/**
* @brief The buffer size limit has been reached
*
* This can only return true for a statically allocated buffer.
*
* @return true Limit reached
* @return false There is still room
*/
bool isOverflow() const { return _overflow; }
/**
* @brief Reset the serializer to the start, clear overflow bit.
*
*/
void rewind() { _pos = 0; _overflow = false; _lastAdvance = 0; }
/**
* @brief Size of the last advance
*
* This can be used to get how many bytes were added in the last operation.
* It is reset on rewind()
*
* @return size_t
*/
size_t lastAdvance() const { return _lastAdvance; }
/**
* @brief Dump the contents of this object into QDebug
*
* This produces a dump of the internal state, and an ASCII/hex dump of
* the contents, for debugging.
*
* @param debug Qt QDebug stream
* @param ds This object
* @return QDebug
*/
friend QDebug operator<<(QDebug debug, const DataSerializer &ds);
private:
bool extendBy(size_t bytes, const QString &type_name) {
//qDebug() << "Extend by" << bytes;
if ( _capacity < _length + bytes) {
if ( _storeIsExternal ) {
qCritical() << "Serializer trying to write past end of output buffer of" << _capacity << "bytes. Error writing" << bytes << "bytes for" << type_name << " from position " << _pos << ", length " << _length;
_overflow = true;
return false;
}
changeAllocation(_length + bytes);
}
_length += bytes;
_lastAdvance = bytes;
return true;
}
// This is split up here to try to make the class as inline-friendly as possible.
void changeAllocation(size_t new_size);
char *_store;
bool _storeIsExternal = false;
bool _overflow = false;
size_t _capacity = 0;
size_t _length = 0;
size_t _pos = 0;
size_t _lastAdvance = 0;
};
/**
* @brief Data deserializer
*
* This class operates on a fixed size buffer. If an attempt to read past the end is made, the read fails,
* and the overflow flag is set.
*
* The class was written for the maximum simplicity possible and inline friendliness.
*
* Example of decoding:
*
* @code {.cpp}
* // Incoming data has been placed in:
* // char buffer[1024];
*
* uint8_t version;
* uint16_t count;
* glm::vec3 pos;
*
* DataDeserializer des(buffer, sizeof(buffer));
* des >> version;
* des >> count;
* des >> pos;
* @endcode
*
* This object should be modified directly to add support for any primitive and common datatypes in the code. To support deserializing
* classes and structures, implement an `operator>>` function for that object, eg:
*
* @code {.cpp}
* DataDeserializer &operator>>(DataDeserializer &des, Object &o) {
* des >> o._borderColor;
* des >> o._maxAnisotropy;
* des >> o._filter;
* return des;
* }
* @endcode
*
*/
class DataDeserializer {
public:
/**
* @brief RAII tracker of advance position
*
* When a custom operator>> is implemented for DataDeserializer,
* this class allows to easily keep track of how much data has been added and
* adjust the parent's lastAdvance() count on this class' destruction.
*
* @code {.cpp}
* DataDeserializer &operator>>(Deserializer &des, Object &o) {
* DataDeserializer::SizeTracker tracker(des);
*
* des >> o._borderColor;
* des >> o._maxAnisotropy;
* des >> o._filter;
* return des;
* }
* @endcode
*/
class SizeTracker {
public:
SizeTracker(DataDeserializer &parent) : _parent(parent) {
_start_pos = _parent.pos();
}
~SizeTracker() {
size_t cur_pos = _parent.pos();
if ( cur_pos >= _start_pos ) {
_parent._lastAdvance = cur_pos - _start_pos;
} else {
_parent._lastAdvance = 0;
}
}
private:
DataDeserializer &_parent;
size_t _start_pos = 0;
};
/**
* @brief Construct a Deserializer
* *
* @param externalStore External data store
* @param storeLength Length of the data store
*/
DataDeserializer(const char *externalStore, size_t storeLength) {
_length = storeLength;
_pos = 0;
_store = externalStore;
_lastAdvance = 0;
}
/**
* @brief Construct a Deserializer
*
* @param externalStore External data store
* @param storeLength Length of the data store
*/
DataDeserializer(const uint8_t *externalStore, size_t storeLength) : DataDeserializer((const char*)externalStore, storeLength) {
}
/**
* @brief Construct a new Deserializer reading data from a Serializer
*
* This is a convenience function for testing.
*
* @param serializer Serializer with data
*/
DataDeserializer(const DataSerializer &serializer) : DataDeserializer(serializer.buffer(), serializer.length()) {
}
/**
* @brief Skips padding in the input
*
* @param bytes Number of bytes to skip
*/
void skipPadding(size_t bytes) {
if (!canAdvanceBy(bytes, "padding")) {
return;
}
_pos += bytes;
_lastAdvance = bytes;
}
/**
* @brief Read an uint8_t from the buffer
*
* @param c Character to read
* @return SerDes& This object
*/
DataDeserializer &operator>>(uint8_t &c) {
return *this >> reinterpret_cast<int8_t&>(c);
}
/**
* @brief Read an int8_t from the buffer
*
* @param c Character to read
* @return SerDes& This object
*/
DataDeserializer &operator>>(int8_t &c) {
if ( canAdvanceBy(1, "int8_t") ) {
c = _store[_pos++];
_lastAdvance = sizeof(c);
}
return *this;
}
///////////////////////////////////////////////////////////
/**
* @brief Read an uint16_t from the buffer
*
* @param val Value to read
* @return SerDes& This object
*/
DataDeserializer &operator>>(uint16_t &val) {
return *this >> reinterpret_cast<int16_t&>(val);
}
/**
* @brief Read an int16_t from the buffer
*
* @param val Value to read
* @return SerDes& This object
*/
DataDeserializer &operator>>(int16_t &val) {
if ( canAdvanceBy(sizeof(val), "int16_t") ) {
memcpy((char*)&val, &_store[_pos], sizeof(val));
_pos += sizeof(val);
_lastAdvance = sizeof(val);
}
return *this;
}
///////////////////////////////////////////////////////////
/**
* @brief Read an uint32_t from the buffer
*
* @param val Value to read
* @return SerDes& This object
*/
DataDeserializer &operator>>(uint32_t &val) {
return *this >> reinterpret_cast<int32_t&>(val);
}
/**
* @brief Read an int32_t from the buffer
*
* @param val Value to read
* @return SerDes& This object
*/
DataDeserializer &operator>>(int32_t &val) {
if ( canAdvanceBy(sizeof(val), "int32_t") ) {
memcpy((char*)&val, &_store[_pos], sizeof(val));
_pos += sizeof(val);
_lastAdvance = sizeof(val);
}
return *this;
}
///////////////////////////////////////////////////////////
/**
* @brief Read an uint64_t from the buffer
*
* @param val Value to read
* @return SerDes& This object
*/
DataDeserializer &operator>>(uint64_t &val) {
return *this >> reinterpret_cast<int64_t&>(val);
}
/**
* @brief Read an int64_t from the buffer
*
* @param val Value to read
* @return SerDes& This object
*/
DataDeserializer &operator>>(int64_t &val) {
if ( canAdvanceBy(sizeof(val), "int64_t") ) {
memcpy((char*)&val, &_store[_pos], sizeof(val));
_pos += sizeof(val);
_lastAdvance = sizeof(val);
}
return *this;
}
///////////////////////////////////////////////////////////
/**
* @brief Read an float from the buffer
*
* @param val Value to read
* @return SerDes& This object
*/
DataDeserializer &operator>>(float &val) {
if ( canAdvanceBy(sizeof(val), "float") ) {
memcpy((char*)&val, &_store[_pos], sizeof(val));
_pos += sizeof(val);
_lastAdvance = sizeof(val);
}
return *this;
}
///////////////////////////////////////////////////////////
/**
* @brief Read a glm::vec3 from the buffer
*
* @param val Value to read
* @return SerDes& This object
*/
DataDeserializer &operator>>(glm::vec3 &val) {
size_t sz = sizeof(val.x);
if ( canAdvanceBy(sz*3, "glm::vec3") ) {
memcpy((char*)&val.x, &_store[_pos ], sz);
memcpy((char*)&val.y, &_store[_pos + sz ], sz);
memcpy((char*)&val.z, &_store[_pos + sz*2], sz);
_pos += sz*3;
_lastAdvance = sz * 3;
}
return *this;
}
///////////////////////////////////////////////////////////
/**
* @brief Read a glm::vec4 from the buffer
*
* @param val Value to read
* @return SerDes& This object
*/
DataDeserializer &operator>>(glm::vec4 &val) {
size_t sz = sizeof(val.x);
if ( canAdvanceBy(sz*4, "glm::vec4")) {
memcpy((char*)&val.x, &_store[_pos ], sz);
memcpy((char*)&val.y, &_store[_pos + sz ], sz);
memcpy((char*)&val.z, &_store[_pos + sz*2], sz);
memcpy((char*)&val.w, &_store[_pos + sz*3], sz);
_pos += sz*4;
_lastAdvance = sz*4;
}
return *this;
}
///////////////////////////////////////////////////////////
/**
* @brief Read a glm::ivec2 from the buffer
*
* @param val Value to read
* @return SerDes& This object
*/
DataDeserializer &operator>>(glm::ivec2 &val) {
size_t sz = sizeof(val.x);
if ( canAdvanceBy(sz*2, "glm::ivec2") ) {
memcpy((char*)&val.x, &_store[_pos ], sz);
memcpy((char*)&val.y, &_store[_pos + sz ], sz);
_pos += sz*2;
_lastAdvance = sz * 2;
}
return *this;
}
///////////////////////////////////////////////////////////
/**
* @brief Pointer to the start of the internal buffer.
*
* The allocated amount can be found with capacity().
*
* The end of the stored data can be found with length().
*
* @return Pointer to buffer
*/
const char *buffer() const { return _store; }
/**
* @brief Current position in the buffer. Starts at 0.
*
* @return size_t
*/
size_t pos() const { return _pos; }
/**
* @brief Last position that was written to in the buffer. Starts at 0.
*
* @return size_t
*/
size_t length() const { return _length; }
/**
* @brief Whether there's any data in the buffer
*
* @return true Something has been written
* @return false The buffer is empty
*/
bool isEmpty() const { return _length == 0; }
/**
* @brief The buffer size limit has been reached
*
* This can only return true for a statically allocated buffer.
*
* @return true Limit reached
* @return false There is still room
*/
bool isOverflow() const { return _overflow; }
/**
* @brief Reset the serializer to the start, clear overflow bit.
*
*/
void rewind() { _pos = 0; _overflow = false; _lastAdvance = 0; }
/**
* @brief Size of the last advance
*
* This can be used to get how many bytes were added in the last operation.
* It is reset on rewind()
*
* @return size_t
*/
size_t lastAdvance() const { return _lastAdvance; }
/**
* @brief Dump the contents of this object into QDebug
*
* This produces a dump of the internal state, and an ASCII/hex dump of
* the contents, for debugging.
*
* @param debug Qt QDebug stream
* @param ds This object
* @return QDebug
*/
friend QDebug operator<<(QDebug debug, const DataDeserializer &ds);
private:
bool canAdvanceBy(size_t bytes, const QString &type_name) {
//qDebug() << "Checking advance by" << bytes;
if ( _length < _pos + bytes) {
qCritical() << "Deserializer trying to read past end of input buffer of" << _length << "bytes, reading" << bytes << "bytes for" << type_name << "from position " << _pos;
_overflow = true;
return false;
}
return true;
}
const char *_store;
bool _overflow = false;
size_t _length = 0;
size_t _pos = 0;
size_t _lastAdvance = 0;
};

22
tests/ktx/src/KtxTests.cpp
View file

@ -16,6 +16,7 @@
#include <gpu/Texture.h>
#include <image/Image.h>
#include <image/TextureProcessing.h>
#include "SerDes.h"
QTEST_GUILESS_MAIN(KtxTests)
@ -31,6 +32,19 @@ QString getRootPath() {
return result;
}
#if 0
ktx::Byte* serializeSPH(ktx::Byte* data, const gpu::IrradianceKTXPayload &payload) const {
*(ktx::IrradianceKTXPayload::Version*)data = IrradianceKTXPayload::CURRENT_VERSION;
data += sizeof(ktx::IrradianceKTXPayload::Version);
memcpy(data, &payload._irradianceSH, sizeof(ktx::SphericalHarmonics));
data += sizeof(SphericalHarmonics);
return data + PADDING;
}
#endif
void KtxTests::initTestCase() {
}
@ -147,6 +161,14 @@ void KtxTests::testKtxSerialization() {
testTexture->setKtxBacking(TEST_IMAGE_KTX.fileName().toStdString());
}
void KtxTests::testKtxNewSerializationSphericalHarmonics() {
DataSerializer ser;
}
#if 0
static const QString TEST_FOLDER { "H:/ktx_cacheold" };

1
tests/ktx/src/KtxTests.h
View file

@ -16,6 +16,7 @@ private slots:
void testKtxEvalFunctions();
void testKhronosCompressionFunctions();
void testKtxSerialization();
void testKtxNewSerializationSphericalHarmonics();
};

232
tests/shared/src/SerializerTests.cpp Normal file
View file

@ -0,0 +1,232 @@
//
// SerializerTests.cpp
//
// Copyright 2022 Dale Glass
//
// Distributed under the Apache License, Version 2.0.
// See the accompanying file LICENSE or http://www.apache.org/licenses/LICENSE-2.0.html
//
#include "SerializerTests.h"
#include <SerDes.h>
#include <QDebug>
#include <glm/glm.hpp>
QTEST_GUILESS_MAIN(SerializerTests)
void SerializerTests::initTestCase() {
}
void SerializerTests::testCreate() {
DataSerializer s;
QCOMPARE(s.length(), 0);
QCOMPARE(s.capacity(), DataSerializer::DEFAULT_SIZE);
QCOMPARE(s.isEmpty(), true);
DataDeserializer d(s);
QCOMPARE(d.length(), 0);
}
void SerializerTests::testAdd() {
DataSerializer s;
s << (qint8)1;
QCOMPARE(s.length(), 1);
QCOMPARE(s.isEmpty(), false);
s << (quint8)-1;
QCOMPARE(s.length(), 2);
s << (qint16)0xaabb;
QCOMPARE(s.length(), 4);
s << (quint16)-18000;
QCOMPARE(s.length(), 6);
s << (qint32)0xCCDDEEFF;
QCOMPARE(s.length(), 10);
s << (quint32)-1818000000;
QCOMPARE(s.length(), 14);
s << "Hello, world!";
QCOMPARE(s.length(), 28);
glm::vec3 v{1.f,2.f,3.f};
s << v;
QCOMPARE(s.length(), 40);
s << 1.2345f;
QCOMPARE(s.length(), 44);
qDebug() << s;
}
void SerializerTests::testAddAndRead() {
DataSerializer s;
glm::vec3 v3_a{1.f, 3.1415f, 2.71828f};
glm::vec3 v3_b;
glm::vec4 v4_a{3.1415f, 2.71828f, 1.4142f, 1.6180f};
glm::vec4 v4_b;
glm::ivec2 iv2_a{10, 24};
glm::ivec2 iv2_b;
float f_a = 1.2345f;
float f_b;
s << (qint8)1;
s << (qint16)0xaabb;
s << (qint32)0xccddeeff;
s << v3_a;
s << v4_a;
s << iv2_a;
s << f_a;
qint8 i8;
qint16 i16;
qint32 i32;
DataDeserializer d(s);
d >> i8;
d >> i16;
d >> i32;
d >> v3_b;
d >> v4_b;
d >> iv2_b;
d >> f_b;
qDebug() << d;
QCOMPARE(i8, (qint8)1);
QCOMPARE(i16, (qint16)0xaabb);
QCOMPARE(i32, (qint32)0xccddeeff);
QCOMPARE(v3_a, v3_b);
QCOMPARE(v4_a, v4_b);
QCOMPARE(iv2_a, iv2_b);
QCOMPARE(f_a, f_b);
}
void SerializerTests::testReadPastEnd() {
DataSerializer s;
qint8 i8;
qint16 i16;
s << (qint8)1;
DataDeserializer d(s);
d >> i8;
QCOMPARE(d.pos(), 1);
d.rewind();
d >> i16;
QCOMPARE(d.pos(), 0);
}
void SerializerTests::testWritePastEnd() {
qint8 i8 = 255;
qint16 i16 = 65535;
char buf[16];
// 1 byte buffer, we can write 1 byte
memset(buf, 0, sizeof(buf));
DataSerializer s1(buf, 1);
s1 << i8;
QCOMPARE(s1.pos(), 1);
QCOMPARE(s1.isOverflow(), false);
QCOMPARE(buf[0], i8);
// 1 byte buffer, we can't write 2 bytes
memset(buf, 0, sizeof(buf));
DataSerializer s2(buf, 1);
s2 << i16;
QCOMPARE(s2.pos(), 0);
QCOMPARE(s2.isOverflow(), true);
QCOMPARE(buf[0], 0); // We didn't write
QCOMPARE(buf[1], 0);
}
void SerializerTests::benchmarkEncodingDynamicAlloc() {
QBENCHMARK {
DataSerializer s;
glm::vec3 v3_a{1.f, 3.1415f, 2.71828f};
glm::vec3 v3_b;
glm::vec4 v4_a{3.1415f, 2.71828f, 1.4142f, 1.6180f};
glm::vec4 v4_b;
glm::ivec2 iv2_a{10, 24};
glm::ivec2 iv2_b;
s << (qint8)1;
s << (qint16)0xaabb;
s << (qint32)0xccddeeff;
s << v3_a;
s << v4_a;
s << iv2_a;
}
}
void SerializerTests::benchmarkEncodingStaticAlloc() {
char buf[1024];
QBENCHMARK {
DataSerializer s(buf, sizeof(buf));
glm::vec3 v3_a{1.f, 3.1415f, 2.71828f};
glm::vec3 v3_b;
glm::vec4 v4_a{3.1415f, 2.71828f, 1.4142f, 1.6180f};
glm::vec4 v4_b;
glm::ivec2 iv2_a{10, 24};
glm::ivec2 iv2_b;
s << (qint8)1;
s << (qint16)0xaabb;
s << (qint32)0xccddeeff;
s << v3_a;
s << v4_a;
s << iv2_a;
}
}
void SerializerTests::benchmarkDecoding() {
DataSerializer s;
qint8 q8 = 1;
qint16 q16 = 0xaabb;
qint32 q32 = 0xccddeeff;
glm::vec3 v3_a{1.f, 3.1415f, 2.71828f};
glm::vec3 v3_b;
glm::vec4 v4_a{3.1415f, 2.71828f, 1.4142f, 1.6180f};
glm::vec4 v4_b;
glm::ivec2 iv2_a{10, 24};
glm::ivec2 iv2_b;
s << q8;
s << q16;
s << q32;
s << v3_a;
s << v4_a;
s << iv2_a;
QBENCHMARK {
DataDeserializer d(s);
d >> q8;
d >> q16;
d >> q32;
d >> v3_a;
d >> v4_a;
d >> iv2_a;
}
}
void SerializerTests::cleanupTestCase() {
}

33
tests/shared/src/SerializerTests.h Normal file
View file

@ -0,0 +1,33 @@
//
// ResourceTests.h
//
// Copyright 2015 High Fidelity, Inc.
//
// Distributed under the Apache License, Version 2.0.
// See the accompanying file LICENSE or http://www.apache.org/licenses/LICENSE-2.0.html
//
#ifndef overte_SerializerTests_h
#define overte_SerializerTests_h
#include <QtTest/QtTest>
#include <QtCore/QTemporaryDir>
class SerializerTests : public QObject {
Q_OBJECT
private slots:
void initTestCase();
void testCreate();
void testAdd();
void testAddAndRead();
void testReadPastEnd();
void testWritePastEnd();
void benchmarkEncodingDynamicAlloc();
void benchmarkEncodingStaticAlloc();
void benchmarkDecoding();
void cleanupTestCase();
private:
};
#endif // overte_SerializerTests_h