Mirrors/overte-lubosz
3
0
Fork 0
mirror of https://github.com/lubosz/overte.git synced 2025-04-13 03:53:09 +02:00
Code Issues Projects Releases Packages Wiki Activity Actions

Working on delta streaming for collections.

Browse source
This commit is contained in:
Andrzej Kapolka 2014-03-20 18:28:43 -07:00
parent 20e9572f60
commit 9c2f6ab2e5
3 changed files with 312 additions and 55 deletions
Show stats Download patch file Download diff file Expand all files Collapse all files

164
libraries/metavoxels/src/Bitstream.cpp
View file

@ -577,16 +577,17 @@ Bitstream& Bitstream::operator<(const TypeStreamer* streamer) {
if (_metadataType == NO_METADATA) {
return *this;
}
TypeStreamer::Category category = streamer->getCategory();
*this << (int)category;
switch (category) {
case TypeStreamer::SIMPLE_CATEGORY:
TypeReader::Type type = streamer->getReaderType();
*this << (int)type;
switch (type) {
case TypeReader::SIMPLE_TYPE:
return *this;
case TypeStreamer::LIST_CATEGORY:
case TypeReader::LIST_TYPE:
case TypeReader::SET_TYPE:
return *this << streamer->getValueStreamer();
case TypeStreamer::MAP_CATEGORY:
case TypeReader::MAP_TYPE:
return *this << streamer->getKeyStreamer() << streamer->getValueStreamer();
default:
@ -628,35 +629,36 @@ Bitstream& Bitstream::operator>(TypeReader& reader) {
reader = TypeReader(typeName, streamer);
return *this;
}
int category;
*this >> category;
switch (category) {
case TypeStreamer::SIMPLE_CATEGORY:
int type;
*this >> type;
switch (type) {
case TypeReader::SIMPLE_TYPE:
reader = TypeReader(typeName, streamer);
return *this;
case TypeStreamer::LIST_CATEGORY: {
case TypeReader::LIST_TYPE:
case TypeReader::SET_TYPE: {
TypeReader valueReader;
*this >> valueReader;
if (streamer && streamer->getCategory() == TypeStreamer::LIST_CATEGORY &&
if (streamer && streamer->getReaderType() == type &&
valueReader.matchesExactly(streamer->getValueStreamer())) {
reader = TypeReader(typeName, streamer);
} else {
reader = TypeReader(typeName, streamer, false, TypeReaderPointer(),
reader = TypeReader(typeName, streamer, false, (TypeReader::Type)type, TypeReaderPointer(),
TypeReaderPointer(new TypeReader(valueReader)));
}
return *this;
}
case TypeStreamer::MAP_CATEGORY: {
case TypeReader::MAP_TYPE: {
TypeReader keyReader, valueReader;
*this >> keyReader >> valueReader;
if (streamer && streamer->getCategory() == TypeStreamer::MAP_CATEGORY &&
if (streamer && streamer->getReaderType() == TypeReader::MAP_TYPE &&
keyReader.matchesExactly(streamer->getKeyStreamer()) &&
valueReader.matchesExactly(streamer->getValueStreamer())) {
reader = TypeReader(typeName, streamer);
} else {
reader = TypeReader(typeName, streamer, false, TypeReaderPointer(new TypeReader(keyReader)),
TypeReaderPointer(new TypeReader(valueReader)));
reader = TypeReader(typeName, streamer, false, TypeReader::MAP_TYPE,
TypeReaderPointer(new TypeReader(keyReader)), TypeReaderPointer(new TypeReader(valueReader)));
}
return *this;
}
@ -714,20 +716,23 @@ Bitstream& Bitstream::operator>(TypeReader& reader) {
// if all fields are the same type and in the right order, we can use the (more efficient) default streamer
const QVector<MetaField>& localFields = streamer->getMetaFields();
if (fieldCount != localFields.size()) {
reader = TypeReader(typeName, streamer, false, TypeReaderPointer(), TypeReaderPointer(), fields);
reader = TypeReader(typeName, streamer, false, TypeReader::STREAMABLE_TYPE,
TypeReaderPointer(), TypeReaderPointer(), fields);
return *this;
}
for (int i = 0; i < fieldCount; i++) {
const FieldReader& fieldReader = fields.at(i);
if (!fieldReader.getReader().matchesExactly(localFields.at(i).getStreamer()) || fieldReader.getIndex() != i) {
reader = TypeReader(typeName, streamer, false, TypeReaderPointer(), TypeReaderPointer(), fields);
reader = TypeReader(typeName, streamer, false, TypeReader::STREAMABLE_TYPE,
TypeReaderPointer(), TypeReaderPointer(), fields);
return *this;
}
}
reader = TypeReader(typeName, streamer);
return *this;
}
reader = TypeReader(typeName, streamer, false, TypeReaderPointer(), TypeReaderPointer(), fields);
reader = TypeReader(typeName, streamer, false, TypeReader::STREAMABLE_TYPE,
TypeReaderPointer(), TypeReaderPointer(), fields);
return *this;
}
@ -823,11 +828,12 @@ QVector<PropertyReader> Bitstream::getPropertyReaders(const QMetaObject* metaObj
return propertyReaders;
}
TypeReader::TypeReader(const QByteArray& typeName, const TypeStreamer* streamer, bool exactMatch,
TypeReader::TypeReader(const QByteArray& typeName, const TypeStreamer* streamer, bool exactMatch, Type type,
const TypeReaderPointer& keyReader, const TypeReaderPointer& valueReader, const QVector<FieldReader>& fields) :
_typeName(typeName),
_streamer(streamer),
_exactMatch(exactMatch),
_type(type),
_keyReader(keyReader),
_valueReader(valueReader),
_fields(fields) {
@ -838,10 +844,28 @@ QVariant TypeReader::read(Bitstream& in) const {
return _streamer->read(in);
}
QVariant object = _streamer ? QVariant(_streamer->getType(), 0) : QVariant();
if (_valueReader) {
int size;
in >> size;
if (_keyReader) {
switch (_type) {
case STREAMABLE_TYPE: {
foreach (const FieldReader& field, _fields) {
field.read(in, _streamer, object);
}
break;
}
case LIST_TYPE:
case SET_TYPE: {
int size;
in >> size;
for (int i = 0; i < size; i++) {
QVariant value = _valueReader->read(in);
if (_streamer) {
_streamer->insert(object, value);
}
}
break;
}
case MAP_TYPE: {
int size;
in >> size;
for (int i = 0; i < size; i++) {
QVariant key = _keyReader->read(in);
QVariant value = _valueReader->read(in);
@ -849,18 +873,10 @@ QVariant TypeReader::read(Bitstream& in) const {
_streamer->insert(object, key, value);
}
}
} else {
for (int i = 0; i < size; i++) {
QVariant value = _valueReader->read(in);
if (_streamer) {
_streamer->insert(object, value);
}
}
}
} else {
foreach (const FieldReader& field, _fields) {
field.read(in, _streamer, object);
break;
}
default:
break;
}
return object;
}
@ -870,13 +886,67 @@ void TypeReader::readDelta(Bitstream& in, QVariant& object, const QVariant& refe
_streamer->readDelta(in, object, reference);
return;
}
if (_valueReader) {
// TODO: collection deltas
} else {
foreach (const FieldReader& field, _fields) {
field.readDelta(in, _streamer, object, reference);
}
bool changed;
in >> changed;
if (!changed) {
object = reference;
return;
}
switch (_type) {
case STREAMABLE_TYPE: {
foreach (const FieldReader& field, _fields) {
field.readDelta(in, _streamer, object, reference);
}
break;
}
case LIST_TYPE: {
object = reference;
int size, referenceSize;
in >> size >> referenceSize;
break;
}
case SET_TYPE: {
object = reference;
int addedOrRemoved;
in >> addedOrRemoved;
for (int i = 0; i < addedOrRemoved; i++) {
QVariant value = _valueReader->read(in);
if (_streamer && !_streamer->remove(object, value)) {
_streamer->insert(object, value);
}
}
break;
}
case MAP_TYPE: {
object = reference;
int added;
in >> added;
for (int i = 0; i < added; i++) {
QVariant key = _keyReader->read(in);
QVariant value = _valueReader->read(in);
if (_streamer) {
_streamer->insert(object, key, value);
}
}
int modified;
in >> modified;
for (int i = 0; i < modified; i++) {
QVariant key = _keyReader->read(in);
}
int removed;
in >> removed;
for (int i = 0; i < removed; i++) {
QVariant key = _keyReader->read(in);
if (_streamer) {
_streamer->remove(object, key);
}
}
break;
}
default:
break;
}
}
@ -958,8 +1028,8 @@ void TypeStreamer::setField(int index, QVariant& object, const QVariant& value)
// nothing by default
}
TypeStreamer::Category TypeStreamer::getCategory() const {
return SIMPLE_CATEGORY;
TypeReader::Type TypeStreamer::getReaderType() const {
return TypeReader::SIMPLE_TYPE;
}
const TypeStreamer* TypeStreamer::getKeyStreamer() const {
@ -977,3 +1047,7 @@ void TypeStreamer::insert(QVariant& object, const QVariant& element) const {
void TypeStreamer::insert(QVariant& object, const QVariant& key, const QVariant& value) const {
// nothing by default
}
bool TypeStreamer::remove(QVariant& object, const QVariant& key) const {
return false;
}

192
libraries/metavoxels/src/Bitstream.h
View file

@ -271,9 +271,15 @@ public:
template<class T> void writeDelta(const T& value, const T& reference);
template<class T> void readDelta(T& value, const T& reference);
template<class T> void writeDelta(const QList<T>& value, const QList<T>& reference);
template<class T> void readDelta(QList<T>& value, const QList<T>& reference);
template<class T> void writeDelta(const QSet<T>& value, const QSet<T>& reference);
template<class T> void readDelta(QSet<T>& value, const QSet<T>& reference);
template<class K, class V> void writeDelta(const QHash<K, V>& value, const QHash<K, V>& reference);
template<class K, class V> void readDelta(QHash<K, V>& value, const QHash<K, V>& reference);
Bitstream& operator<<(bool value);
Bitstream& operator>>(bool& value);
@ -403,12 +409,171 @@ template<class T> inline void Bitstream::readDelta(T& value, const T& reference)
}
}
template<class T> inline void Bitstream::writeDelta(const QList<T>& value, const QList<T>& reference) {
if (value == reference) {
*this << false;
return;
}
*this << true;
*this << value.size();
*this << reference.size();
for (int i = 0; i < value.size(); i++) {
if (i < reference.size()) {
writeDelta(value.at(i), reference.at(i));
} else {
*this << value.at(i);
}
}
}
template<class T> inline void Bitstream::readDelta(QList<T>& value, const QList<T>& reference) {
value = reference;
bool changed;
*this >> changed;
if (!changed) {
return;
}
int size, referenceSize;
*this >> size >> referenceSize;
if (size < value.size()) {
value.erase(value.begin() + size, value.end());
}
for (int i = 0; i < size; i++) {
if (i < referenceSize) {
readDelta(value[i], reference.at(i));
} else {
T element;
*this >> element;
value.append(element);
}
}
}
template<class T> inline void Bitstream::writeDelta(const QSet<T>& value, const QSet<T>& reference) {
if (value == reference) {
*this << false;
return;
}
*this << true;
int addedOrRemoved = 0;
foreach (const T& element, value) {
if (!reference.contains(element)) {
addedOrRemoved++;
}
}
foreach (const T& element, reference) {
if (!value.contains(element)) {
addedOrRemoved++;
}
}
*this << addedOrRemoved;
foreach (const T& element, value) {
if (!reference.contains(element)) {
*this << element;
}
}
foreach (const T& element, reference) {
if (!value.contains(element)) {
*this << element;
}
}
}
template<class T> inline void Bitstream::readDelta(QSet<T>& value, const QSet<T>& reference) {
value = reference;
bool changed;
*this >> changed;
if (!changed) {
return;
}
int addedOrRemoved;
*this >> addedOrRemoved;
for (int i = 0; i < addedOrRemoved; i++) {
T element;
*this >> element;
if (!value.remove(element)) {
value.insert(element);
}
}
}
template<class K, class V> inline void Bitstream::writeDelta(const QHash<K, V>& value, const QHash<K, V>& reference) {
if (value == reference) {
*this << false;
return;
}
*this << true;
int added = 0;
int modified = 0;
for (typename QHash<K, V>::const_iterator it = value.constBegin(); it != value.constEnd(); it++) {
typename QHash<K, V>::const_iterator previous = reference.find(it.key());
if (previous == reference.constEnd()) {
added++;
} else if (previous.value() != it.value()) {
modified++;
}
}
*this << added;
for (typename QHash<K, V>::const_iterator it = value.constBegin(); it != value.constEnd(); it++) {
if (!reference.contains(it.key())) {
*this << it.key();
*this << it.value();
}
}
*this << modified;
for (typename QHash<K, V>::const_iterator it = value.constBegin(); it != value.constEnd(); it++) {
typename QHash<K, V>::const_iterator previous = reference.find(it.key());
if (previous != reference.constEnd() && previous.value() != it.value()) {
*this << it.key();
writeDelta(it.value(), previous.value());
}
}
int removed = 0;
for (typename QHash<K, V>::const_iterator it = reference.constBegin(); it != reference.constEnd(); it++) {
if (!value.contains(it.key())) {
removed++;
}
}
*this << removed;
for (typename QHash<K, V>::const_iterator it = reference.constBegin(); it != reference.constEnd(); it++) {
if (!value.contains(it.key())) {
*this << it.key();
}
}
}
template<class K, class V> inline void Bitstream::readDelta(QHash<K, V>& value, const QHash<K, V>& reference) {
value = reference;
bool changed;
*this >> changed;
if (!changed) {
return;
}
int added;
*this >> added;
for (int i = 0; i < added; i++) {
K key;
V mapping;
*this >> key >> mapping;
value.insert(key, mapping);
}
int modified;
*this >> modified;
for (int i = 0; i < modified; i++) {
K key;
*this >> key;
V& mapping = value[key];
V newMapping;
readDelta(newMapping, mapping);
mapping = newMapping;
}
int removed;
*this >> removed;
for (int i = 0; i < removed; i++) {
K key;
*this >> key;
value.remove(key);
}
}
template<class T> inline Bitstream& Bitstream::operator<<(const QList<T>& list) {
@ -483,8 +648,11 @@ typedef QSharedPointer<TypeReader> TypeReaderPointer;
class TypeReader {
public:
enum Type { SIMPLE_TYPE, STREAMABLE_TYPE, LIST_TYPE, SET_TYPE, MAP_TYPE };
TypeReader(const QByteArray& typeName = QByteArray(), const TypeStreamer* streamer = NULL, bool exactMatch = true,
const TypeReaderPointer& keyReader = TypeReaderPointer(), const TypeReaderPointer& valueReader = TypeReaderPointer(),
Type type = SIMPLE_TYPE, const TypeReaderPointer& keyReader = TypeReaderPointer(),
const TypeReaderPointer& valueReader = TypeReaderPointer(),
const QVector<FieldReader>& fields = QVector<FieldReader>());
const QByteArray& getTypeName() const { return _typeName; }
@ -503,6 +671,7 @@ private:
QByteArray _typeName;
const TypeStreamer* _streamer;
bool _exactMatch;
Type _type;
TypeReaderPointer _keyReader;
TypeReaderPointer _valueReader;
QVector<FieldReader> _fields;
@ -607,15 +776,14 @@ public:
virtual int getFieldIndex(const QByteArray& name) const;
virtual void setField(int index, QVariant& object, const QVariant& value) const;
enum Category { SIMPLE_CATEGORY, STREAMABLE_CATEGORY, LIST_CATEGORY, MAP_CATEGORY };
virtual Category getCategory() const;
virtual TypeReader::Type getReaderType() const;
virtual const TypeStreamer* getKeyStreamer() const;
virtual const TypeStreamer* getValueStreamer() const;
virtual void insert(QVariant& object, const QVariant& value) const;
virtual void insert(QVariant& object, const QVariant& key, const QVariant& value) const;
virtual bool remove(QVariant& object, const QVariant& key) const;
private:
@ -638,7 +806,7 @@ public:
template<class T> class StreamableTypeStreamer : public SimpleTypeStreamer<T> {
public:
virtual TypeStreamer::Category getCategory() const { return TypeStreamer::STREAMABLE_CATEGORY; }
virtual TypeReader::Type getReaderType() const { return TypeReader::STREAMABLE_TYPE; }
virtual const QVector<MetaField>& getMetaFields() const { return T::getMetaFields(); }
virtual int getFieldIndex(const QByteArray& name) const { return T::getFieldIndex(name); }
virtual void setField(int index, QVariant& object, const QVariant& value) const {
@ -653,7 +821,7 @@ template<class T> class CollectionTypeStreamer : public SimpleTypeStreamer<T> {
template<class T> class CollectionTypeStreamer<QList<T> > : public SimpleTypeStreamer<QList<T> > {
public:
virtual TypeStreamer::Category getCategory() const { return TypeStreamer::LIST_CATEGORY; }
virtual TypeReader::Type getReaderType() const { return TypeReader::LIST_TYPE; }
virtual const TypeStreamer* getValueStreamer() const { return Bitstream::getTypeStreamer(qMetaTypeId<T>()); }
virtual void insert(QVariant& object, const QVariant& value) const {
static_cast<QList<T>*>(object.data())->append(value.value<T>()); }
@ -663,21 +831,25 @@ public:
template<class T> class CollectionTypeStreamer<QSet<T> > : public SimpleTypeStreamer<QSet<T> > {
public:
virtual TypeStreamer::Category getCategory() const { return TypeStreamer::LIST_CATEGORY; }
virtual TypeReader::Type getReaderType() const { return TypeReader::SET_TYPE; }
virtual const TypeStreamer* getValueStreamer() const { return Bitstream::getTypeStreamer(qMetaTypeId<T>()); }
virtual void insert(QVariant& object, const QVariant& value) const {
static_cast<QSet<T>*>(object.data())->insert(value.value<T>()); }
virtual bool remove(QVariant& object, const QVariant& key) const {
return static_cast<QSet<T>*>(object.data())->remove(key.value<T>()); }
};
/// A streamer for hash types.
template<class K, class V> class CollectionTypeStreamer<QHash<K, V> > : public SimpleTypeStreamer<QHash<K, V> > {
public:
virtual TypeStreamer::Category getCategory() const { return TypeStreamer::MAP_CATEGORY; }
virtual TypeReader::Type getReaderType() const { return TypeReader::MAP_TYPE; }
virtual const TypeStreamer* getKeyStreamer() const { return Bitstream::getTypeStreamer(qMetaTypeId<K>()); }
virtual const TypeStreamer* getValueStreamer() const { return Bitstream::getTypeStreamer(qMetaTypeId<V>()); }
virtual void insert(QVariant& object, const QVariant& key, const QVariant& value) const {
static_cast<QHash<K, V>*>(object.data())->insert(key.value<K>(), value.value<V>()); }
virtual bool remove(QVariant& object, const QVariant& key) const {
return static_cast<QHash<K, V>*>(object.data())->remove(key.value<K>()); }
};
/// Macro for registering simple type streamers.

11
tools/mtc/src/main.cpp
View file

@ -173,6 +173,11 @@ void generateOutput (QTextStream& out, const QList<Streamable>& streamables) {
out << "}\n";
out << "template<> void Bitstream::writeDelta(const " << name << "& value, const " << name << "& reference) {\n";
out << " if (value == reference) {\n";
out << " *this << false;\n";
out << " return;\n";
out << " }\n";
out << " *this << true;\n";
foreach (const QString& base, str.clazz.bases) {
out << " writeDelta(static_cast<const " << base << "&>(value), static_cast<const " <<
base << "&>(reference));\n";
@ -183,6 +188,12 @@ void generateOutput (QTextStream& out, const QList<Streamable>& streamables) {
out << "}\n";
out << "template<> void Bitstream::readDelta(" << name << "& value, const " << name << "& reference) {\n";
out << " bool changed;\n";
out << " *this >> changed;\n";
out << " if (!changed) {\n";
out << " value = reference;\n";
out << " return;\n";
out << " }\n";
foreach (const QString& base, str.clazz.bases) {
out << " readDelta(static_cast<" << base << "&>(value), static_cast<const " <<
base << "&>(reference));\n";