From 9c2f6ab2e598fff890b0c0ad8b4a21e46be5dc51 Mon Sep 17 00:00:00 2001 From: Andrzej Kapolka Date: Thu, 20 Mar 2014 18:28:43 -0700 Subject: [PATCH] Working on delta streaming for collections. --- libraries/metavoxels/src/Bitstream.cpp | 164 +++++++++++++++------ libraries/metavoxels/src/Bitstream.h | 192 +++++++++++++++++++++++-- tools/mtc/src/main.cpp | 11 ++ 3 files changed, 312 insertions(+), 55 deletions(-) diff --git a/libraries/metavoxels/src/Bitstream.cpp b/libraries/metavoxels/src/Bitstream.cpp index 6bad482ca1..070ad550d8 100644 --- a/libraries/metavoxels/src/Bitstream.cpp +++ b/libraries/metavoxels/src/Bitstream.cpp @@ -577,16 +577,17 @@ Bitstream& Bitstream::operator<(const TypeStreamer* streamer) { if (_metadataType == NO_METADATA) { return *this; } - TypeStreamer::Category category = streamer->getCategory(); - *this << (int)category; - switch (category) { - case TypeStreamer::SIMPLE_CATEGORY: + TypeReader::Type type = streamer->getReaderType(); + *this << (int)type; + switch (type) { + case TypeReader::SIMPLE_TYPE: return *this; - case TypeStreamer::LIST_CATEGORY: + case TypeReader::LIST_TYPE: + case TypeReader::SET_TYPE: return *this << streamer->getValueStreamer(); - case TypeStreamer::MAP_CATEGORY: + case TypeReader::MAP_TYPE: return *this << streamer->getKeyStreamer() << streamer->getValueStreamer(); default: @@ -628,35 +629,36 @@ Bitstream& Bitstream::operator>(TypeReader& reader) { reader = TypeReader(typeName, streamer); return *this; } - int category; - *this >> category; - switch (category) { - case TypeStreamer::SIMPLE_CATEGORY: + int type; + *this >> type; + switch (type) { + case TypeReader::SIMPLE_TYPE: reader = TypeReader(typeName, streamer); return *this; - case TypeStreamer::LIST_CATEGORY: { + case TypeReader::LIST_TYPE: + case TypeReader::SET_TYPE: { TypeReader valueReader; *this >> valueReader; - if (streamer && streamer->getCategory() == TypeStreamer::LIST_CATEGORY && + if (streamer && streamer->getReaderType() == type && valueReader.matchesExactly(streamer->getValueStreamer())) { reader = TypeReader(typeName, streamer); } else { - reader = TypeReader(typeName, streamer, false, TypeReaderPointer(), + reader = TypeReader(typeName, streamer, false, (TypeReader::Type)type, TypeReaderPointer(), TypeReaderPointer(new TypeReader(valueReader))); } return *this; } - case TypeStreamer::MAP_CATEGORY: { + case TypeReader::MAP_TYPE: { TypeReader keyReader, valueReader; *this >> keyReader >> valueReader; - if (streamer && streamer->getCategory() == TypeStreamer::MAP_CATEGORY && + if (streamer && streamer->getReaderType() == TypeReader::MAP_TYPE && keyReader.matchesExactly(streamer->getKeyStreamer()) && valueReader.matchesExactly(streamer->getValueStreamer())) { reader = TypeReader(typeName, streamer); } else { - reader = TypeReader(typeName, streamer, false, TypeReaderPointer(new TypeReader(keyReader)), - TypeReaderPointer(new TypeReader(valueReader))); + reader = TypeReader(typeName, streamer, false, TypeReader::MAP_TYPE, + TypeReaderPointer(new TypeReader(keyReader)), TypeReaderPointer(new TypeReader(valueReader))); } return *this; } @@ -714,20 +716,23 @@ Bitstream& Bitstream::operator>(TypeReader& reader) { // if all fields are the same type and in the right order, we can use the (more efficient) default streamer const QVector& localFields = streamer->getMetaFields(); if (fieldCount != localFields.size()) { - reader = TypeReader(typeName, streamer, false, TypeReaderPointer(), TypeReaderPointer(), fields); + reader = TypeReader(typeName, streamer, false, TypeReader::STREAMABLE_TYPE, + TypeReaderPointer(), TypeReaderPointer(), fields); return *this; } for (int i = 0; i < fieldCount; i++) { const FieldReader& fieldReader = fields.at(i); if (!fieldReader.getReader().matchesExactly(localFields.at(i).getStreamer()) || fieldReader.getIndex() != i) { - reader = TypeReader(typeName, streamer, false, TypeReaderPointer(), TypeReaderPointer(), fields); + reader = TypeReader(typeName, streamer, false, TypeReader::STREAMABLE_TYPE, + TypeReaderPointer(), TypeReaderPointer(), fields); return *this; } } reader = TypeReader(typeName, streamer); return *this; } - reader = TypeReader(typeName, streamer, false, TypeReaderPointer(), TypeReaderPointer(), fields); + reader = TypeReader(typeName, streamer, false, TypeReader::STREAMABLE_TYPE, + TypeReaderPointer(), TypeReaderPointer(), fields); return *this; } @@ -823,11 +828,12 @@ QVector Bitstream::getPropertyReaders(const QMetaObject* metaObj return propertyReaders; } -TypeReader::TypeReader(const QByteArray& typeName, const TypeStreamer* streamer, bool exactMatch, +TypeReader::TypeReader(const QByteArray& typeName, const TypeStreamer* streamer, bool exactMatch, Type type, const TypeReaderPointer& keyReader, const TypeReaderPointer& valueReader, const QVector& fields) : _typeName(typeName), _streamer(streamer), _exactMatch(exactMatch), + _type(type), _keyReader(keyReader), _valueReader(valueReader), _fields(fields) { @@ -838,10 +844,28 @@ QVariant TypeReader::read(Bitstream& in) const { return _streamer->read(in); } QVariant object = _streamer ? QVariant(_streamer->getType(), 0) : QVariant(); - if (_valueReader) { - int size; - in >> size; - if (_keyReader) { + switch (_type) { + case STREAMABLE_TYPE: { + foreach (const FieldReader& field, _fields) { + field.read(in, _streamer, object); + } + break; + } + case LIST_TYPE: + case SET_TYPE: { + int size; + in >> size; + for (int i = 0; i < size; i++) { + QVariant value = _valueReader->read(in); + if (_streamer) { + _streamer->insert(object, value); + } + } + break; + } + case MAP_TYPE: { + int size; + in >> size; for (int i = 0; i < size; i++) { QVariant key = _keyReader->read(in); QVariant value = _valueReader->read(in); @@ -849,18 +873,10 @@ QVariant TypeReader::read(Bitstream& in) const { _streamer->insert(object, key, value); } } - } else { - for (int i = 0; i < size; i++) { - QVariant value = _valueReader->read(in); - if (_streamer) { - _streamer->insert(object, value); - } - } - } - } else { - foreach (const FieldReader& field, _fields) { - field.read(in, _streamer, object); + break; } + default: + break; } return object; } @@ -870,13 +886,67 @@ void TypeReader::readDelta(Bitstream& in, QVariant& object, const QVariant& refe _streamer->readDelta(in, object, reference); return; } - if (_valueReader) { - // TODO: collection deltas - - } else { - foreach (const FieldReader& field, _fields) { - field.readDelta(in, _streamer, object, reference); - } + bool changed; + in >> changed; + if (!changed) { + object = reference; + return; + } + switch (_type) { + case STREAMABLE_TYPE: { + foreach (const FieldReader& field, _fields) { + field.readDelta(in, _streamer, object, reference); + } + break; + } + case LIST_TYPE: { + object = reference; + int size, referenceSize; + in >> size >> referenceSize; + + break; + } + case SET_TYPE: { + object = reference; + int addedOrRemoved; + in >> addedOrRemoved; + for (int i = 0; i < addedOrRemoved; i++) { + QVariant value = _valueReader->read(in); + if (_streamer && !_streamer->remove(object, value)) { + _streamer->insert(object, value); + } + } + break; + } + case MAP_TYPE: { + object = reference; + int added; + in >> added; + for (int i = 0; i < added; i++) { + QVariant key = _keyReader->read(in); + QVariant value = _valueReader->read(in); + if (_streamer) { + _streamer->insert(object, key, value); + } + } + int modified; + in >> modified; + for (int i = 0; i < modified; i++) { + QVariant key = _keyReader->read(in); + + } + int removed; + in >> removed; + for (int i = 0; i < removed; i++) { + QVariant key = _keyReader->read(in); + if (_streamer) { + _streamer->remove(object, key); + } + } + break; + } + default: + break; } } @@ -958,8 +1028,8 @@ void TypeStreamer::setField(int index, QVariant& object, const QVariant& value) // nothing by default } -TypeStreamer::Category TypeStreamer::getCategory() const { - return SIMPLE_CATEGORY; +TypeReader::Type TypeStreamer::getReaderType() const { + return TypeReader::SIMPLE_TYPE; } const TypeStreamer* TypeStreamer::getKeyStreamer() const { @@ -977,3 +1047,7 @@ void TypeStreamer::insert(QVariant& object, const QVariant& element) const { void TypeStreamer::insert(QVariant& object, const QVariant& key, const QVariant& value) const { // nothing by default } + +bool TypeStreamer::remove(QVariant& object, const QVariant& key) const { + return false; +} diff --git a/libraries/metavoxels/src/Bitstream.h b/libraries/metavoxels/src/Bitstream.h index 49b4ddd95f..cfee647d28 100644 --- a/libraries/metavoxels/src/Bitstream.h +++ b/libraries/metavoxels/src/Bitstream.h @@ -271,9 +271,15 @@ public: template void writeDelta(const T& value, const T& reference); template void readDelta(T& value, const T& reference); + template void writeDelta(const QList& value, const QList& reference); + template void readDelta(QList& value, const QList& reference); + template void writeDelta(const QSet& value, const QSet& reference); template void readDelta(QSet& value, const QSet& reference); + template void writeDelta(const QHash& value, const QHash& reference); + template void readDelta(QHash& value, const QHash& reference); + Bitstream& operator<<(bool value); Bitstream& operator>>(bool& value); @@ -403,12 +409,171 @@ template inline void Bitstream::readDelta(T& value, const T& reference) } } +template inline void Bitstream::writeDelta(const QList& value, const QList& reference) { + if (value == reference) { + *this << false; + return; + } + *this << true; + *this << value.size(); + *this << reference.size(); + for (int i = 0; i < value.size(); i++) { + if (i < reference.size()) { + writeDelta(value.at(i), reference.at(i)); + } else { + *this << value.at(i); + } + } +} + +template inline void Bitstream::readDelta(QList& value, const QList& reference) { + value = reference; + bool changed; + *this >> changed; + if (!changed) { + return; + } + int size, referenceSize; + *this >> size >> referenceSize; + if (size < value.size()) { + value.erase(value.begin() + size, value.end()); + } + for (int i = 0; i < size; i++) { + if (i < referenceSize) { + readDelta(value[i], reference.at(i)); + } else { + T element; + *this >> element; + value.append(element); + } + } +} + template inline void Bitstream::writeDelta(const QSet& value, const QSet& reference) { - + if (value == reference) { + *this << false; + return; + } + *this << true; + int addedOrRemoved = 0; + foreach (const T& element, value) { + if (!reference.contains(element)) { + addedOrRemoved++; + } + } + foreach (const T& element, reference) { + if (!value.contains(element)) { + addedOrRemoved++; + } + } + *this << addedOrRemoved; + foreach (const T& element, value) { + if (!reference.contains(element)) { + *this << element; + } + } + foreach (const T& element, reference) { + if (!value.contains(element)) { + *this << element; + } + } } template inline void Bitstream::readDelta(QSet& value, const QSet& reference) { - + value = reference; + bool changed; + *this >> changed; + if (!changed) { + return; + } + int addedOrRemoved; + *this >> addedOrRemoved; + for (int i = 0; i < addedOrRemoved; i++) { + T element; + *this >> element; + if (!value.remove(element)) { + value.insert(element); + } + } +} + +template inline void Bitstream::writeDelta(const QHash& value, const QHash& reference) { + if (value == reference) { + *this << false; + return; + } + *this << true; + int added = 0; + int modified = 0; + for (typename QHash::const_iterator it = value.constBegin(); it != value.constEnd(); it++) { + typename QHash::const_iterator previous = reference.find(it.key()); + if (previous == reference.constEnd()) { + added++; + } else if (previous.value() != it.value()) { + modified++; + } + } + *this << added; + for (typename QHash::const_iterator it = value.constBegin(); it != value.constEnd(); it++) { + if (!reference.contains(it.key())) { + *this << it.key(); + *this << it.value(); + } + } + *this << modified; + for (typename QHash::const_iterator it = value.constBegin(); it != value.constEnd(); it++) { + typename QHash::const_iterator previous = reference.find(it.key()); + if (previous != reference.constEnd() && previous.value() != it.value()) { + *this << it.key(); + writeDelta(it.value(), previous.value()); + } + } + int removed = 0; + for (typename QHash::const_iterator it = reference.constBegin(); it != reference.constEnd(); it++) { + if (!value.contains(it.key())) { + removed++; + } + } + *this << removed; + for (typename QHash::const_iterator it = reference.constBegin(); it != reference.constEnd(); it++) { + if (!value.contains(it.key())) { + *this << it.key(); + } + } +} + +template inline void Bitstream::readDelta(QHash& value, const QHash& reference) { + value = reference; + bool changed; + *this >> changed; + if (!changed) { + return; + } + int added; + *this >> added; + for (int i = 0; i < added; i++) { + K key; + V mapping; + *this >> key >> mapping; + value.insert(key, mapping); + } + int modified; + *this >> modified; + for (int i = 0; i < modified; i++) { + K key; + *this >> key; + V& mapping = value[key]; + V newMapping; + readDelta(newMapping, mapping); + mapping = newMapping; + } + int removed; + *this >> removed; + for (int i = 0; i < removed; i++) { + K key; + *this >> key; + value.remove(key); + } } template inline Bitstream& Bitstream::operator<<(const QList& list) { @@ -483,8 +648,11 @@ typedef QSharedPointer TypeReaderPointer; class TypeReader { public: + enum Type { SIMPLE_TYPE, STREAMABLE_TYPE, LIST_TYPE, SET_TYPE, MAP_TYPE }; + TypeReader(const QByteArray& typeName = QByteArray(), const TypeStreamer* streamer = NULL, bool exactMatch = true, - const TypeReaderPointer& keyReader = TypeReaderPointer(), const TypeReaderPointer& valueReader = TypeReaderPointer(), + Type type = SIMPLE_TYPE, const TypeReaderPointer& keyReader = TypeReaderPointer(), + const TypeReaderPointer& valueReader = TypeReaderPointer(), const QVector& fields = QVector()); const QByteArray& getTypeName() const { return _typeName; } @@ -503,6 +671,7 @@ private: QByteArray _typeName; const TypeStreamer* _streamer; bool _exactMatch; + Type _type; TypeReaderPointer _keyReader; TypeReaderPointer _valueReader; QVector _fields; @@ -607,15 +776,14 @@ public: virtual int getFieldIndex(const QByteArray& name) const; virtual void setField(int index, QVariant& object, const QVariant& value) const; - enum Category { SIMPLE_CATEGORY, STREAMABLE_CATEGORY, LIST_CATEGORY, MAP_CATEGORY }; - - virtual Category getCategory() const; + virtual TypeReader::Type getReaderType() const; virtual const TypeStreamer* getKeyStreamer() const; virtual const TypeStreamer* getValueStreamer() const; virtual void insert(QVariant& object, const QVariant& value) const; virtual void insert(QVariant& object, const QVariant& key, const QVariant& value) const; + virtual bool remove(QVariant& object, const QVariant& key) const; private: @@ -638,7 +806,7 @@ public: template class StreamableTypeStreamer : public SimpleTypeStreamer { public: - virtual TypeStreamer::Category getCategory() const { return TypeStreamer::STREAMABLE_CATEGORY; } + virtual TypeReader::Type getReaderType() const { return TypeReader::STREAMABLE_TYPE; } virtual const QVector& getMetaFields() const { return T::getMetaFields(); } virtual int getFieldIndex(const QByteArray& name) const { return T::getFieldIndex(name); } virtual void setField(int index, QVariant& object, const QVariant& value) const { @@ -653,7 +821,7 @@ template class CollectionTypeStreamer : public SimpleTypeStreamer { template class CollectionTypeStreamer > : public SimpleTypeStreamer > { public: - virtual TypeStreamer::Category getCategory() const { return TypeStreamer::LIST_CATEGORY; } + virtual TypeReader::Type getReaderType() const { return TypeReader::LIST_TYPE; } virtual const TypeStreamer* getValueStreamer() const { return Bitstream::getTypeStreamer(qMetaTypeId()); } virtual void insert(QVariant& object, const QVariant& value) const { static_cast*>(object.data())->append(value.value()); } @@ -663,21 +831,25 @@ public: template class CollectionTypeStreamer > : public SimpleTypeStreamer > { public: - virtual TypeStreamer::Category getCategory() const { return TypeStreamer::LIST_CATEGORY; } + virtual TypeReader::Type getReaderType() const { return TypeReader::SET_TYPE; } virtual const TypeStreamer* getValueStreamer() const { return Bitstream::getTypeStreamer(qMetaTypeId()); } virtual void insert(QVariant& object, const QVariant& value) const { static_cast*>(object.data())->insert(value.value()); } + virtual bool remove(QVariant& object, const QVariant& key) const { + return static_cast*>(object.data())->remove(key.value()); } }; /// A streamer for hash types. template class CollectionTypeStreamer > : public SimpleTypeStreamer > { public: - virtual TypeStreamer::Category getCategory() const { return TypeStreamer::MAP_CATEGORY; } + virtual TypeReader::Type getReaderType() const { return TypeReader::MAP_TYPE; } virtual const TypeStreamer* getKeyStreamer() const { return Bitstream::getTypeStreamer(qMetaTypeId()); } virtual const TypeStreamer* getValueStreamer() const { return Bitstream::getTypeStreamer(qMetaTypeId()); } virtual void insert(QVariant& object, const QVariant& key, const QVariant& value) const { static_cast*>(object.data())->insert(key.value(), value.value()); } + virtual bool remove(QVariant& object, const QVariant& key) const { + return static_cast*>(object.data())->remove(key.value()); } }; /// Macro for registering simple type streamers. diff --git a/tools/mtc/src/main.cpp b/tools/mtc/src/main.cpp index e6f1827870..861b7310b8 100644 --- a/tools/mtc/src/main.cpp +++ b/tools/mtc/src/main.cpp @@ -173,6 +173,11 @@ void generateOutput (QTextStream& out, const QList& streamables) { out << "}\n"; out << "template<> void Bitstream::writeDelta(const " << name << "& value, const " << name << "& reference) {\n"; + out << " if (value == reference) {\n"; + out << " *this << false;\n"; + out << " return;\n"; + out << " }\n"; + out << " *this << true;\n"; foreach (const QString& base, str.clazz.bases) { out << " writeDelta(static_cast(value), static_cast(reference));\n"; @@ -183,6 +188,12 @@ void generateOutput (QTextStream& out, const QList& streamables) { out << "}\n"; out << "template<> void Bitstream::readDelta(" << name << "& value, const " << name << "& reference) {\n"; + out << " bool changed;\n"; + out << " *this >> changed;\n"; + out << " if (!changed) {\n"; + out << " value = reference;\n"; + out << " return;\n"; + out << " }\n"; foreach (const QString& base, str.clazz.bases) { out << " readDelta(static_cast<" << base << "&>(value), static_cast(reference));\n";