From 7db05f2c8e822635d28f31af8c63095ef26518ac Mon Sep 17 00:00:00 2001 From: Andrzej Kapolka Date: Fri, 13 Jun 2014 16:25:38 -0700 Subject: [PATCH 01/16] Working on JSON encoding. --- libraries/metavoxels/src/Bitstream.cpp | 223 +++++++++++++++++++++++++ libraries/metavoxels/src/Bitstream.h | 54 +++++- 2 files changed, 276 insertions(+), 1 deletion(-) diff --git a/libraries/metavoxels/src/Bitstream.cpp b/libraries/metavoxels/src/Bitstream.cpp index 80aa07b026..1266e8efa4 100644 --- a/libraries/metavoxels/src/Bitstream.cpp +++ b/libraries/metavoxels/src/Bitstream.cpp @@ -1657,6 +1657,49 @@ const TypeStreamer* Bitstream::createInvalidTypeStreamer() { return streamer; } +void JSONWriter::addTypeStreamer(const TypeStreamer* streamer) { + if (!_typeStreamerNames.contains(streamer->getName())) { + _typeStreamerNames.insert(streamer->getName()); + + // start with a placeholder; then remove/replace with actual metadata + int index = _typeStreamers.size(); + _typeStreamers.append(QJsonValue()); + QJsonValue metadata = streamer->getJSONMetadata(*this); + if (metadata.isNull()) { + _typeStreamers.removeAt(index); + } else { + _typeStreamers.replace(index, metadata); + } + } +} + +void JSONWriter::addObjectStreamer(const ObjectStreamer* streamer) { + if (!_objectStreamerNames.contains(streamer->getName())) { + _objectStreamerNames.insert(streamer->getName()); + + // start with a placeholder; then replace with actual metadata + int index = _objectStreamers.size(); + _objectStreamers.append(QJsonValue()); + _objectStreamers.replace(index, streamer->getJSONMetadata(*this)); + } +} + +void JSONWriter::addSharedObject(const SharedObjectPointer& object) { + if (!_sharedObjectIDs.contains(object->getID())) { + _sharedObjectIDs.insert(object->getID()); + + // start with a placeholder; then replace with actual object + int index = _sharedObjects.size(); + _sharedObjects.append(QJsonValue()); + + QJsonObject sharedObject; + sharedObject.insert("id", object->getID()); + sharedObject.insert("originID", object->getOriginID()); + + _sharedObjects.replace(index, sharedObject); + } +} + ObjectStreamer::ObjectStreamer(const QMetaObject* metaObject) : _metaObject(metaObject) { } @@ -1699,6 +1742,33 @@ void MappedObjectStreamer::writeMetadata(Bitstream& out, bool full) const { } } +QJsonObject MappedObjectStreamer::getJSONMetadata(JSONWriter& writer) const { + QJsonObject metadata; + metadata.insert("name", QString(_metaObject->className())); + QJsonArray properties; + foreach (const StreamerPropertyPair& property, _properties) { + QJsonObject object; + writer.addTypeStreamer(property.first.data()); + object.insert("type", QString(property.first->getName())); + object.insert("name", QString(property.second.name())); + properties.append(object); + } + metadata.insert("properties", properties); + return metadata; +} + +QJsonObject MappedObjectStreamer::getJSONData(JSONWriter& writer, const QObject* object) const { + QJsonObject data; + writer.addObjectStreamer(this); + data.insert("class", QString(_metaObject->className())); + QJsonArray properties; + foreach (const StreamerPropertyPair& property, _properties) { + properties.append(property.first->getJSONData(writer, property.second.read(object))); + } + data.insert("properties", properties); + return data; +} + void MappedObjectStreamer::write(Bitstream& out, const QObject* object) const { foreach (const StreamerPropertyPair& property, _properties) { property.first->write(out, property.second.read(object)); @@ -1775,6 +1845,34 @@ void GenericObjectStreamer::writeMetadata(Bitstream& out, bool full) const { } } +QJsonObject GenericObjectStreamer::getJSONMetadata(JSONWriter& writer) const { + QJsonObject metadata; + metadata.insert("name", QString(_name)); + QJsonArray properties; + foreach (const StreamerNamePair& property, _properties) { + QJsonObject object; + writer.addTypeStreamer(property.first.data()); + object.insert("type", QString(property.first->getName())); + object.insert("name", QString(property.second)); + properties.append(object); + } + metadata.insert("properties", properties); + return metadata; +} + +QJsonObject GenericObjectStreamer::getJSONData(JSONWriter& writer, const QObject* object) const { + QJsonObject data; + writer.addObjectStreamer(this); + data.insert("class", QString(_name)); + QJsonArray properties; + const QVariantList& values = static_cast(object)->getValues(); + for (int i = 0; i < _properties.size(); i++) { + properties.append(_properties.at(i).first->getJSONData(writer, values.at(i))); + } + data.insert("properties", properties); + return data; +} + void GenericObjectStreamer::write(Bitstream& out, const QObject* object) const { const QVariantList& values = static_cast(object)->getValues(); for (int i = 0; i < _properties.size(); i++) { @@ -1876,6 +1974,55 @@ void TypeStreamer::writeMetadata(Bitstream& out, bool full) const { } } +QJsonValue TypeStreamer::getJSONMetadata(JSONWriter& writer) const { + Category category = getCategory(); + switch (category) { + case STREAMABLE_CATEGORY: { + QJsonObject metadata; + metadata.insert("name", QString(getName())); + metadata.insert("category", QString("STREAMABLE")); + QJsonArray fields; + foreach (const MetaField& metaField, getMetaFields()) { + QJsonObject field; + writer.addTypeStreamer(metaField.getStreamer()); + field.insert("type", QString(metaField.getStreamer()->getName())); + field.insert("name", QString(metaField.getName())); + fields.append(field); + } + metadata.insert("fields", fields); + return metadata; + } + case LIST_CATEGORY: + case SET_CATEGORY: { + QJsonObject metadata; + metadata.insert("name", QString(getName())); + metadata.insert("category", QString(category == LIST_CATEGORY ? "LIST" : "SET")); + const TypeStreamer* valueStreamer = getValueStreamer(); + writer.addTypeStreamer(valueStreamer); + metadata.insert("valueType", QString(valueStreamer->getName())); + return metadata; + } + case MAP_CATEGORY: { + QJsonObject metadata; + metadata.insert("name", QString(getName())); + metadata.insert("category", QString("MAP")); + const TypeStreamer* keyStreamer = getKeyStreamer(); + writer.addTypeStreamer(keyStreamer); + metadata.insert("keyType", QString(keyStreamer->getName())); + const TypeStreamer* valueStreamer = getValueStreamer(); + writer.addTypeStreamer(valueStreamer); + metadata.insert("valueType", QString(valueStreamer->getName())); + return metadata; + } + default: + return QJsonValue(); + } +} + +QJsonValue TypeStreamer::getJSONData(JSONWriter& writer, const QVariant& value) const { + return QJsonValue(); +} + bool TypeStreamer::equal(const QVariant& first, const QVariant& second) const { return first == second; } @@ -2045,6 +2192,22 @@ void EnumTypeStreamer::writeMetadata(Bitstream& out, bool full) const { } } +QJsonValue EnumTypeStreamer::getJSONMetadata(JSONWriter& writer) const { + QJsonObject metadata; + metadata.insert("name", QString(getName())); + metadata.insert("category", QString("ENUM")); + QJsonArray values; + QMetaEnum metaEnum = getMetaEnum(); + for (int i = 0; i < metaEnum.keyCount(); i++) { + QJsonObject value; + value.insert("key", QString(metaEnum.key(i))); + value.insert("value", metaEnum.value(i)); + values.append(value); + } + metadata.insert("values", values); + return metadata; +} + TypeStreamer::Category EnumTypeStreamer::getCategory() const { return ENUM_CATEGORY; } @@ -2194,6 +2357,21 @@ void GenericEnumTypeStreamer::writeMetadata(Bitstream& out, bool full) const { } } +QJsonValue GenericEnumTypeStreamer::getJSONMetadata(JSONWriter& writer) const { + QJsonObject metadata; + metadata.insert("name", QString(getName())); + metadata.insert("category", QString("ENUM")); + QJsonArray values; + foreach (const NameIntPair& value, _values) { + QJsonObject object; + object.insert("key", QString(value.first)); + object.insert("value", value.second); + values.append(object); + } + metadata.insert("values", values); + return metadata; +} + void GenericEnumTypeStreamer::write(Bitstream& out, const QVariant& value) const { int intValue = value.toInt(); out.write(&intValue, _bits); @@ -2270,6 +2448,22 @@ void GenericStreamableTypeStreamer::writeMetadata(Bitstream& out, bool full) con } } +QJsonValue GenericStreamableTypeStreamer::getJSONMetadata(JSONWriter& writer) const { + QJsonObject metadata; + metadata.insert("name", QString(getName())); + metadata.insert("category", QString("STREAMABLE")); + QJsonArray fields; + foreach (const StreamerNamePair& field, _fields) { + QJsonObject object; + writer.addTypeStreamer(field.first.data()); + object.insert("type", QString(field.first->getName())); + object.insert("name", QString(field.second)); + fields.append(object); + } + metadata.insert("fields", fields); + return metadata; +} + void GenericStreamableTypeStreamer::write(Bitstream& out, const QVariant& value) const { QVariantList values = value.toList(); for (int i = 0; i < _fields.size(); i++) { @@ -2334,6 +2528,15 @@ void GenericListTypeStreamer::writeMetadata(Bitstream& out, bool full) const { out << _valueStreamer.data(); } +QJsonValue GenericListTypeStreamer::getJSONMetadata(JSONWriter& writer) const { + QJsonObject metadata; + metadata.insert("name", QString(getName())); + metadata.insert("category", QString("LIST")); + writer.addTypeStreamer(_valueStreamer.data()); + metadata.insert("valueType", QString(_valueStreamer->getName())); + return metadata; +} + void GenericListTypeStreamer::write(Bitstream& out, const QVariant& value) const { QVariantList values = value.toList(); out << values.size(); @@ -2376,6 +2579,15 @@ GenericSetTypeStreamer::GenericSetTypeStreamer(const QByteArray& name, const Typ GenericListTypeStreamer(name, valueStreamer) { } +QJsonValue GenericSetTypeStreamer::getJSONMetadata(JSONWriter& writer) const { + QJsonObject metadata; + metadata.insert("name", QString(getName())); + metadata.insert("category", QString("SET")); + writer.addTypeStreamer(_valueStreamer.data()); + metadata.insert("valueType", QString(_valueStreamer->getName())); + return metadata; +} + TypeStreamer::Category GenericSetTypeStreamer::getCategory() const { return SET_CATEGORY; } @@ -2437,6 +2649,17 @@ void GenericMapTypeStreamer::writeMetadata(Bitstream& out, bool full) const { out << _keyStreamer.data() << _valueStreamer.data(); } +QJsonValue GenericMapTypeStreamer::getJSONMetadata(JSONWriter& writer) const { + QJsonObject metadata; + metadata.insert("name", QString(getName())); + metadata.insert("category", QString("MAP")); + writer.addTypeStreamer(_keyStreamer.data()); + metadata.insert("keyType", QString(_keyStreamer->getName())); + writer.addTypeStreamer(_valueStreamer.data()); + metadata.insert("valueType", QString(_valueStreamer->getName())); + return metadata; +} + void GenericMapTypeStreamer::write(Bitstream& out, const QVariant& value) const { QVariantPairList values = value.value(); out << values.size(); diff --git a/libraries/metavoxels/src/Bitstream.h b/libraries/metavoxels/src/Bitstream.h index 1589473b0e..732d1f8f7b 100644 --- a/libraries/metavoxels/src/Bitstream.h +++ b/libraries/metavoxels/src/Bitstream.h @@ -13,6 +13,8 @@ #define hifi_Bitstream_h #include +#include +#include #include #include #include @@ -766,6 +768,41 @@ template inline Bitstream& Bitstream::operator>>(QHash& return *this; } +/// Tracks state when writing to JSON. +class JSONWriter { +public: + + void addTypeStreamer(const TypeStreamer* streamer); + void addObjectStreamer(const ObjectStreamer* streamer); + void addSharedObject(const SharedObjectPointer& object); + +private: + + QSet _typeStreamerNames; + QJsonArray _typeStreamers; + + QSet _objectStreamerNames; + QJsonArray _objectStreamers; + + QSet _sharedObjectIDs; + QJsonArray _sharedObjects; +}; + +/// Tracks state when reading from JSON. +class JSONReader { +public: + + TypeStreamerPointer getTypeStreamer(const QByteArray& name) const { return _typeStreamers.value(name); } + ObjectStreamerPointer getObjectStreamer(const QByteArray& name) const { return _objectStreamers.value(name); } + SharedObjectPointer getSharedObject(int id) const { return _sharedObjects.value(id); } + +private: + + QHash _typeStreamers; + QHash _objectStreamers; + QHash _sharedObjects; +}; + typedef QPair StreamerPropertyPair; /// Contains the information required to stream an object. @@ -780,6 +817,8 @@ public: virtual const char* getName() const = 0; virtual const QVector& getProperties() const; virtual void writeMetadata(Bitstream& out, bool full) const = 0; + virtual QJsonObject getJSONMetadata(JSONWriter& writer) const = 0; + virtual QJsonObject getJSONData(JSONWriter& writer, const QObject* object) const = 0; virtual void write(Bitstream& out, const QObject* object) const = 0; virtual void writeRawDelta(Bitstream& out, const QObject* object, const QObject* reference) const = 0; virtual QObject* read(Bitstream& in, QObject* object = NULL) const = 0; @@ -802,6 +841,8 @@ public: virtual const char* getName() const; virtual const QVector& getProperties() const; virtual void writeMetadata(Bitstream& out, bool full) const; + virtual QJsonObject getJSONMetadata(JSONWriter& writer) const; + virtual QJsonObject getJSONData(JSONWriter& writer, const QObject* object) const; virtual void write(Bitstream& out, const QObject* object) const; virtual void writeRawDelta(Bitstream& out, const QObject* object, const QObject* reference) const; virtual QObject* read(Bitstream& in, QObject* object = NULL) const; @@ -822,6 +863,8 @@ public: virtual const char* getName() const; virtual void writeMetadata(Bitstream& out, bool full) const; + virtual QJsonObject getJSONMetadata(JSONWriter& writer) const; + virtual QJsonObject getJSONData(JSONWriter& writer, const QObject* object) const; virtual void write(Bitstream& out, const QObject* object) const; virtual void writeRawDelta(Bitstream& out, const QObject* object, const QObject* reference) const; virtual QObject* read(Bitstream& in, QObject* object = NULL) const; @@ -913,6 +956,9 @@ public: virtual void writeMetadata(Bitstream& out, bool full) const; + virtual QJsonValue getJSONMetadata(JSONWriter& writer) const; + virtual QJsonValue getJSONData(JSONWriter& writer, const QVariant& value) const; + virtual bool equal(const QVariant& first, const QVariant& second) const; virtual void write(Bitstream& out, const QVariant& value) const; @@ -990,6 +1036,7 @@ public: virtual const char* getName() const; virtual void writeMetadata(Bitstream& out, bool full) const; + virtual QJsonValue getJSONMetadata(JSONWriter& writer) const; virtual Category getCategory() const; virtual int getBits() const; virtual QMetaEnum getMetaEnum() const; @@ -1051,6 +1098,7 @@ public: GenericEnumTypeStreamer(const QByteArray& name, const QVector& values, int bits, const QByteArray& hash); virtual void writeMetadata(Bitstream& out, bool full) const; + virtual QJsonValue getJSONMetadata(JSONWriter& writer) const; virtual void write(Bitstream& out, const QVariant& value) const; virtual QVariant read(Bitstream& in) const; virtual Category getCategory() const; @@ -1099,6 +1147,7 @@ public: GenericStreamableTypeStreamer(const QByteArray& name, const QVector& fields, const QByteArray& hash); virtual void writeMetadata(Bitstream& out, bool full) const; + virtual QJsonValue getJSONMetadata(JSONWriter& writer) const; virtual void write(Bitstream& out, const QVariant& value) const; virtual QVariant read(Bitstream& in) const; virtual Category getCategory() const; @@ -1169,11 +1218,12 @@ public: GenericListTypeStreamer(const QByteArray& name, const TypeStreamerPointer& valueStreamer); virtual void writeMetadata(Bitstream& out, bool full) const; + virtual QJsonValue getJSONMetadata(JSONWriter& writer) const; virtual void write(Bitstream& out, const QVariant& value) const; virtual QVariant read(Bitstream& in) const; virtual Category getCategory() const; -private: +protected: TypeStreamerPointer _valueStreamer; }; @@ -1206,6 +1256,7 @@ public: GenericSetTypeStreamer(const QByteArray& name, const TypeStreamerPointer& valueStreamer); + virtual QJsonValue getJSONMetadata(JSONWriter& writer) const; virtual Category getCategory() const; }; @@ -1250,6 +1301,7 @@ public: const TypeStreamerPointer& valueStreamer); virtual void writeMetadata(Bitstream& out, bool full) const; + virtual QJsonValue getJSONMetadata(JSONWriter& writer) const; virtual void write(Bitstream& out, const QVariant& value) const; virtual QVariant read(Bitstream& in) const; virtual Category getCategory() const; From aac7459f42846e0c924e24a78d25c9a670b0841b Mon Sep 17 00:00:00 2001 From: Andrzej Kapolka Date: Fri, 13 Jun 2014 18:32:57 -0700 Subject: [PATCH 02/16] More work on JSON encoding. --- libraries/metavoxels/src/Bitstream.cpp | 180 +++++++++++++++++++++++- libraries/metavoxels/src/Bitstream.h | 48 ++++++- tests/metavoxels/src/MetavoxelTests.cpp | 12 ++ 3 files changed, 232 insertions(+), 8 deletions(-) diff --git a/libraries/metavoxels/src/Bitstream.cpp b/libraries/metavoxels/src/Bitstream.cpp index 1266e8efa4..aa5ed3fa5e 100644 --- a/libraries/metavoxels/src/Bitstream.cpp +++ b/libraries/metavoxels/src/Bitstream.cpp @@ -1657,6 +1657,109 @@ const TypeStreamer* Bitstream::createInvalidTypeStreamer() { return streamer; } +JSONWriter& JSONWriter::operator<<(bool value) { + _contents.append(value); + return *this; +} + +JSONWriter& JSONWriter::operator<<(int value) { + _contents.append(value); + return *this; +} + +JSONWriter& JSONWriter::operator<<(uint value) { + _contents.append((int)value); + return *this; +} + +JSONWriter& JSONWriter::operator<<(float value) { + _contents.append((double)value); + return *this; +} + +JSONWriter& JSONWriter::operator<<(const QByteArray& value) { + _contents.append(QString(value.toPercentEncoding())); + return *this; +} + +JSONWriter& JSONWriter::operator<<(const QColor& value) { + _contents.append(value.name()); + return *this; +} + +JSONWriter& JSONWriter::operator<<(const QScriptValue& value) { + return *this; +} + +JSONWriter& JSONWriter::operator<<(const QString& value) { + _contents.append(value); + return *this; +} + +JSONWriter& JSONWriter::operator<<(const QUrl& value) { + _contents.append(value.toString()); + return *this; +} + +JSONWriter& JSONWriter::operator<<(const glm::vec3& value) { + QJsonArray array; + array.append(value.x); + array.append(value.y); + array.append(value.z); + _contents.append(array); + return *this; +} + +JSONWriter& JSONWriter::operator<<(const glm::quat& value) { + QJsonArray array; + array.append(value.x); + array.append(value.y); + array.append(value.z); + array.append(value.w); + _contents.append(array); + return *this; +} + +JSONWriter& JSONWriter::operator<<(const QMetaObject* metaObject) { + if (!metaObject) { + _contents.append(QJsonValue()); + return *this; + } + const ObjectStreamer* streamer = Bitstream::getObjectStreamers().value(metaObject); + addObjectStreamer(streamer); + _contents.append(QString(streamer->getName())); + return *this; +} + +JSONWriter& JSONWriter::operator<<(const QVariant& value) { + if (!value.isValid()) { + _contents.append(QJsonValue()); + return *this; + } + const TypeStreamer* streamer = Bitstream::getTypeStreamers().value(value.userType()); + if (streamer) { + _contents.append(streamer->getJSONVariantData(*this, value)); + } else { + qWarning() << "Non-streamable type:" << value.typeName(); + } + return *this; +} + +JSONWriter& JSONWriter::operator<<(const SharedObjectPointer& object) { + if (object) { + addSharedObject(object); + _contents.append(object->getID()); + } else { + _contents.append(0); + } + return *this; +} + +JSONWriter& JSONWriter::operator<<(const QObject* object) { + _contents.append(getData(object)); + return *this; +} + void JSONWriter::addTypeStreamer(const TypeStreamer* streamer) { if (!_typeStreamerNames.contains(streamer->getName())) { _typeStreamerNames.insert(streamer->getName()); @@ -1695,11 +1798,31 @@ void JSONWriter::addSharedObject(const SharedObjectPointer& object) { QJsonObject sharedObject; sharedObject.insert("id", object->getID()); sharedObject.insert("originID", object->getOriginID()); - + sharedObject.insert("data", getData(object.data())); _sharedObjects.replace(index, sharedObject); } } +QJsonDocument JSONWriter::getDocument() const { + QJsonObject top; + top.insert("contents", _contents); + top.insert("objects", _sharedObjects); + top.insert("classes", _objectStreamers); + top.insert("types", _typeStreamers); + return QJsonDocument(top); +} + +QJsonValue JSONWriter::getData(const QObject* object) { + if (!object) { + return QJsonValue(); + } + const QMetaObject* metaObject = object->metaObject(); + const ObjectStreamer* streamer = (metaObject == &GenericSharedObject::staticMetaObject) ? + static_cast(object)->getStreamer().data() : + Bitstream::getObjectStreamers().value(metaObject); + return streamer->getJSONData(*this, object); +} + ObjectStreamer::ObjectStreamer(const QMetaObject* metaObject) : _metaObject(metaObject) { } @@ -2023,6 +2146,14 @@ QJsonValue TypeStreamer::getJSONData(JSONWriter& writer, const QVariant& value) return QJsonValue(); } +QJsonValue TypeStreamer::getJSONVariantData(JSONWriter& writer, const QVariant& value) const { + writer.addTypeStreamer(this); + QJsonObject data; + data.insert("type", QString(getName())); + data.insert("value", getJSONData(writer, value)); + return data; +} + bool TypeStreamer::equal(const QVariant& first, const QVariant& second) const { return first == second; } @@ -2208,6 +2339,10 @@ QJsonValue EnumTypeStreamer::getJSONMetadata(JSONWriter& writer) const { return metadata; } +QJsonValue EnumTypeStreamer::getJSONData(JSONWriter& writer, const QVariant& value) const { + return value.toInt(); +} + TypeStreamer::Category EnumTypeStreamer::getCategory() const { return ENUM_CATEGORY; } @@ -2372,6 +2507,10 @@ QJsonValue GenericEnumTypeStreamer::getJSONMetadata(JSONWriter& writer) const { return metadata; } +QJsonValue GenericEnumTypeStreamer::getJSONData(JSONWriter& writer, const QVariant& value) const { + return value.toInt(); +} + void GenericEnumTypeStreamer::write(Bitstream& out, const QVariant& value) const { int intValue = value.toInt(); out.write(&intValue, _bits); @@ -2464,6 +2603,15 @@ QJsonValue GenericStreamableTypeStreamer::getJSONMetadata(JSONWriter& writer) co return metadata; } +QJsonValue GenericStreamableTypeStreamer::getJSONData(JSONWriter& writer, const QVariant& value) const { + QVariantList values = value.toList(); + QJsonArray array; + for (int i = 0; i < _fields.size(); i++) { + array.append(_fields.at(i).first->getJSONData(writer, values.at(i))); + } + return array; +} + void GenericStreamableTypeStreamer::write(Bitstream& out, const QVariant& value) const { QVariantList values = value.toList(); for (int i = 0; i < _fields.size(); i++) { @@ -2537,6 +2685,15 @@ QJsonValue GenericListTypeStreamer::getJSONMetadata(JSONWriter& writer) const { return metadata; } +QJsonValue GenericListTypeStreamer::getJSONData(JSONWriter& writer, const QVariant& value) const { + QVariantList values = value.toList(); + QJsonArray array; + foreach (const QVariant& element, values) { + array.append(_valueStreamer->getJSONData(writer, element)); + } + return array; +} + void GenericListTypeStreamer::write(Bitstream& out, const QVariant& value) const { QVariantList values = value.toList(); out << values.size(); @@ -2660,6 +2817,18 @@ QJsonValue GenericMapTypeStreamer::getJSONMetadata(JSONWriter& writer) const { return metadata; } +QJsonValue GenericMapTypeStreamer::getJSONData(JSONWriter& writer, const QVariant& value) const { + QVariantPairList values = value.value(); + QJsonArray array; + foreach (const QVariantPair& pair, values) { + QJsonArray pairArray; + pairArray.append(_keyStreamer->getJSONData(writer, pair.first)); + pairArray.append(_valueStreamer->getJSONData(writer, pair.second)); + array.append(pairArray); + } + return array; +} + void GenericMapTypeStreamer::write(Bitstream& out, const QVariant& value) const { QVariantPairList values = value.value(); out << values.size(); @@ -2685,6 +2854,15 @@ TypeStreamer::Category GenericMapTypeStreamer::getCategory() const { return MAP_CATEGORY; } +QJsonValue GenericValueStreamer::getJSONVariantData(JSONWriter& writer, const QVariant& value) const { + GenericValue genericValue = value.value(); + writer.addTypeStreamer(genericValue.getStreamer().data()); + QJsonObject data; + data.insert("type", QString(genericValue.getStreamer()->getName())); + data.insert("value", genericValue.getStreamer()->getJSONData(writer, genericValue.getValue())); + return data; +} + void GenericValueStreamer::writeVariant(Bitstream& out, const QVariant& value) const { GenericValue genericValue = value.value(); out << genericValue.getStreamer().data(); diff --git a/libraries/metavoxels/src/Bitstream.h b/libraries/metavoxels/src/Bitstream.h index 732d1f8f7b..159633c810 100644 --- a/libraries/metavoxels/src/Bitstream.h +++ b/libraries/metavoxels/src/Bitstream.h @@ -14,6 +14,7 @@ #include #include +#include #include #include #include @@ -38,6 +39,7 @@ class Attribute; class AttributeValue; class Bitstream; class GenericValue; +class JSONWriter; class ObjectReader; class ObjectStreamer; class OwnedAttributeValue; @@ -447,6 +449,8 @@ private slots: private: + friend class JSONWriter; + ObjectStreamerPointer readGenericObjectStreamer(const QByteArray& name); TypeStreamerPointer readGenericTypeStreamer(const QByteArray& name, int category); @@ -771,21 +775,44 @@ template inline Bitstream& Bitstream::operator>>(QHash& /// Tracks state when writing to JSON. class JSONWriter { public: - - void addTypeStreamer(const TypeStreamer* streamer); - void addObjectStreamer(const ObjectStreamer* streamer); + + JSONWriter& operator<<(bool value); + JSONWriter& operator<<(int value); + JSONWriter& operator<<(uint value); + JSONWriter& operator<<(float value); + JSONWriter& operator<<(const QByteArray& value); + JSONWriter& operator<<(const QColor& value); + JSONWriter& operator<<(const QScriptValue& value); + JSONWriter& operator<<(const QString& value); + JSONWriter& operator<<(const QUrl& value); + JSONWriter& operator<<(const glm::vec3& value); + JSONWriter& operator<<(const glm::quat& value); + JSONWriter& operator<<(const QMetaObject* metaObject); + + JSONWriter& operator<<(const QVariant& value); + JSONWriter& operator<<(const SharedObjectPointer& object); + JSONWriter& operator<<(const QObject* object); + void addSharedObject(const SharedObjectPointer& object); + void addObjectStreamer(const ObjectStreamer* streamer); + void addTypeStreamer(const TypeStreamer* streamer); + + QJsonDocument getDocument() const; private: - QSet _typeStreamerNames; - QJsonArray _typeStreamers; + QJsonValue getData(const QObject* object); + + QJsonArray _contents; + + QSet _sharedObjectIDs; + QJsonArray _sharedObjects; QSet _objectStreamerNames; QJsonArray _objectStreamers; - QSet _sharedObjectIDs; - QJsonArray _sharedObjects; + QSet _typeStreamerNames; + QJsonArray _typeStreamers; }; /// Tracks state when reading from JSON. @@ -958,6 +985,7 @@ public: virtual QJsonValue getJSONMetadata(JSONWriter& writer) const; virtual QJsonValue getJSONData(JSONWriter& writer, const QVariant& value) const; + virtual QJsonValue getJSONVariantData(JSONWriter& writer, const QVariant& value) const; virtual bool equal(const QVariant& first, const QVariant& second) const; @@ -1037,6 +1065,7 @@ public: virtual const char* getName() const; virtual void writeMetadata(Bitstream& out, bool full) const; virtual QJsonValue getJSONMetadata(JSONWriter& writer) const; + virtual QJsonValue getJSONData(JSONWriter& writer, const QVariant& value) const; virtual Category getCategory() const; virtual int getBits() const; virtual QMetaEnum getMetaEnum() const; @@ -1099,6 +1128,7 @@ public: virtual void writeMetadata(Bitstream& out, bool full) const; virtual QJsonValue getJSONMetadata(JSONWriter& writer) const; + virtual QJsonValue getJSONData(JSONWriter& writer, const QVariant& value) const; virtual void write(Bitstream& out, const QVariant& value) const; virtual QVariant read(Bitstream& in) const; virtual Category getCategory() const; @@ -1148,6 +1178,7 @@ public: virtual void writeMetadata(Bitstream& out, bool full) const; virtual QJsonValue getJSONMetadata(JSONWriter& writer) const; + virtual QJsonValue getJSONData(JSONWriter& writer, const QVariant& value) const; virtual void write(Bitstream& out, const QVariant& value) const; virtual QVariant read(Bitstream& in) const; virtual Category getCategory() const; @@ -1219,6 +1250,7 @@ public: virtual void writeMetadata(Bitstream& out, bool full) const; virtual QJsonValue getJSONMetadata(JSONWriter& writer) const; + virtual QJsonValue getJSONData(JSONWriter& writer, const QVariant& value) const; virtual void write(Bitstream& out, const QVariant& value) const; virtual QVariant read(Bitstream& in) const; virtual Category getCategory() const; @@ -1302,6 +1334,7 @@ public: virtual void writeMetadata(Bitstream& out, bool full) const; virtual QJsonValue getJSONMetadata(JSONWriter& writer) const; + virtual QJsonValue getJSONData(JSONWriter& writer, const QVariant& value) const; virtual void write(Bitstream& out, const QVariant& value) const; virtual QVariant read(Bitstream& in) const; virtual Category getCategory() const; @@ -1316,6 +1349,7 @@ private: class GenericValueStreamer : public SimpleTypeStreamer { public: + QJsonValue getJSONVariantData(JSONWriter& writer, const QVariant& value) const; virtual void writeVariant(Bitstream& out, const QVariant& value) const; }; diff --git a/tests/metavoxels/src/MetavoxelTests.cpp b/tests/metavoxels/src/MetavoxelTests.cpp index 6bd99a6c82..618178383f 100644 --- a/tests/metavoxels/src/MetavoxelTests.cpp +++ b/tests/metavoxels/src/MetavoxelTests.cpp @@ -243,6 +243,18 @@ static bool testSerialization(Bitstream::MetadataType metadataType) { return true; } + if (metadataType != Bitstream::FULL_METADATA) { + return false; + } + + // now write to JSON + JSONWriter jsonWriter; + jsonWriter << testObjectReadA; + jsonWriter << testObjectReadB; + jsonWriter << messageRead; + qDebug() << jsonWriter.getDocument().toJson(); + qDebug(); + return false; } From f4bd2a8beb31a88a564a7d93977c7b15b4a81a9d Mon Sep 17 00:00:00 2001 From: Andrzej Kapolka Date: Sun, 15 Jun 2014 16:23:54 -0700 Subject: [PATCH 03/16] I think that basically covers the JSON writing portion. --- libraries/metavoxels/src/Bitstream.cpp | 176 ++++++++++++++++--------- libraries/metavoxels/src/Bitstream.h | 83 +++++++++--- 2 files changed, 181 insertions(+), 78 deletions(-) diff --git a/libraries/metavoxels/src/Bitstream.cpp b/libraries/metavoxels/src/Bitstream.cpp index aa5ed3fa5e..b88f536d1a 100644 --- a/libraries/metavoxels/src/Bitstream.cpp +++ b/libraries/metavoxels/src/Bitstream.cpp @@ -1657,107 +1657,172 @@ const TypeStreamer* Bitstream::createInvalidTypeStreamer() { return streamer; } -JSONWriter& JSONWriter::operator<<(bool value) { - _contents.append(value); - return *this; +QJsonValue JSONWriter::getData(bool value) { + return value; } -JSONWriter& JSONWriter::operator<<(int value) { - _contents.append(value); - return *this; +QJsonValue JSONWriter::getData(int value) { + return value; } -JSONWriter& JSONWriter::operator<<(uint value) { - _contents.append((int)value); - return *this; +QJsonValue JSONWriter::getData(uint value) { + return (int)value; } -JSONWriter& JSONWriter::operator<<(float value) { - _contents.append((double)value); - return *this; +QJsonValue JSONWriter::getData(float value) { + return (double)value; } -JSONWriter& JSONWriter::operator<<(const QByteArray& value) { - _contents.append(QString(value.toPercentEncoding())); - return *this; +QJsonValue JSONWriter::getData(const QByteArray& value) { + return QString(value.toPercentEncoding()); } -JSONWriter& JSONWriter::operator<<(const QColor& value) { - _contents.append(value.name()); - return *this; +QJsonValue JSONWriter::getData(const QColor& value) { + return value.name(); } -JSONWriter& JSONWriter::operator<<(const QScriptValue& value) { - return *this; +QJsonValue JSONWriter::getData(const QScriptValue& value) { + QJsonObject object; + if (value.isUndefined()) { + object.insert("type", QString("UNDEFINED")); + + } else if (value.isNull()) { + object.insert("type", QString("NULL")); + + } else if (value.isBool()) { + object.insert("type", QString("BOOL")); + object.insert("value", value.toBool()); + + } else if (value.isNumber()) { + object.insert("type", QString("NUMBER")); + object.insert("value", value.toNumber()); + + } else if (value.isString()) { + object.insert("type", QString("STRING")); + object.insert("value", value.toString()); + + } else if (value.isVariant()) { + object.insert("type", QString("VARIANT")); + object.insert("value", getData(value.toVariant())); + + } else if (value.isQObject()) { + object.insert("type", QString("QOBJECT")); + object.insert("value", getData(value.toQObject())); + + } else if (value.isQMetaObject()) { + object.insert("type", QString("QMETAOBJECT")); + object.insert("value", getData(value.toQMetaObject())); + + } else if (value.isDate()) { + object.insert("type", QString("DATE")); + object.insert("value", getData(value.toDateTime())); + + } else if (value.isRegExp()) { + object.insert("type", QString("REGEXP")); + object.insert("value", getData(value.toRegExp())); + + } else if (value.isArray()) { + object.insert("type", QString("ARRAY")); + QJsonArray array; + int length = value.property(ScriptCache::getInstance()->getLengthString()).toInt32(); + for (int i = 0; i < length; i++) { + array.append(getData(value.property(i))); + } + object.insert("value", array); + + } else if (value.isObject()) { + object.insert("type", QString("OBJECT")); + QJsonObject valueObject; + for (QScriptValueIterator it(value); it.hasNext(); ) { + it.next(); + valueObject.insert(it.name(), getData(it.value())); + } + object.insert("value", valueObject); + + } else { + object.insert("type", QString("INVALID")); + } + return object; } -JSONWriter& JSONWriter::operator<<(const QString& value) { - _contents.append(value); - return *this; +QJsonValue JSONWriter::getData(const QString& value) { + return value; } -JSONWriter& JSONWriter::operator<<(const QUrl& value) { - _contents.append(value.toString()); - return *this; +QJsonValue JSONWriter::getData(const QUrl& value) { + return value.toString(); } -JSONWriter& JSONWriter::operator<<(const glm::vec3& value) { +QJsonValue JSONWriter::getData(const QDateTime& value) { + return (qsreal)value.toMSecsSinceEpoch(); +} + +QJsonValue JSONWriter::getData(const QRegExp& value) { + QJsonObject object; + object.insert("pattern", value.pattern()); + object.insert("caseSensitivity", (int)value.caseSensitivity()); + object.insert("patternSyntax", (int)value.patternSyntax()); + object.insert("minimal", value.isMinimal()); + return object; +} + +QJsonValue JSONWriter::getData(const glm::vec3& value) { QJsonArray array; array.append(value.x); array.append(value.y); array.append(value.z); - _contents.append(array); - return *this; + return array; } -JSONWriter& JSONWriter::operator<<(const glm::quat& value) { +QJsonValue JSONWriter::getData(const glm::quat& value) { QJsonArray array; array.append(value.x); array.append(value.y); array.append(value.z); array.append(value.w); - _contents.append(array); - return *this; + return array; } -JSONWriter& JSONWriter::operator<<(const QMetaObject* metaObject) { +QJsonValue JSONWriter::getData(const QMetaObject* metaObject) { if (!metaObject) { - _contents.append(QJsonValue()); - return *this; + return QJsonValue(); } const ObjectStreamer* streamer = Bitstream::getObjectStreamers().value(metaObject); addObjectStreamer(streamer); - _contents.append(QString(streamer->getName())); - return *this; + return QString(streamer->getName()); } -JSONWriter& JSONWriter::operator<<(const QVariant& value) { +QJsonValue JSONWriter::getData(const QVariant& value) { if (!value.isValid()) { - _contents.append(QJsonValue()); - return *this; + return QJsonValue(); } const TypeStreamer* streamer = Bitstream::getTypeStreamers().value(value.userType()); if (streamer) { - _contents.append(streamer->getJSONVariantData(*this, value)); + return streamer->getJSONVariantData(*this, value); } else { qWarning() << "Non-streamable type:" << value.typeName(); + return QJsonValue(); } - return *this; } -JSONWriter& JSONWriter::operator<<(const SharedObjectPointer& object) { +QJsonValue JSONWriter::getData(const SharedObjectPointer& object) { if (object) { addSharedObject(object); - _contents.append(object->getID()); + return object->getID(); } else { - _contents.append(0); + return 0; } - return *this; } -JSONWriter& JSONWriter::operator<<(const QObject* object) { - _contents.append(getData(object)); - return *this; +QJsonValue JSONWriter::getData(const QObject* object) { + if (!object) { + return QJsonValue(); + } + const QMetaObject* metaObject = object->metaObject(); + const ObjectStreamer* streamer = (metaObject == &GenericSharedObject::staticMetaObject) ? + static_cast(object)->getStreamer().data() : + Bitstream::getObjectStreamers().value(metaObject); + return streamer->getJSONData(*this, object); } void JSONWriter::addTypeStreamer(const TypeStreamer* streamer) { @@ -1798,7 +1863,7 @@ void JSONWriter::addSharedObject(const SharedObjectPointer& object) { QJsonObject sharedObject; sharedObject.insert("id", object->getID()); sharedObject.insert("originID", object->getOriginID()); - sharedObject.insert("data", getData(object.data())); + sharedObject.insert("data", getData(static_cast(object.data()))); _sharedObjects.replace(index, sharedObject); } } @@ -1812,17 +1877,6 @@ QJsonDocument JSONWriter::getDocument() const { return QJsonDocument(top); } -QJsonValue JSONWriter::getData(const QObject* object) { - if (!object) { - return QJsonValue(); - } - const QMetaObject* metaObject = object->metaObject(); - const ObjectStreamer* streamer = (metaObject == &GenericSharedObject::staticMetaObject) ? - static_cast(object)->getStreamer().data() : - Bitstream::getObjectStreamers().value(metaObject); - return streamer->getJSONData(*this, object); -} - ObjectStreamer::ObjectStreamer(const QMetaObject* metaObject) : _metaObject(metaObject) { } diff --git a/libraries/metavoxels/src/Bitstream.h b/libraries/metavoxels/src/Bitstream.h index 159633c810..23943444d9 100644 --- a/libraries/metavoxels/src/Bitstream.h +++ b/libraries/metavoxels/src/Bitstream.h @@ -776,22 +776,32 @@ template inline Bitstream& Bitstream::operator>>(QHash& class JSONWriter { public: - JSONWriter& operator<<(bool value); - JSONWriter& operator<<(int value); - JSONWriter& operator<<(uint value); - JSONWriter& operator<<(float value); - JSONWriter& operator<<(const QByteArray& value); - JSONWriter& operator<<(const QColor& value); - JSONWriter& operator<<(const QScriptValue& value); - JSONWriter& operator<<(const QString& value); - JSONWriter& operator<<(const QUrl& value); - JSONWriter& operator<<(const glm::vec3& value); - JSONWriter& operator<<(const glm::quat& value); - JSONWriter& operator<<(const QMetaObject* metaObject); + QJsonValue getData(bool value); + QJsonValue getData(int value); + QJsonValue getData(uint value); + QJsonValue getData(float value); + QJsonValue getData(const QByteArray& value); + QJsonValue getData(const QColor& value); + QJsonValue getData(const QScriptValue& value); + QJsonValue getData(const QString& value); + QJsonValue getData(const QUrl& value); + QJsonValue getData(const QDateTime& value); + QJsonValue getData(const QRegExp& value); + QJsonValue getData(const glm::vec3& value); + QJsonValue getData(const glm::quat& value); + QJsonValue getData(const QMetaObject* value); + QJsonValue getData(const QVariant& value); + QJsonValue getData(const SharedObjectPointer& value); + QJsonValue getData(const QObject* value); - JSONWriter& operator<<(const QVariant& value); - JSONWriter& operator<<(const SharedObjectPointer& object); - JSONWriter& operator<<(const QObject* object); + template QJsonValue getData(const T& value); + + template QJsonValue getData(const QList& list); + template QJsonValue getData(const QVector& list); + template QJsonValue getData(const QSet& set); + template QJsonValue getData(const QHash& hash); + + template JSONWriter& operator<<(const T& value) { _contents.append(getData(value)); return *this; } void addSharedObject(const SharedObjectPointer& object); void addObjectStreamer(const ObjectStreamer* streamer); @@ -801,8 +811,6 @@ public: private: - QJsonValue getData(const QObject* object); - QJsonArray _contents; QSet _sharedObjectIDs; @@ -815,6 +823,45 @@ private: QJsonArray _typeStreamers; }; +template inline QJsonValue JSONWriter::getData(const T& value) { + return QJsonValue(); +} + +template inline QJsonValue JSONWriter::getData(const QList& list) { + QJsonArray array; + foreach (const T& value, list) { + array.append(getData(value)); + } + return array; +} + +template inline QJsonValue JSONWriter::getData(const QVector& vector) { + QJsonArray array; + foreach (const T& value, vector) { + array.append(getData(value)); + } + return array; +} + +template inline QJsonValue JSONWriter::getData(const QSet& set) { + QJsonArray array; + foreach (const T& value, set) { + array.append(getData(value)); + } + return array; +} + +template inline QJsonValue JSONWriter::getData(const QHash& hash) { + QJsonArray array; + for (typename QHash::const_iterator it = hash.constBegin(); it != hash.constEnd(); it++) { + QJsonArray pair; + pair.append(getData(it.key())); + pair.append(getData(it.value())); + array.append(pair); + } + return array; +} + /// Tracks state when reading from JSON. class JSONReader { public: @@ -1042,6 +1089,8 @@ QDebug& operator<<(QDebug& debug, const QMetaObject* metaObject); template class SimpleTypeStreamer : public TypeStreamer { public: + virtual QJsonValue getJSONData(JSONWriter& writer, const QVariant& value) const { + return writer.getData(value.value()); } virtual bool equal(const QVariant& first, const QVariant& second) const { return first.value() == second.value(); } virtual void write(Bitstream& out, const QVariant& value) const { out << value.value(); } virtual QVariant read(Bitstream& in) const { T value; in >> value; return QVariant::fromValue(value); } From 3ec127afde9eaf23fae65779f6c7633ae19623a9 Mon Sep 17 00:00:00 2001 From: Andrzej Kapolka Date: Sun, 15 Jun 2014 16:46:43 -0700 Subject: [PATCH 04/16] Spoke a little too soon. --- libraries/metavoxels/src/Bitstream.h | 6 +++++- tools/mtc/src/main.cpp | 13 +++++++++++++ 2 files changed, 18 insertions(+), 1 deletion(-) diff --git a/libraries/metavoxels/src/Bitstream.h b/libraries/metavoxels/src/Bitstream.h index 23943444d9..e9599e9e87 100644 --- a/libraries/metavoxels/src/Bitstream.h +++ b/libraries/metavoxels/src/Bitstream.h @@ -1418,6 +1418,7 @@ public: Bitstream& operator>>(Bitstream& in, X& obj); \ template<> void Bitstream::writeRawDelta(const X& value, const X& reference); \ template<> void Bitstream::readRawDelta(X& value, const X& reference); \ + template<> QJsonValue JSONWriter::getData(const X& value); \ bool operator==(const X& first, const X& second); \ bool operator!=(const X& first, const X& second); \ static const int* _TypePtr##X = &X::Type; @@ -1427,6 +1428,7 @@ public: Bitstream& operator>>(Bitstream& in, X& obj); \ template<> void Bitstream::writeRawDelta(const X& value, const X& reference); \ template<> void Bitstream::readRawDelta(X& value, const X& reference); \ + template<> QJsonValue JSONWriter::getData(const X& value); \ bool operator==(const X& first, const X& second); \ bool operator!=(const X& first, const X& second); \ __attribute__((unused)) static const int* _TypePtr##X = &X::Type; @@ -1437,6 +1439,7 @@ public: Bitstream& operator>>(Bitstream& in, X& obj); \ template<> void Bitstream::writeRawDelta(const X& value, const X& reference); \ template<> void Bitstream::readRawDelta(X& value, const X& reference); \ + template<> QJsonValue JSONWriter::getData(const X& value); \ bool operator==(const X& first, const X& second); \ bool operator!=(const X& first, const X& second); \ static const int* _TypePtr##X = &X::Type; \ @@ -1447,7 +1450,8 @@ public: Bitstream& operator<<(Bitstream& out, const S::N& obj); \ Bitstream& operator>>(Bitstream& in, S::N& obj); \ template<> inline void Bitstream::writeRawDelta(const S::N& value, const S::N& reference) { *this << value; } \ - template<> inline void Bitstream::readRawDelta(S::N& value, const S::N& reference) { *this >> value; } + template<> inline void Bitstream::readRawDelta(S::N& value, const S::N& reference) { *this >> value; } \ + template<> inline QJsonValue JSONWriter::getData(const S::N& value) { return (int)value; } #define IMPLEMENT_ENUM_METATYPE(S, N) \ static int S##N##MetaTypeId = registerEnumMetaType(&S::staticMetaObject, #N); \ diff --git a/tools/mtc/src/main.cpp b/tools/mtc/src/main.cpp index 096ade4625..eadccdb310 100644 --- a/tools/mtc/src/main.cpp +++ b/tools/mtc/src/main.cpp @@ -217,6 +217,19 @@ void generateOutput (QTextStream& out, const QList& streamables) { } out << "}\n"; + out << "template<> QJsonValue JSONWriter::getData(const " << name << "& value) {\n"; + out << " QJsonArray array;\n"; + foreach (const QString& base, str.clazz.bases) { + out << " foreach (const QJsonValue& element, getData(static_cast(value)).toArray()) {\n"; + out << " array.append(element);\n"; + out << " }\n"; + } + foreach (const Field& field, str.fields) { + out << " array.append(getData(value." << field.name << "));\n"; + } + out << " return array;\n"; + out << "}\n"; + out << "bool operator==(const " << name << "& first, const " << name << "& second) {\n"; if (str.clazz.bases.isEmpty() && str.fields.isEmpty()) { out << " return true"; From 24ec441df76cc498643b03221b5d1f175c81c745 Mon Sep 17 00:00:00 2001 From: Andrzej Kapolka Date: Sun, 15 Jun 2014 17:57:09 -0700 Subject: [PATCH 05/16] Starting to work on JSON reading. --- libraries/metavoxels/src/Bitstream.cpp | 43 ++++++++++++++++++++++++++ libraries/metavoxels/src/Bitstream.h | 10 +++--- 2 files changed, 49 insertions(+), 4 deletions(-) diff --git a/libraries/metavoxels/src/Bitstream.cpp b/libraries/metavoxels/src/Bitstream.cpp index b88f536d1a..91e2cd8723 100644 --- a/libraries/metavoxels/src/Bitstream.cpp +++ b/libraries/metavoxels/src/Bitstream.cpp @@ -1877,6 +1877,49 @@ QJsonDocument JSONWriter::getDocument() const { return QJsonDocument(top); } +JSONReader::JSONReader(const QJsonDocument& document) { + QJsonObject top = document.object(); + QJsonArray types = top.value("types").toArray(); + for (int i = types.size() - 1; i >= 0; i--) { + QJsonObject type = types.at(i).toObject(); + QString name = type.value("name").toString(); + QString category = type.value("category").toString(); + if (category == "ENUM") { + + } else if (category == "STREAMABLE") { + + } else if (category == "LIST") { + + } else if (category == "SET") { + + } else if (category == "MAP") { + + } + } + + QJsonArray classes = top.value("classes").toArray(); + for (int i = classes.size() - 1; i >= 0; i--) { + QJsonObject clazz = classes.at(i).toObject(); + QString name = clazz.value("name").toString(); + QJsonArray properties = clazz.value("properties").toArray(); + foreach (const QJsonValue& property, properties) { + QJsonObject object = property.toObject(); + object.value("type"); + object.value("name"); + } + } + + QJsonArray objects = top.value("objects").toArray(); + for (int i = objects.size() - 1; i >= 0; i--) { + QJsonObject object = objects.at(i).toObject(); + int id = object.value("id").toInt(); + int originID = object.value("originID").toInt(); + QJsonObject data = object.value("data").toObject(); + + } + +} + ObjectStreamer::ObjectStreamer(const QMetaObject* metaObject) : _metaObject(metaObject) { } diff --git a/libraries/metavoxels/src/Bitstream.h b/libraries/metavoxels/src/Bitstream.h index e9599e9e87..afacbf6dac 100644 --- a/libraries/metavoxels/src/Bitstream.h +++ b/libraries/metavoxels/src/Bitstream.h @@ -866,14 +866,16 @@ template inline QJsonValue JSONWriter::getData(const QHash _typeStreamers; - QHash _objectStreamers; + QHash _typeStreamers; + QHash _objectStreamers; QHash _sharedObjects; }; From 77b6a209abeafd059af2148b2eac12254841fa09 Mon Sep 17 00:00:00 2001 From: Andrzej Kapolka Date: Mon, 16 Jun 2014 14:51:22 -0700 Subject: [PATCH 06/16] More progress on json reading. --- libraries/metavoxels/src/Bitstream.cpp | 272 ++++++++++++++++++++++++- libraries/metavoxels/src/Bitstream.h | 103 +++++++++- tools/mtc/src/main.cpp | 13 ++ 3 files changed, 379 insertions(+), 9 deletions(-) diff --git a/libraries/metavoxels/src/Bitstream.cpp b/libraries/metavoxels/src/Bitstream.cpp index 91e2cd8723..8f880a1803 100644 --- a/libraries/metavoxels/src/Bitstream.cpp +++ b/libraries/metavoxels/src/Bitstream.cpp @@ -1825,6 +1825,10 @@ QJsonValue JSONWriter::getData(const QObject* object) { return streamer->getJSONData(*this, object); } +QJsonValue JSONWriter::getData(const GenericValue& value) { + return value.getStreamer()->getJSONData(*this, value.getValue()); +} + void JSONWriter::addTypeStreamer(const TypeStreamer* streamer) { if (!_typeStreamerNames.contains(streamer->getName())) { _typeStreamerNames.insert(streamer->getName()); @@ -1877,7 +1881,7 @@ QJsonDocument JSONWriter::getDocument() const { return QJsonDocument(top); } -JSONReader::JSONReader(const QJsonDocument& document) { +JSONReader::JSONReader(const QJsonDocument& document, Bitstream::GenericsMode genericsMode) { QJsonObject top = document.object(); QJsonArray types = top.value("types").toArray(); for (int i = types.size() - 1; i >= 0; i--) { @@ -1918,6 +1922,155 @@ JSONReader::JSONReader(const QJsonDocument& document) { } + _contents = top.value("contents").toArray(); + _contentsIterator = _contents.constBegin(); +} + +void JSONReader::putData(const QJsonValue& data, bool& value) { + value = data.toBool(); +} + +void JSONReader::putData(const QJsonValue& data, int& value) { + value = data.toInt(); +} + +void JSONReader::putData(const QJsonValue& data, uint& value) { + value = data.toInt(); +} + +void JSONReader::putData(const QJsonValue& data, float& value) { + value = data.toDouble(); +} + +void JSONReader::putData(const QJsonValue& data, QByteArray& value) { + value = QByteArray::fromPercentEncoding(data.toString().toLatin1()); +} + +void JSONReader::putData(const QJsonValue& data, QColor& value) { + value.setNamedColor(data.toString()); +} + +void JSONReader::putData(const QJsonValue& data, QScriptValue& value) { + QJsonObject object = data.toObject(); + QString type = object.value("type").toString(); + if (type == "UNDEFINED") { + value = QScriptValue(QScriptValue::UndefinedValue); + + } else if (type == "NULL") { + value = QScriptValue(QScriptValue::NullValue); + + } else if (type == "BOOL") { + value = QScriptValue(object.value("value").toBool()); + + } else if (type == "NUMBER") { + value = QScriptValue(object.value("value").toDouble()); + + } else if (type == "STRING") { + value = QScriptValue(object.value("value").toString()); + + } else if (type == "VARIANT") { + QVariant variant; + putData(object.value("value"), variant); + value = ScriptCache::getInstance()->getEngine()->newVariant(variant); + + } else if (type == "QOBJECT") { + QObject* qObject; + putData(object.value("value"), qObject); + value = ScriptCache::getInstance()->getEngine()->newQObject(qObject, QScriptEngine::ScriptOwnership); + + } else if (type == "QMETAOBJECT") { + const QMetaObject* metaObject; + putData(object.value("value"), metaObject); + value = ScriptCache::getInstance()->getEngine()->newQMetaObject(metaObject); + + } else if (type == "DATE") { + QDateTime dateTime; + putData(object.value("value"), dateTime); + value = ScriptCache::getInstance()->getEngine()->newDate(dateTime); + + } else if (type == "REGEXP") { + QRegExp regExp; + putData(object.value("value"), regExp); + value = ScriptCache::getInstance()->getEngine()->newRegExp(regExp); + + } else if (type == "ARRAY") { + QJsonArray array = object.value("value").toArray(); + value = ScriptCache::getInstance()->getEngine()->newArray(array.size()); + for (int i = 0; i < array.size(); i++) { + QScriptValue element; + putData(array.at(i), element); + value.setProperty(i, element); + } + } else if (type == "OBJECT") { + QJsonObject jsonObject = object.value("value").toObject(); + value = ScriptCache::getInstance()->getEngine()->newObject(); + for (QJsonObject::const_iterator it = jsonObject.constBegin(); it != jsonObject.constEnd(); it++) { + QScriptValue element; + putData(it.value(), element); + value.setProperty(it.key(), element); + } + } else { + value = QScriptValue(); + } +} + +void JSONReader::putData(const QJsonValue& data, QString& value) { + value = data.toString(); +} + +void JSONReader::putData(const QJsonValue& data, QUrl& value) { + value = data.toString(); +} + +void JSONReader::putData(const QJsonValue& data, QDateTime& value) { + value.setMSecsSinceEpoch((qint64)data.toDouble()); +} + +void JSONReader::putData(const QJsonValue& data, QRegExp& value) { + QJsonObject object = data.toObject(); + value = QRegExp(object.value("pattern").toString(), (Qt::CaseSensitivity)object.value("caseSensitivity").toInt(), + (QRegExp::PatternSyntax)object.value("patternSyntax").toInt()); + value.setMinimal(object.value("minimal").toBool()); +} + +void JSONReader::putData(const QJsonValue& data, glm::vec3& value) { + QJsonArray array = data.toArray(); + value = glm::vec3(array.at(0).toDouble(), array.at(1).toDouble(), array.at(2).toDouble()); +} + +void JSONReader::putData(const QJsonValue& data, glm::quat& value) { + QJsonArray array = data.toArray(); + value = glm::quat(array.at(0).toDouble(), array.at(1).toDouble(), array.at(2).toDouble(), array.at(3).toDouble()); +} + +void JSONReader::putData(const QJsonValue& data, const QMetaObject*& value) { + ObjectStreamerPointer streamer = _objectStreamers.value(data.toString()); + value = streamer ? streamer->getMetaObject() : NULL; +} + +void JSONReader::putData(const QJsonValue& data, QVariant& value) { + QJsonObject object = data.toObject(); + QString type = object.value("type").toString(); + const TypeStreamer* streamer = _typeStreamers.value(type).data(); + if (!streamer) { + streamer = Bitstream::getTypeStreamers().value(QMetaType::type(type.toLatin1())); + if (!streamer) { + qWarning() << "Unknown type:" << type; + value = QVariant(); + return; + } + } + streamer->putJSONData(*this, object.value("value"), value); +} + +void JSONReader::putData(const QJsonValue& data, SharedObjectPointer& value) { + value = _sharedObjects.value(data.toInt()); +} + +void JSONReader::putData(const QJsonValue& data, QObject*& value) { + QJsonObject object = data.toObject(); + ObjectStreamerPointer streamer = _objectStreamers.value(object.value("class").toString()); + value = streamer ? streamer->putJSONData(*this, object) : NULL; } ObjectStreamer::ObjectStreamer(const QMetaObject* metaObject) : @@ -1989,6 +2142,23 @@ QJsonObject MappedObjectStreamer::getJSONData(JSONWriter& writer, const QObject* return data; } +QObject* MappedObjectStreamer::putJSONData(JSONReader& reader, const QJsonObject& jsonObject) const { + if (!_metaObject) { + return NULL; + } + QObject* object = _metaObject->newInstance(); + QJsonArray properties = jsonObject.value("properties").toArray(); + for (int i = 0; i < _properties.size(); i++) { + const StreamerPropertyPair& property = _properties.at(i); + if (property.second.isValid()) { + QVariant value; + property.first->putJSONData(reader, properties.at(i), value); + property.second.write(object, value); + } + } + return object; +} + void MappedObjectStreamer::write(Bitstream& out, const QObject* object) const { foreach (const StreamerPropertyPair& property, _properties) { property.first->write(out, property.second.read(object)); @@ -2093,6 +2263,19 @@ QJsonObject GenericObjectStreamer::getJSONData(JSONWriter& writer, const QObject return data; } +QObject* GenericObjectStreamer::putJSONData(JSONReader& reader, const QJsonObject& jsonObject) const { + GenericSharedObject* object = new GenericSharedObject(_weakSelf); + QJsonArray properties = jsonObject.value("properties").toArray(); + QVariantList values; + for (int i = 0; i < _properties.size(); i++) { + QVariant value; + _properties.at(i).first->putJSONData(reader, properties.at(i), value); + values.append(value); + } + object->setValues(values); + return object; +} + void GenericObjectStreamer::write(Bitstream& out, const QObject* object) const { const QVariantList& values = static_cast(object)->getValues(); for (int i = 0; i < _properties.size(); i++) { @@ -2251,6 +2434,10 @@ QJsonValue TypeStreamer::getJSONVariantData(JSONWriter& writer, const QVariant& return data; } +void TypeStreamer::putJSONData(JSONReader& reader, const QJsonValue& data, QVariant& value) const { + value = QVariant(); +} + bool TypeStreamer::equal(const QVariant& first, const QVariant& second) const { return first == second; } @@ -2440,6 +2627,10 @@ QJsonValue EnumTypeStreamer::getJSONData(JSONWriter& writer, const QVariant& val return value.toInt(); } +void EnumTypeStreamer::putJSONData(JSONReader& reader, const QJsonValue& data, QVariant& value) const { + value = data.toInt(); +} + TypeStreamer::Category EnumTypeStreamer::getCategory() const { return ENUM_CATEGORY; } @@ -2532,6 +2723,11 @@ MappedEnumTypeStreamer::MappedEnumTypeStreamer(const TypeStreamer* baseStreamer, _mappings(mappings) { } +void MappedEnumTypeStreamer::putJSONData(JSONReader& reader, const QJsonValue& data, QVariant& value) const { + value = QVariant(_baseStreamer->getType(), 0); + _baseStreamer->setEnumValue(value, data.toInt(), _mappings); +} + QVariant MappedEnumTypeStreamer::read(Bitstream& in) const { QVariant object = QVariant(_baseStreamer->getType(), 0); int value = 0; @@ -2608,6 +2804,10 @@ QJsonValue GenericEnumTypeStreamer::getJSONData(JSONWriter& writer, const QVaria return value.toInt(); } +void GenericEnumTypeStreamer::putJSONData(JSONReader& reader, const QJsonValue& data, QVariant& value) const { + value = data.toInt(); +} + void GenericEnumTypeStreamer::write(Bitstream& out, const QVariant& value) const { int intValue = value.toInt(); out.write(&intValue, _bits); @@ -2629,6 +2829,19 @@ MappedStreamableTypeStreamer::MappedStreamableTypeStreamer(const TypeStreamer* b _fields(fields) { } +void MappedStreamableTypeStreamer::putJSONData(JSONReader& reader, const QJsonValue& data, QVariant& value) const { + value = QVariant(_baseStreamer->getType(), 0); + QJsonArray array = data.toArray(); + for (int i = 0; i < _fields.size(); i++) { + const StreamerIndexPair& pair = _fields.at(i); + if (pair.second != -1) { + QVariant element; + pair.first->putJSONData(reader, array.at(i), element); + _baseStreamer->setField(value, pair.second, element); + } + } +} + QVariant MappedStreamableTypeStreamer::read(Bitstream& in) const { QVariant object = QVariant(_baseStreamer->getType(), 0); foreach (const StreamerIndexPair& pair, _fields) { @@ -2709,6 +2922,17 @@ QJsonValue GenericStreamableTypeStreamer::getJSONData(JSONWriter& writer, const return array; } +void GenericStreamableTypeStreamer::putJSONData(JSONReader& reader, const QJsonValue& data, QVariant& value) const { + QJsonArray array = data.toArray(); + QVariantList values; + for (int i = 0; i < _fields.size(); i++) { + QVariant element; + _fields.at(i).first->putJSONData(reader, array.at(i), element); + values.append(element); + } + value = values; +} + void GenericStreamableTypeStreamer::write(Bitstream& out, const QVariant& value) const { QVariantList values = value.toList(); for (int i = 0; i < _fields.size(); i++) { @@ -2733,6 +2957,15 @@ MappedListTypeStreamer::MappedListTypeStreamer(const TypeStreamer* baseStreamer, _valueStreamer(valueStreamer) { } +void MappedListTypeStreamer::putJSONData(JSONReader& reader, const QJsonValue& data, QVariant& value) const { + value = QVariant(_baseStreamer->getType(), 0); + foreach (const QJsonValue& element, data.toArray()) { + QVariant elementValue; + _valueStreamer->putJSONData(reader, element, elementValue); + _baseStreamer->insert(value, elementValue); + } +} + QVariant MappedListTypeStreamer::read(Bitstream& in) const { QVariant object = QVariant(_baseStreamer->getType(), 0); int size; @@ -2791,6 +3024,17 @@ QJsonValue GenericListTypeStreamer::getJSONData(JSONWriter& writer, const QVaria return array; } +void GenericListTypeStreamer::putJSONData(JSONReader& reader, const QJsonValue& data, QVariant& value) const { + QJsonArray array = data.toArray(); + QVariantList values; + foreach (const QJsonValue& element, array) { + QVariant elementValue; + _valueStreamer->putJSONData(reader, element, elementValue); + values.append(elementValue); + } + value = values; +} + void GenericListTypeStreamer::write(Bitstream& out, const QVariant& value) const { QVariantList values = value.toList(); out << values.size(); @@ -2853,6 +3097,18 @@ MappedMapTypeStreamer::MappedMapTypeStreamer(const TypeStreamer* baseStreamer, c _valueStreamer(valueStreamer) { } +void MappedMapTypeStreamer::putJSONData(JSONReader& reader, const QJsonValue& data, QVariant& value) const { + value = QVariant(_baseStreamer->getType(), 0); + foreach (const QJsonValue& element, data.toArray()) { + QJsonArray pair = element.toArray(); + QVariant elementKey; + _keyStreamer->putJSONData(reader, pair.at(0), elementKey); + QVariant elementValue; + _valueStreamer->putJSONData(reader, pair.at(1), elementValue); + _baseStreamer->insert(value, elementKey, elementValue); + } +} + QVariant MappedMapTypeStreamer::read(Bitstream& in) const { QVariant object = QVariant(_baseStreamer->getType(), 0); int size; @@ -2926,6 +3182,20 @@ QJsonValue GenericMapTypeStreamer::getJSONData(JSONWriter& writer, const QVarian return array; } +void GenericMapTypeStreamer::putJSONData(JSONReader& reader, const QJsonValue& data, QVariant& value) const { + QJsonArray array = data.toArray(); + QVariantPairList values; + foreach (const QJsonValue& element, array) { + QJsonArray pair = element.toArray(); + QVariant elementKey; + _keyStreamer->putJSONData(reader, pair.at(0), elementKey); + QVariant elementValue; + _valueStreamer->putJSONData(reader, pair.at(1), elementValue); + values.append(QVariantPair(elementKey, elementValue)); + } + value = QVariant::fromValue(values); +} + void GenericMapTypeStreamer::write(Bitstream& out, const QVariant& value) const { QVariantPairList values = value.value(); out << values.size(); diff --git a/libraries/metavoxels/src/Bitstream.h b/libraries/metavoxels/src/Bitstream.h index afacbf6dac..0cad33acea 100644 --- a/libraries/metavoxels/src/Bitstream.h +++ b/libraries/metavoxels/src/Bitstream.h @@ -449,6 +449,7 @@ private slots: private: + friend class JSONReader; friend class JSONWriter; ObjectStreamerPointer readGenericObjectStreamer(const QByteArray& name); @@ -477,7 +478,7 @@ private: static QHash& getMetaObjects(); static QMultiHash& getMetaObjectSubClasses(); static QHash& getTypeStreamers(); - + static const QHash& getObjectStreamers(); static QHash createObjectStreamers(); @@ -793,8 +794,9 @@ public: QJsonValue getData(const QVariant& value); QJsonValue getData(const SharedObjectPointer& value); QJsonValue getData(const QObject* value); + QJsonValue getData(const GenericValue& value); - template QJsonValue getData(const T& value); + template QJsonValue getData(const T& value) { return QJsonValue(); } template QJsonValue getData(const QList& list); template QJsonValue getData(const QVector& list); @@ -823,10 +825,6 @@ private: QJsonArray _typeStreamers; }; -template inline QJsonValue JSONWriter::getData(const T& value) { - return QJsonValue(); -} - template inline QJsonValue JSONWriter::getData(const QList& list) { QJsonArray array; foreach (const T& value, list) { @@ -866,19 +864,88 @@ template inline QJsonValue JSONWriter::getData(const QHash void putData(const QJsonValue& data, T& value) { value = T(); } + + template void putData(const QJsonValue& data, QList& list); + template void putData(const QJsonValue& data, QVector& list); + template void putData(const QJsonValue& data, QSet& set); + template void putData(const QJsonValue& data, QHash& hash); + + template JSONReader& operator>>(T& value) { putData(*_contentsIterator++, value); return *this; } + TypeStreamerPointer getTypeStreamer(const QString& name) const { return _typeStreamers.value(name); } ObjectStreamerPointer getObjectStreamer(const QString& name) const { return _objectStreamers.value(name); } SharedObjectPointer getSharedObject(int id) const { return _sharedObjects.value(id); } private: + QJsonArray _contents; + QJsonArray::const_iterator _contentsIterator; + QHash _typeStreamers; QHash _objectStreamers; QHash _sharedObjects; }; +template inline void JSONReader::putData(const QJsonValue& data, QList& list) { + list.clear(); + foreach (const QJsonValue& element, data.toArray()) { + T value; + putData(element, value); + list.append(value); + } +} + +template inline void JSONReader::putData(const QJsonValue& data, QVector& list) { + list.clear(); + foreach (const QJsonValue& element, data.toArray()) { + T value; + putData(element, value); + list.append(value); + } +} + +template inline void JSONReader::putData(const QJsonValue& data, QSet& set) { + set.clear(); + foreach (const QJsonValue& element, data.toArray()) { + T value; + putData(element, value); + set.insert(value); + } +} + +template inline void JSONReader::putData(const QJsonValue& data, QHash& hash) { + hash.clear(); + foreach (const QJsonValue& element, data.toArray()) { + QJsonArray pair = element.toArray(); + K key; + putData(pair.at(0), key); + V value; + putData(pair.at(1), value); + hash.insert(key, value); + } +} + typedef QPair StreamerPropertyPair; /// Contains the information required to stream an object. @@ -895,6 +962,8 @@ public: virtual void writeMetadata(Bitstream& out, bool full) const = 0; virtual QJsonObject getJSONMetadata(JSONWriter& writer) const = 0; virtual QJsonObject getJSONData(JSONWriter& writer, const QObject* object) const = 0; + virtual QObject* putJSONData(JSONReader& reader, const QJsonObject& jsonObject) const = 0; + virtual void write(Bitstream& out, const QObject* object) const = 0; virtual void writeRawDelta(Bitstream& out, const QObject* object, const QObject* reference) const = 0; virtual QObject* read(Bitstream& in, QObject* object = NULL) const = 0; @@ -919,6 +988,7 @@ public: virtual void writeMetadata(Bitstream& out, bool full) const; virtual QJsonObject getJSONMetadata(JSONWriter& writer) const; virtual QJsonObject getJSONData(JSONWriter& writer, const QObject* object) const; + virtual QObject* putJSONData(JSONReader& reader, const QJsonObject& jsonObject) const; virtual void write(Bitstream& out, const QObject* object) const; virtual void writeRawDelta(Bitstream& out, const QObject* object, const QObject* reference) const; virtual QObject* read(Bitstream& in, QObject* object = NULL) const; @@ -941,6 +1011,7 @@ public: virtual void writeMetadata(Bitstream& out, bool full) const; virtual QJsonObject getJSONMetadata(JSONWriter& writer) const; virtual QJsonObject getJSONData(JSONWriter& writer, const QObject* object) const; + virtual QObject* putJSONData(JSONReader& reader, const QJsonObject& jsonObject) const; virtual void write(Bitstream& out, const QObject* object) const; virtual void writeRawDelta(Bitstream& out, const QObject* object, const QObject* reference) const; virtual QObject* read(Bitstream& in, QObject* object = NULL) const; @@ -1035,6 +1106,7 @@ public: virtual QJsonValue getJSONMetadata(JSONWriter& writer) const; virtual QJsonValue getJSONData(JSONWriter& writer, const QVariant& value) const; virtual QJsonValue getJSONVariantData(JSONWriter& writer, const QVariant& value) const; + virtual void putJSONData(JSONReader& reader, const QJsonValue& data, QVariant& value) const; virtual bool equal(const QVariant& first, const QVariant& second) const; @@ -1093,6 +1165,8 @@ public: virtual QJsonValue getJSONData(JSONWriter& writer, const QVariant& value) const { return writer.getData(value.value()); } + virtual void putJSONData(JSONReader& reader, const QJsonValue& data, QVariant& value) const { + T rawValue; reader.putData(data, rawValue); value = QVariant::fromValue(rawValue); } virtual bool equal(const QVariant& first, const QVariant& second) const { return first.value() == second.value(); } virtual void write(Bitstream& out, const QVariant& value) const { out << value.value(); } virtual QVariant read(Bitstream& in) const { T value; in >> value; return QVariant::fromValue(value); } @@ -1117,6 +1191,7 @@ public: virtual void writeMetadata(Bitstream& out, bool full) const; virtual QJsonValue getJSONMetadata(JSONWriter& writer) const; virtual QJsonValue getJSONData(JSONWriter& writer, const QVariant& value) const; + virtual void putJSONData(JSONReader& reader, const QJsonValue& data, QVariant& value) const; virtual Category getCategory() const; virtual int getBits() const; virtual QMetaEnum getMetaEnum() const; @@ -1144,6 +1219,7 @@ public: MappedEnumTypeStreamer(const TypeStreamer* baseStreamer, int bits, const QHash& mappings); + virtual void putJSONData(JSONReader& reader, const QJsonValue& data, QVariant& value) const; virtual QVariant read(Bitstream& in) const; virtual void readRawDelta(Bitstream& in, QVariant& object, const QVariant& reference) const; @@ -1180,6 +1256,7 @@ public: virtual void writeMetadata(Bitstream& out, bool full) const; virtual QJsonValue getJSONMetadata(JSONWriter& writer) const; virtual QJsonValue getJSONData(JSONWriter& writer, const QVariant& value) const; + virtual void putJSONData(JSONReader& reader, const QJsonValue& data, QVariant& value) const; virtual void write(Bitstream& out, const QVariant& value) const; virtual QVariant read(Bitstream& in) const; virtual Category getCategory() const; @@ -1212,6 +1289,7 @@ public: MappedStreamableTypeStreamer(const TypeStreamer* baseStreamer, const QVector& fields); + virtual void putJSONData(JSONReader& reader, const QJsonValue& data, QVariant& value) const; virtual QVariant read(Bitstream& in) const; virtual void readRawDelta(Bitstream& in, QVariant& object, const QVariant& reference) const; @@ -1230,6 +1308,7 @@ public: virtual void writeMetadata(Bitstream& out, bool full) const; virtual QJsonValue getJSONMetadata(JSONWriter& writer) const; virtual QJsonValue getJSONData(JSONWriter& writer, const QVariant& value) const; + virtual void putJSONData(JSONReader& reader, const QJsonValue& data, QVariant& value) const; virtual void write(Bitstream& out, const QVariant& value) const; virtual QVariant read(Bitstream& in) const; virtual Category getCategory() const; @@ -1284,6 +1363,7 @@ public: MappedListTypeStreamer(const TypeStreamer* baseStreamer, const TypeStreamerPointer& valueStreamer); + virtual void putJSONData(JSONReader& reader, const QJsonValue& data, QVariant& value) const; virtual QVariant read(Bitstream& in) const; virtual void readRawDelta(Bitstream& in, QVariant& object, const QVariant& reference) const; @@ -1302,6 +1382,7 @@ public: virtual void writeMetadata(Bitstream& out, bool full) const; virtual QJsonValue getJSONMetadata(JSONWriter& writer) const; virtual QJsonValue getJSONData(JSONWriter& writer, const QVariant& value) const; + virtual void putJSONData(JSONReader& reader, const QJsonValue& data, QVariant& value) const; virtual void write(Bitstream& out, const QVariant& value) const; virtual QVariant read(Bitstream& in) const; virtual Category getCategory() const; @@ -1366,6 +1447,7 @@ public: MappedMapTypeStreamer(const TypeStreamer* baseStreamer, const TypeStreamerPointer& keyStreamer, const TypeStreamerPointer& valueStreamer); + virtual void putJSONData(JSONReader& reader, const QJsonValue& data, QVariant& value) const; virtual QVariant read(Bitstream& in) const; virtual void readRawDelta(Bitstream& in, QVariant& object, const QVariant& reference) const; @@ -1386,6 +1468,7 @@ public: virtual void writeMetadata(Bitstream& out, bool full) const; virtual QJsonValue getJSONMetadata(JSONWriter& writer) const; virtual QJsonValue getJSONData(JSONWriter& writer, const QVariant& value) const; + virtual void putJSONData(JSONReader& reader, const QJsonValue& data, QVariant& value) const; virtual void write(Bitstream& out, const QVariant& value) const; virtual QVariant read(Bitstream& in) const; virtual Category getCategory() const; @@ -1421,6 +1504,7 @@ public: template<> void Bitstream::writeRawDelta(const X& value, const X& reference); \ template<> void Bitstream::readRawDelta(X& value, const X& reference); \ template<> QJsonValue JSONWriter::getData(const X& value); \ + template<> void JSONReader::putData(const QJsonValue& data, X& value); \ bool operator==(const X& first, const X& second); \ bool operator!=(const X& first, const X& second); \ static const int* _TypePtr##X = &X::Type; @@ -1431,6 +1515,7 @@ public: template<> void Bitstream::writeRawDelta(const X& value, const X& reference); \ template<> void Bitstream::readRawDelta(X& value, const X& reference); \ template<> QJsonValue JSONWriter::getData(const X& value); \ + template<> void JSONReader::putData(const QJsonValue& data, X& value); \ bool operator==(const X& first, const X& second); \ bool operator!=(const X& first, const X& second); \ __attribute__((unused)) static const int* _TypePtr##X = &X::Type; @@ -1442,6 +1527,7 @@ public: template<> void Bitstream::writeRawDelta(const X& value, const X& reference); \ template<> void Bitstream::readRawDelta(X& value, const X& reference); \ template<> QJsonValue JSONWriter::getData(const X& value); \ + template<> void JSONReader::putData(const QJsonValue& data, X& value); \ bool operator==(const X& first, const X& second); \ bool operator!=(const X& first, const X& second); \ static const int* _TypePtr##X = &X::Type; \ @@ -1453,7 +1539,8 @@ public: Bitstream& operator>>(Bitstream& in, S::N& obj); \ template<> inline void Bitstream::writeRawDelta(const S::N& value, const S::N& reference) { *this << value; } \ template<> inline void Bitstream::readRawDelta(S::N& value, const S::N& reference) { *this >> value; } \ - template<> inline QJsonValue JSONWriter::getData(const S::N& value) { return (int)value; } + template<> inline QJsonValue JSONWriter::getData(const S::N& value) { return (int)value; } \ + template<> inline void JSONReader::putData(const QJsonValue& data, S::N& value) { value = (S::N)data.toInt(); } #define IMPLEMENT_ENUM_METATYPE(S, N) \ static int S##N##MetaTypeId = registerEnumMetaType(&S::staticMetaObject, #N); \ diff --git a/tools/mtc/src/main.cpp b/tools/mtc/src/main.cpp index eadccdb310..267e45a5a5 100644 --- a/tools/mtc/src/main.cpp +++ b/tools/mtc/src/main.cpp @@ -230,6 +230,19 @@ void generateOutput (QTextStream& out, const QList& streamables) { out << " return array;\n"; out << "}\n"; + out << "template<> void JSONReader::putData(const QJsonValue& data, " << name << "& value) {\n"; + out << " QJsonArray array = data.toArray();\n"; + out << " QJsonArray::const_iterator it = array.constBegin();\n"; + /* foreach (const QString& base, str.clazz.bases) { + out << " foreach (const QJsonValue& element, getData(static_cast(value)).toArray()) {\n"; + out << " array.append(element);\n"; + out << " }\n"; + } */ + foreach (const Field& field, str.fields) { + out << " putData(*it++, value." << field.name << ");\n"; + } + out << "}\n"; + out << "bool operator==(const " << name << "& first, const " << name << "& second) {\n"; if (str.clazz.bases.isEmpty() && str.fields.isEmpty()) { out << " return true"; From 37c977af024966ea2cde01d843f8ec6f60c54308 Mon Sep 17 00:00:00 2001 From: Andrzej Kapolka Date: Mon, 16 Jun 2014 19:12:21 -0700 Subject: [PATCH 07/16] Tests working for reading JSON. --- libraries/metavoxels/src/Bitstream.cpp | 208 +++++++++++++++++++++--- libraries/metavoxels/src/Bitstream.h | 6 +- tests/metavoxels/src/MetavoxelTests.cpp | 60 ++++++- tools/mtc/src/main.cpp | 27 ++- 4 files changed, 260 insertions(+), 41 deletions(-) diff --git a/libraries/metavoxels/src/Bitstream.cpp b/libraries/metavoxels/src/Bitstream.cpp index 8f880a1803..86f508ca11 100644 --- a/libraries/metavoxels/src/Bitstream.cpp +++ b/libraries/metavoxels/src/Bitstream.cpp @@ -1178,7 +1178,9 @@ Bitstream& Bitstream::operator>(ObjectStreamerPointer& streamer) { } for (int i = 0; i < localProperties.size(); i++) { const StreamerPropertyPair& property = properties.at(i); - if (localProperties.at(i).first != property.first || property.second.propertyIndex() != i) { + const StreamerPropertyPair& localProperty = localProperties.at(i); + if (property.first != localProperty.first || + property.second.propertyIndex() != localProperty.second.propertyIndex()) { streamer = ObjectStreamerPointer(new MappedObjectStreamer(metaObject, properties)); return *this; } @@ -1866,7 +1868,6 @@ void JSONWriter::addSharedObject(const SharedObjectPointer& object) { QJsonObject sharedObject; sharedObject.insert("id", object->getID()); - sharedObject.insert("originID", object->getOriginID()); sharedObject.insert("data", getData(static_cast(object.data()))); _sharedObjects.replace(index, sharedObject); } @@ -1887,17 +1888,121 @@ JSONReader::JSONReader(const QJsonDocument& document, Bitstream::GenericsMode ge for (int i = types.size() - 1; i >= 0; i--) { QJsonObject type = types.at(i).toObject(); QString name = type.value("name").toString(); + QByteArray latinName = name.toLatin1(); + const TypeStreamer* baseStreamer = Bitstream::getTypeStreamers().value(QMetaType::type(latinName)); + if (!baseStreamer) { + baseStreamer = Bitstream::getEnumStreamersByName().value(latinName); + } + if (!baseStreamer && genericsMode == Bitstream::NO_GENERICS) { + continue; + } QString category = type.value("category").toString(); + if (!baseStreamer || genericsMode == Bitstream::ALL_GENERICS) { + TypeStreamerPointer streamer; + if (category == "ENUM") { + QVector values; + int highestValue = 0; + foreach (const QJsonValue& value, type.value("values").toArray()) { + QJsonObject object = value.toObject(); + int value = object.value("value").toInt(); + highestValue = qMax(value, highestValue); + values.append(NameIntPair(object.value("key").toString().toLatin1(), value)); + } + streamer = TypeStreamerPointer(new GenericEnumTypeStreamer(latinName, + values, getBitsForHighestValue(highestValue), QByteArray())); + + } else if (category == "STREAMABLE") { + QVector fields; + foreach (const QJsonValue& field, type.value("fields").toArray()) { + QJsonObject object = field.toObject(); + fields.append(StreamerNamePair(getTypeStreamer(object.value("type").toString()), + object.value("name").toString().toLatin1())); + } + streamer = TypeStreamerPointer(new GenericStreamableTypeStreamer(latinName, + fields, QByteArray())); + + } else if (category == "LIST") { + streamer = TypeStreamerPointer(new GenericListTypeStreamer(latinName, + getTypeStreamer(type.value("valueType").toString()))); + + } else if (category == "SET") { + streamer = TypeStreamerPointer(new GenericSetTypeStreamer(latinName, + getTypeStreamer(type.value("valueType").toString()))); + + } else if (category == "MAP") { + streamer = TypeStreamerPointer(new GenericMapTypeStreamer(latinName, + getTypeStreamer(type.value("keyType").toString()), + getTypeStreamer(type.value("valueType").toString()))); + } + _typeStreamers.insert(name, streamer); + static_cast(streamer.data())->_weakSelf = streamer; + continue; + } if (category == "ENUM") { - + QHash mappings; + int highestValue = 0; + QMetaEnum metaEnum = baseStreamer->getMetaEnum(); + QJsonArray array = type.value("values").toArray(); + bool matches = (array.size() == metaEnum.keyCount()); + foreach (const QJsonValue& value, array) { + QJsonObject object = value.toObject(); + int value = object.value("value").toInt(); + highestValue = qMax(value, highestValue); + int mapping = metaEnum.keyToValue(object.value("key").toString().toLatin1()); + if (mapping != -1) { + mappings.insert(value, mapping); + } + matches &= (value == mapping); + } + if (matches) { + _typeStreamers.insert(name, baseStreamer->getSelf()); + } else { + _typeStreamers.insert(name, TypeStreamerPointer(new MappedEnumTypeStreamer(baseStreamer, + getBitsForHighestValue(highestValue), mappings))); + } } else if (category == "STREAMABLE") { - + QVector fields; + QJsonArray array = type.value("fields").toArray(); + const QVector& metaFields = baseStreamer->getMetaFields(); + bool matches = (array.size() == metaFields.size()); + for (int j = 0; j < array.size(); j++) { + QJsonObject object = array.at(j).toObject(); + TypeStreamerPointer streamer = getTypeStreamer(object.value("type").toString()); + int index = baseStreamer->getFieldIndex(object.value("name").toString().toLatin1()); + fields.append(StreamerIndexPair(streamer, index)); + matches &= (index == j && streamer == metaFields.at(j).getStreamer()); + } + if (matches) { + _typeStreamers.insert(name, baseStreamer->getSelf()); + } else { + _typeStreamers.insert(name, TypeStreamerPointer(new MappedStreamableTypeStreamer(baseStreamer, fields))); + } } else if (category == "LIST") { - + TypeStreamerPointer valueStreamer = getTypeStreamer(type.value("valueType").toString()); + if (valueStreamer == baseStreamer->getValueStreamer()) { + _typeStreamers.insert(name, baseStreamer->getSelf()); + + } else { + _typeStreamers.insert(name, TypeStreamerPointer(new MappedListTypeStreamer(baseStreamer, valueStreamer))); + } } else if (category == "SET") { - + TypeStreamerPointer valueStreamer = getTypeStreamer(type.value("valueType").toString()); + if (valueStreamer == baseStreamer->getValueStreamer()) { + _typeStreamers.insert(name, baseStreamer->getSelf()); + + } else { + _typeStreamers.insert(name, TypeStreamerPointer(new MappedSetTypeStreamer(baseStreamer, valueStreamer))); + } } else if (category == "MAP") { - + TypeStreamerPointer keyStreamer = getTypeStreamer(type.value("keyType").toString()); + TypeStreamerPointer valueStreamer = getTypeStreamer(type.value("valueType").toString()); + if (keyStreamer == baseStreamer->getKeyStreamer() && valueStreamer == baseStreamer->getValueStreamer()) { + _typeStreamers.insert(name, baseStreamer->getSelf()); + + } else { + _typeStreamers.insert(name, TypeStreamerPointer(new MappedMapTypeStreamer( + baseStreamer, keyStreamer, valueStreamer))); + } } } @@ -1905,11 +2010,45 @@ JSONReader::JSONReader(const QJsonDocument& document, Bitstream::GenericsMode ge for (int i = classes.size() - 1; i >= 0; i--) { QJsonObject clazz = classes.at(i).toObject(); QString name = clazz.value("name").toString(); - QJsonArray properties = clazz.value("properties").toArray(); - foreach (const QJsonValue& property, properties) { - QJsonObject object = property.toObject(); - object.value("type"); - object.value("name"); + QByteArray latinName = name.toLatin1(); + const ObjectStreamer* baseStreamer = Bitstream::getObjectStreamers().value( + Bitstream::getMetaObjects().value(latinName)); + if (!baseStreamer && genericsMode == Bitstream::NO_GENERICS) { + continue; + } + if (!baseStreamer || genericsMode == Bitstream::ALL_GENERICS) { + QVector properties; + foreach (const QJsonValue& property, clazz.value("properties").toArray()) { + QJsonObject object = property.toObject(); + properties.append(StreamerNamePair(getTypeStreamer(object.value("type").toString()), + object.value("name").toString().toLatin1())); + } + ObjectStreamerPointer streamer = ObjectStreamerPointer(new GenericObjectStreamer( + latinName, properties, QByteArray())); + _objectStreamers.insert(name, streamer); + static_cast(streamer.data())->_weakSelf = streamer; + continue; + } + const QMetaObject* metaObject = baseStreamer->getMetaObject(); + const QVector& baseProperties = baseStreamer->getProperties(); + QVector properties; + QJsonArray propertyArray = clazz.value("properties").toArray(); + bool matches = (baseProperties.size() == propertyArray.size()); + for (int j = 0; j < propertyArray.size(); j++) { + QJsonObject object = propertyArray.at(j).toObject(); + TypeStreamerPointer typeStreamer = getTypeStreamer(object.value("type").toString()); + QMetaProperty metaProperty = metaObject->property(metaObject->indexOfProperty( + object.value("name").toString().toLatin1())); + properties.append(StreamerPropertyPair(typeStreamer, metaProperty)); + + const StreamerPropertyPair& baseProperty = baseProperties.at(i); + matches &= (typeStreamer == baseProperty.first && + metaProperty.propertyIndex() == baseProperty.second.propertyIndex()); + } + if (matches) { + _objectStreamers.insert(name, baseStreamer->getSelf()); + } else { + _objectStreamers.insert(name, ObjectStreamerPointer(new MappedObjectStreamer(metaObject, properties))); } } @@ -1917,9 +2056,11 @@ JSONReader::JSONReader(const QJsonDocument& document, Bitstream::GenericsMode ge for (int i = objects.size() - 1; i >= 0; i--) { QJsonObject object = objects.at(i).toObject(); int id = object.value("id").toInt(); - int originID = object.value("originID").toInt(); - QJsonObject data = object.value("data").toObject(); - + QObject* qObject; + putData(object.value("data"), qObject); + if (qObject) { + _sharedObjects.insert(id, static_cast(qObject)); + } } _contents = top.value("contents").toArray(); @@ -2051,16 +2192,12 @@ void JSONReader::putData(const QJsonValue& data, const QMetaObject*& value) { void JSONReader::putData(const QJsonValue& data, QVariant& value) { QJsonObject object = data.toObject(); QString type = object.value("type").toString(); - const TypeStreamer* streamer = _typeStreamers.value(type).data(); - if (!streamer) { - streamer = Bitstream::getTypeStreamers().value(QMetaType::type(type.toLatin1())); - if (!streamer) { - qWarning() << "Unknown type:" << type; - value = QVariant(); - return; - } + TypeStreamerPointer streamer = getTypeStreamer(type); + if (streamer) { + streamer->putJSONVariantData(*this, object.value("value"), value); + } else { + value = QVariant(); } - streamer->putJSONData(*this, object.value("value"), value); } void JSONReader::putData(const QJsonValue& data, SharedObjectPointer& value) { @@ -2073,6 +2210,19 @@ void JSONReader::putData(const QJsonValue& data, QObject*& value) { value = streamer ? streamer->putJSONData(*this, object) : NULL; } +TypeStreamerPointer JSONReader::getTypeStreamer(const QString& name) const { + TypeStreamerPointer streamer = _typeStreamers.value(name); + if (!streamer) { + const TypeStreamer* defaultStreamer = Bitstream::getTypeStreamers().value(QMetaType::type(name.toLatin1())); + if (defaultStreamer) { + streamer = defaultStreamer->getSelf(); + } else { + qWarning() << "Unknown type:" << name; + } + } + return streamer; +} + ObjectStreamer::ObjectStreamer(const QMetaObject* metaObject) : _metaObject(metaObject) { } @@ -2438,6 +2588,10 @@ void TypeStreamer::putJSONData(JSONReader& reader, const QJsonValue& data, QVari value = QVariant(); } +void TypeStreamer::putJSONVariantData(JSONReader& reader, const QJsonValue& data, QVariant& value) const { + putJSONData(reader, data, value); +} + bool TypeStreamer::equal(const QVariant& first, const QVariant& second) const { return first == second; } @@ -2750,6 +2904,12 @@ const char* GenericTypeStreamer::getName() const { return _name.constData(); } +void GenericTypeStreamer::putJSONVariantData(JSONReader& reader, const QJsonValue& data, QVariant& value) const { + QVariant containedValue; + putJSONData(reader, data, containedValue); + value = QVariant::fromValue(GenericValue(_weakSelf, containedValue)); +} + QVariant GenericTypeStreamer::readVariant(Bitstream& in) const { return QVariant::fromValue(GenericValue(_weakSelf, read(in))); } diff --git a/libraries/metavoxels/src/Bitstream.h b/libraries/metavoxels/src/Bitstream.h index 0cad33acea..776a362717 100644 --- a/libraries/metavoxels/src/Bitstream.h +++ b/libraries/metavoxels/src/Bitstream.h @@ -893,7 +893,7 @@ public: template JSONReader& operator>>(T& value) { putData(*_contentsIterator++, value); return *this; } - TypeStreamerPointer getTypeStreamer(const QString& name) const { return _typeStreamers.value(name); } + TypeStreamerPointer getTypeStreamer(const QString& name) const; ObjectStreamerPointer getObjectStreamer(const QString& name) const { return _objectStreamers.value(name); } SharedObjectPointer getSharedObject(int id) const { return _sharedObjects.value(id); } @@ -1020,6 +1020,7 @@ public: private: friend class Bitstream; + friend class JSONReader; QByteArray _name; WeakObjectStreamerPointer _weakSelf; @@ -1107,6 +1108,7 @@ public: virtual QJsonValue getJSONData(JSONWriter& writer, const QVariant& value) const; virtual QJsonValue getJSONVariantData(JSONWriter& writer, const QVariant& value) const; virtual void putJSONData(JSONReader& reader, const QJsonValue& data, QVariant& value) const; + virtual void putJSONVariantData(JSONReader& reader, const QJsonValue& data, QVariant& value) const; virtual bool equal(const QVariant& first, const QVariant& second) const; @@ -1237,11 +1239,13 @@ public: GenericTypeStreamer(const QByteArray& name); virtual const char* getName() const; + virtual void putJSONVariantData(JSONReader& reader, const QJsonValue& data, QVariant& value) const; virtual QVariant readVariant(Bitstream& in) const; protected: friend class Bitstream; + friend class JSONReader; QByteArray _name; WeakTypeStreamerPointer _weakSelf; diff --git a/tests/metavoxels/src/MetavoxelTests.cpp b/tests/metavoxels/src/MetavoxelTests.cpp index 618178383f..4a3010caf4 100644 --- a/tests/metavoxels/src/MetavoxelTests.cpp +++ b/tests/metavoxels/src/MetavoxelTests.cpp @@ -252,8 +252,64 @@ static bool testSerialization(Bitstream::MetadataType metadataType) { jsonWriter << testObjectReadA; jsonWriter << testObjectReadB; jsonWriter << messageRead; - qDebug() << jsonWriter.getDocument().toJson(); - qDebug(); + jsonWriter << endRead; + QByteArray encodedJson = jsonWriter.getDocument().toJson(); + + // and read from JSON + JSONReader jsonReader(QJsonDocument::fromJson(encodedJson), Bitstream::ALL_GENERICS); + jsonReader >> testObjectReadA; + jsonReader >> testObjectReadB; + jsonReader >> messageRead; + jsonReader >> endRead; + + // reassign the ids + testObjectReadA->setID(testObjectWrittenA->getID()); + testObjectReadA->setOriginID(testObjectWrittenA->getOriginID()); + testObjectReadB->setID(testObjectWrittenB->getID()); + testObjectReadB->setOriginID(testObjectWrittenB->getOriginID()); + + // and back to binary + QByteArray secondCompareArray; + QDataStream secondCompareOutStream(&secondCompareArray, QIODevice::WriteOnly); + Bitstream secondCompareOut(secondCompareOutStream, Bitstream::FULL_METADATA); + secondCompareOut << testObjectReadA; + secondCompareOut << testObjectReadB; + secondCompareOut << messageRead; + secondCompareOut << endRead; + secondCompareOut.flush(); + + if (compareArray != secondCompareArray) { + qDebug() << "Mismatch between written/JSON streams (generics)."; + return true; + } + + // once more, with mapping! + JSONReader secondJSONReader(QJsonDocument::fromJson(encodedJson)); + secondJSONReader >> testObjectReadA; + secondJSONReader >> testObjectReadB; + secondJSONReader >> messageRead; + secondJSONReader >> endRead; + + // reassign the ids + testObjectReadA->setID(testObjectWrittenA->getID()); + testObjectReadA->setOriginID(testObjectWrittenA->getOriginID()); + testObjectReadB->setID(testObjectWrittenB->getID()); + testObjectReadB->setOriginID(testObjectWrittenB->getOriginID()); + + // and back to binary + QByteArray thirdCompareArray; + QDataStream thirdCompareOutStream(&thirdCompareArray, QIODevice::WriteOnly); + Bitstream thirdCompareOut(thirdCompareOutStream, Bitstream::FULL_METADATA); + thirdCompareOut << testObjectReadA; + thirdCompareOut << testObjectReadB; + thirdCompareOut << messageRead; + thirdCompareOut << endRead; + thirdCompareOut.flush(); + + if (compareArray != thirdCompareArray) { + qDebug() << "Mismatch between written/JSON streams (mapped)."; + return true; + } return false; } diff --git a/tools/mtc/src/main.cpp b/tools/mtc/src/main.cpp index 267e45a5a5..dd0ab837a5 100644 --- a/tools/mtc/src/main.cpp +++ b/tools/mtc/src/main.cpp @@ -123,11 +123,6 @@ void generateOutput (QTextStream& out, const QList& streamables) { out << "QHash " << name << "::createFieldIndices() {\n"; out << " QHash indices;\n"; out << " int index = 1;\n"; - foreach (const QString& base, str.clazz.bases) { - out << " foreach (const MetaField& field, " << base << "::getMetaFields()) {\n"; - out << " indices.insert(field.getName(), index++);\n"; - out << " }\n"; - } out << " foreach (const MetaField& field, getMetaFields()) {\n"; out << " indices.insert(field.getName(), index++);\n"; out << " }\n"; @@ -231,15 +226,19 @@ void generateOutput (QTextStream& out, const QList& streamables) { out << "}\n"; out << "template<> void JSONReader::putData(const QJsonValue& data, " << name << "& value) {\n"; - out << " QJsonArray array = data.toArray();\n"; - out << " QJsonArray::const_iterator it = array.constBegin();\n"; - /* foreach (const QString& base, str.clazz.bases) { - out << " foreach (const QJsonValue& element, getData(static_cast(value)).toArray()) {\n"; - out << " array.append(element);\n"; - out << " }\n"; - } */ - foreach (const Field& field, str.fields) { - out << " putData(*it++, value." << field.name << ");\n"; + if (!(str.clazz.bases.isEmpty() && str.fields.isEmpty())) { + out << " QJsonArray array = data.toArray(), subarray;\n"; + out << " QJsonArray::const_iterator it = array.constBegin();\n"; + foreach (const QString& base, str.clazz.bases) { + out << " subarray = QJsonArray();\n"; + out << " for (int i = 0; i < " << base << "::getMetaFields().size(); i++) {\n"; + out << " subarray.append(*it++);\n"; + out << " }\n"; + out << " putData(subarray, static_cast<" << base << "&>(value));\n"; + } + foreach (const Field& field, str.fields) { + out << " putData(*it++, value." << field.name << ");\n"; + } } out << "}\n"; From 793117367155fd9e92ebb6e086d65e6b03afc5e6 Mon Sep 17 00:00:00 2001 From: Andrzej Kapolka Date: Tue, 17 Jun 2014 12:33:31 -0700 Subject: [PATCH 08/16] Working on tools to convert bitstreams to/from json. --- CMakeLists.txt | 1 + cmake/macros/AutoMTC.cmake | 6 +- libraries/metavoxels/src/Bitstream.cpp | 1 + libraries/metavoxels/src/Bitstream.h | 4 ++ tests/metavoxels/src/MetavoxelTests.cpp | 2 + tools/CMakeLists.txt | 6 ++ tools/bitstream2json/CMakeLists.txt | 20 +++++++ tools/bitstream2json/src/main.cpp | 70 +++++++++++++++++++++++ tools/json2bitstream/CMakeLists.txt | 20 +++++++ tools/json2bitstream/src/main.cpp | 76 +++++++++++++++++++++++++ 10 files changed, 201 insertions(+), 5 deletions(-) create mode 100644 tools/CMakeLists.txt create mode 100644 tools/bitstream2json/CMakeLists.txt create mode 100644 tools/bitstream2json/src/main.cpp create mode 100644 tools/json2bitstream/CMakeLists.txt create mode 100644 tools/json2bitstream/src/main.cpp diff --git a/CMakeLists.txt b/CMakeLists.txt index cb1e4224cf..a399e11168 100644 --- a/CMakeLists.txt +++ b/CMakeLists.txt @@ -49,4 +49,5 @@ add_subdirectory(assignment-client) add_subdirectory(domain-server) add_subdirectory(interface) add_subdirectory(tests) +add_subdirectory(tools) add_subdirectory(voxel-edit) diff --git a/cmake/macros/AutoMTC.cmake b/cmake/macros/AutoMTC.cmake index 1682b9cd56..6f0216de7f 100644 --- a/cmake/macros/AutoMTC.cmake +++ b/cmake/macros/AutoMTC.cmake @@ -9,13 +9,9 @@ # macro(AUTO_MTC TARGET ROOT_DIR) - if (NOT TARGET mtc) - add_subdirectory("${ROOT_DIR}/tools/mtc" "${ROOT_DIR}/tools/mtc") - endif () - set(AUTOMTC_SRC ${TARGET}_automtc.cpp) file(GLOB INCLUDE_FILES src/*.h) add_custom_command(OUTPUT ${AUTOMTC_SRC} COMMAND mtc -o ${AUTOMTC_SRC} ${INCLUDE_FILES} DEPENDS mtc ${INCLUDE_FILES}) -endmacro() \ No newline at end of file +endmacro() diff --git a/libraries/metavoxels/src/Bitstream.cpp b/libraries/metavoxels/src/Bitstream.cpp index 86f508ca11..0667e71fb1 100644 --- a/libraries/metavoxels/src/Bitstream.cpp +++ b/libraries/metavoxels/src/Bitstream.cpp @@ -33,6 +33,7 @@ REGISTER_SIMPLE_TYPE_STREAMER(QByteArray) REGISTER_SIMPLE_TYPE_STREAMER(QColor) REGISTER_SIMPLE_TYPE_STREAMER(QScriptValue) REGISTER_SIMPLE_TYPE_STREAMER(QString) +REGISTER_SIMPLE_TYPE_STREAMER(QVariant) REGISTER_SIMPLE_TYPE_STREAMER(QUrl) REGISTER_SIMPLE_TYPE_STREAMER(QVariantList) REGISTER_SIMPLE_TYPE_STREAMER(QVariantHash) diff --git a/libraries/metavoxels/src/Bitstream.h b/libraries/metavoxels/src/Bitstream.h index 776a362717..66a42e6991 100644 --- a/libraries/metavoxels/src/Bitstream.h +++ b/libraries/metavoxels/src/Bitstream.h @@ -805,6 +805,8 @@ public: template JSONWriter& operator<<(const T& value) { _contents.append(getData(value)); return *this; } + void appendToContents(const QJsonValue& value) { _contents.append(value); } + void addSharedObject(const SharedObjectPointer& object); void addObjectStreamer(const ObjectStreamer* streamer); void addTypeStreamer(const TypeStreamer* streamer); @@ -893,6 +895,8 @@ public: template JSONReader& operator>>(T& value) { putData(*_contentsIterator++, value); return *this; } + QJsonValue retrieveNextFromContents() { return *_contentsIterator++; } + TypeStreamerPointer getTypeStreamer(const QString& name) const; ObjectStreamerPointer getObjectStreamer(const QString& name) const { return _objectStreamers.value(name); } SharedObjectPointer getSharedObject(int id) const { return _sharedObjects.value(id); } diff --git a/tests/metavoxels/src/MetavoxelTests.cpp b/tests/metavoxels/src/MetavoxelTests.cpp index 4a3010caf4..85be9a5c8b 100644 --- a/tests/metavoxels/src/MetavoxelTests.cpp +++ b/tests/metavoxels/src/MetavoxelTests.cpp @@ -255,6 +255,8 @@ static bool testSerialization(Bitstream::MetadataType metadataType) { jsonWriter << endRead; QByteArray encodedJson = jsonWriter.getDocument().toJson(); + qDebug() << encodedJson; + // and read from JSON JSONReader jsonReader(QJsonDocument::fromJson(encodedJson), Bitstream::ALL_GENERICS); jsonReader >> testObjectReadA; diff --git a/tools/CMakeLists.txt b/tools/CMakeLists.txt new file mode 100644 index 0000000000..79db82e90f --- /dev/null +++ b/tools/CMakeLists.txt @@ -0,0 +1,6 @@ +cmake_minimum_required(VERSION 2.8) + +# add the tool directories +add_subdirectory(bitstream2json) +add_subdirectory(json2bitstream) +add_subdirectory(mtc) diff --git a/tools/bitstream2json/CMakeLists.txt b/tools/bitstream2json/CMakeLists.txt new file mode 100644 index 0000000000..6483c024a7 --- /dev/null +++ b/tools/bitstream2json/CMakeLists.txt @@ -0,0 +1,20 @@ +cmake_minimum_required(VERSION 2.8) + +if (WIN32) + cmake_policy (SET CMP0020 NEW) +endif (WIN32) + +set(TARGET_NAME bitstream2json) + +set(ROOT_DIR ../..) +set(MACRO_DIR "${ROOT_DIR}/cmake/macros") + +find_package(Qt5 COMPONENTS Network Script Widgets) + +include(${MACRO_DIR}/SetupHifiProject.cmake) +setup_hifi_project(${TARGET_NAME} TRUE) + +link_hifi_library(metavoxels ${TARGET_NAME} "${ROOT_DIR}") +link_hifi_library(shared ${TARGET_NAME} "${ROOT_DIR}") + +target_link_libraries(${TARGET_NAME} Qt5::Network Qt5::Widgets Qt5::Script) diff --git a/tools/bitstream2json/src/main.cpp b/tools/bitstream2json/src/main.cpp new file mode 100644 index 0000000000..0f299527b0 --- /dev/null +++ b/tools/bitstream2json/src/main.cpp @@ -0,0 +1,70 @@ +// +// main.cpp +// tools/bitstream2json/src +// +// Created by Andrzej Kapolka on 6/17/14. +// Copyright 2014 High Fidelity, Inc. +// +// Distributed under the Apache License, Version 2.0. +// See the accompanying file LICENSE or http://www.apache.org/licenses/LICENSE-2.0.html + +#include + +#include +#include +#include + +#include + +using namespace std; + +int main (int argc, char** argv) { + // need the core application for the script engine + QCoreApplication app(argc, argv); + + if (argc < 3) { + cerr << "Usage: bitstream2json inputfile outputfile [types...]" << endl; + return 0; + } + QFile inputFile(argv[1]); + if (!inputFile.open(QIODevice::ReadOnly)) { + cerr << "Failed to open input file: " << inputFile.errorString().toLatin1().constData() << endl; + return 1; + } + QDataStream inputData(&inputFile); + Bitstream input(inputData, Bitstream::FULL_METADATA, Bitstream::ALL_GENERICS); + + QFile outputFile(argv[2]); + if (!outputFile.open(QIODevice::WriteOnly)) { + cerr << "Failed to open output file: " << outputFile.errorString().toLatin1().constData() << endl; + return 1; + } + JSONWriter output; + + if (argc < 4) { + // default type is a single QVariant + QVariant value; + input >> value; + output << value; + + } else { + for (int i = 3; i < argc; i++) { + int type = QMetaType::type(argv[i]); + if (type == QMetaType::UnknownType) { + cerr << "Unknown type: " << argv[i] << endl; + return 1; + } + const TypeStreamer* streamer = Bitstream::getTypeStreamer(type); + if (!streamer) { + cerr << "Non-streamable type: " << argv[i] << endl; + return 1; + } + QVariant value = streamer->read(input); + output.appendToContents(streamer->getJSONData(output, value)); + } + } + + outputFile.write(output.getDocument().toJson()); + + return 0; +} diff --git a/tools/json2bitstream/CMakeLists.txt b/tools/json2bitstream/CMakeLists.txt new file mode 100644 index 0000000000..799f3bcc4c --- /dev/null +++ b/tools/json2bitstream/CMakeLists.txt @@ -0,0 +1,20 @@ +cmake_minimum_required(VERSION 2.8) + +if (WIN32) + cmake_policy (SET CMP0020 NEW) +endif (WIN32) + +set(TARGET_NAME json2bitstream) + +set(ROOT_DIR ../..) +set(MACRO_DIR "${ROOT_DIR}/cmake/macros") + +find_package(Qt5 COMPONENTS Network Script Widgets) + +include(${MACRO_DIR}/SetupHifiProject.cmake) +setup_hifi_project(${TARGET_NAME} TRUE) + +link_hifi_library(metavoxels ${TARGET_NAME} "${ROOT_DIR}") +link_hifi_library(shared ${TARGET_NAME} "${ROOT_DIR}") + +target_link_libraries(${TARGET_NAME} Qt5::Network Qt5::Widgets Qt5::Script) diff --git a/tools/json2bitstream/src/main.cpp b/tools/json2bitstream/src/main.cpp new file mode 100644 index 0000000000..ff09bcfdc6 --- /dev/null +++ b/tools/json2bitstream/src/main.cpp @@ -0,0 +1,76 @@ +// +// main.cpp +// tools/json2bitstream/src +// +// Created by Andrzej Kapolka on 6/17/14. +// Copyright 2014 High Fidelity, Inc. +// +// Distributed under the Apache License, Version 2.0. +// See the accompanying file LICENSE or http://www.apache.org/licenses/LICENSE-2.0.html + +#include + +#include +#include +#include + +#include + +using namespace std; + +int main (int argc, char** argv) { + // need the core application for the script engine + QCoreApplication app(argc, argv); + + if (argc < 3) { + cerr << "Usage: bitstream2json inputfile outputfile [types...]" << endl; + return 0; + } + QFile inputFile(argv[1]); + if (!inputFile.open(QIODevice::ReadOnly)) { + cerr << "Failed to open input file: " << inputFile.errorString().toLatin1().constData() << endl; + return 1; + } + QJsonParseError error; + QJsonDocument document = QJsonDocument::fromJson(inputFile.readAll(), &error); + if (error.error != QJsonParseError::NoError) { + cerr << "Failed to read input file: " << error.errorString().toLatin1().constData() << endl; + return 1; + } + JSONReader input(document, Bitstream::ALL_GENERICS); + + QFile outputFile(argv[2]); + if (!outputFile.open(QIODevice::WriteOnly)) { + cerr << "Failed to open output file: " << outputFile.errorString().toLatin1().constData() << endl; + return 1; + } + QDataStream outputData(&outputFile); + Bitstream output(outputData, Bitstream::FULL_METADATA); + + if (argc < 4) { + // default type is a single QVariant + QVariant value; + input >> value; + output << value; + + } else { + for (int i = 3; i < argc; i++) { + int type = QMetaType::type(argv[i]); + if (type == QMetaType::UnknownType) { + cerr << "Unknown type: " << argv[i] << endl; + return 1; + } + const TypeStreamer* streamer = Bitstream::getTypeStreamer(type); + if (!streamer) { + cerr << "Non-streamable type: " << argv[i] << endl; + return 1; + } + QVariant value; + streamer->putJSONData(input, input.retrieveNextFromContents(), value); + streamer->write(output, value); + } + } + output.flush(); + + return 0; +} From c3090dd223e87dc0b0bdc191b86592d7afa11ef5 Mon Sep 17 00:00:00 2001 From: Andrzej Kapolka Date: Tue, 17 Jun 2014 16:12:06 -0700 Subject: [PATCH 09/16] Tweak to the way streamers are written to JSON. --- libraries/metavoxels/src/Bitstream.cpp | 48 +++++++++----------------- 1 file changed, 16 insertions(+), 32 deletions(-) diff --git a/libraries/metavoxels/src/Bitstream.cpp b/libraries/metavoxels/src/Bitstream.cpp index 0667e71fb1..9f6ceb9b2e 100644 --- a/libraries/metavoxels/src/Bitstream.cpp +++ b/libraries/metavoxels/src/Bitstream.cpp @@ -1836,14 +1836,9 @@ void JSONWriter::addTypeStreamer(const TypeStreamer* streamer) { if (!_typeStreamerNames.contains(streamer->getName())) { _typeStreamerNames.insert(streamer->getName()); - // start with a placeholder; then remove/replace with actual metadata - int index = _typeStreamers.size(); - _typeStreamers.append(QJsonValue()); QJsonValue metadata = streamer->getJSONMetadata(*this); - if (metadata.isNull()) { - _typeStreamers.removeAt(index); - } else { - _typeStreamers.replace(index, metadata); + if (!metadata.isNull()) { + _typeStreamers.append(metadata); } } } @@ -1851,11 +1846,7 @@ void JSONWriter::addTypeStreamer(const TypeStreamer* streamer) { void JSONWriter::addObjectStreamer(const ObjectStreamer* streamer) { if (!_objectStreamerNames.contains(streamer->getName())) { _objectStreamerNames.insert(streamer->getName()); - - // start with a placeholder; then replace with actual metadata - int index = _objectStreamers.size(); - _objectStreamers.append(QJsonValue()); - _objectStreamers.replace(index, streamer->getJSONMetadata(*this)); + _objectStreamers.append(streamer->getJSONMetadata(*this)); } } @@ -1863,14 +1854,10 @@ void JSONWriter::addSharedObject(const SharedObjectPointer& object) { if (!_sharedObjectIDs.contains(object->getID())) { _sharedObjectIDs.insert(object->getID()); - // start with a placeholder; then replace with actual object - int index = _sharedObjects.size(); - _sharedObjects.append(QJsonValue()); - QJsonObject sharedObject; sharedObject.insert("id", object->getID()); sharedObject.insert("data", getData(static_cast(object.data()))); - _sharedObjects.replace(index, sharedObject); + _sharedObjects.append(sharedObject); } } @@ -1885,9 +1872,8 @@ QJsonDocument JSONWriter::getDocument() const { JSONReader::JSONReader(const QJsonDocument& document, Bitstream::GenericsMode genericsMode) { QJsonObject top = document.object(); - QJsonArray types = top.value("types").toArray(); - for (int i = types.size() - 1; i >= 0; i--) { - QJsonObject type = types.at(i).toObject(); + foreach (const QJsonValue& element, top.value("types").toArray()) { + QJsonObject type = element.toObject(); QString name = type.value("name").toString(); QByteArray latinName = name.toLatin1(); const TypeStreamer* baseStreamer = Bitstream::getTypeStreamers().value(QMetaType::type(latinName)); @@ -1966,12 +1952,12 @@ JSONReader::JSONReader(const QJsonDocument& document, Bitstream::GenericsMode ge QJsonArray array = type.value("fields").toArray(); const QVector& metaFields = baseStreamer->getMetaFields(); bool matches = (array.size() == metaFields.size()); - for (int j = 0; j < array.size(); j++) { - QJsonObject object = array.at(j).toObject(); + for (int i = 0; i < array.size(); i++) { + QJsonObject object = array.at(i).toObject(); TypeStreamerPointer streamer = getTypeStreamer(object.value("type").toString()); int index = baseStreamer->getFieldIndex(object.value("name").toString().toLatin1()); fields.append(StreamerIndexPair(streamer, index)); - matches &= (index == j && streamer == metaFields.at(j).getStreamer()); + matches &= (index == i && streamer == metaFields.at(i).getStreamer()); } if (matches) { _typeStreamers.insert(name, baseStreamer->getSelf()); @@ -2006,10 +1992,9 @@ JSONReader::JSONReader(const QJsonDocument& document, Bitstream::GenericsMode ge } } } - - QJsonArray classes = top.value("classes").toArray(); - for (int i = classes.size() - 1; i >= 0; i--) { - QJsonObject clazz = classes.at(i).toObject(); + + foreach (const QJsonValue& element, top.value("classes").toArray()) { + QJsonObject clazz = element.toObject(); QString name = clazz.value("name").toString(); QByteArray latinName = name.toLatin1(); const ObjectStreamer* baseStreamer = Bitstream::getObjectStreamers().value( @@ -2035,8 +2020,8 @@ JSONReader::JSONReader(const QJsonDocument& document, Bitstream::GenericsMode ge QVector properties; QJsonArray propertyArray = clazz.value("properties").toArray(); bool matches = (baseProperties.size() == propertyArray.size()); - for (int j = 0; j < propertyArray.size(); j++) { - QJsonObject object = propertyArray.at(j).toObject(); + for (int i = 0; i < propertyArray.size(); i++) { + QJsonObject object = propertyArray.at(i).toObject(); TypeStreamerPointer typeStreamer = getTypeStreamer(object.value("type").toString()); QMetaProperty metaProperty = metaObject->property(metaObject->indexOfProperty( object.value("name").toString().toLatin1())); @@ -2053,9 +2038,8 @@ JSONReader::JSONReader(const QJsonDocument& document, Bitstream::GenericsMode ge } } - QJsonArray objects = top.value("objects").toArray(); - for (int i = objects.size() - 1; i >= 0; i--) { - QJsonObject object = objects.at(i).toObject(); + foreach (const QJsonValue& element, top.value("objects").toArray()) { + QJsonObject object = element.toObject(); int id = object.value("id").toInt(); QObject* qObject; putData(object.value("data"), qObject); From 758beff3d9b15391b656af283e1ab2626ed6d815 Mon Sep 17 00:00:00 2001 From: Andrzej Kapolka Date: Tue, 17 Jun 2014 17:06:28 -0700 Subject: [PATCH 10/16] Make sure to use same enum streamer for the ones registered by type and the ones registered by scope/name. --- libraries/metavoxels/src/Bitstream.cpp | 14 +++++++++++--- tests/metavoxels/src/MetavoxelTests.cpp | 2 -- 2 files changed, 11 insertions(+), 5 deletions(-) diff --git a/libraries/metavoxels/src/Bitstream.cpp b/libraries/metavoxels/src/Bitstream.cpp index 9f6ceb9b2e..d78b16aab4 100644 --- a/libraries/metavoxels/src/Bitstream.cpp +++ b/libraries/metavoxels/src/Bitstream.cpp @@ -1621,6 +1621,10 @@ const QHash& Bitstream::getEnumStreamers() { return enumStreamers; } +static QByteArray getEnumName(const char* scope, const char* name) { + return QByteArray(scope) + "::" + name; +} + QHash Bitstream::createEnumStreamers() { QHash enumStreamers; foreach (const QMetaObject* metaObject, getMetaObjects()) { @@ -1628,7 +1632,11 @@ QHash Bitstream::createEnumStreamers() { QMetaEnum metaEnum = metaObject->enumerator(i); const TypeStreamer*& streamer = enumStreamers[ScopeNamePair(metaEnum.scope(), metaEnum.name())]; if (!streamer) { - streamer = new EnumTypeStreamer(metaEnum); + // look for a streamer registered by name + streamer = getTypeStreamers().value(QMetaType::type(getEnumName(metaEnum.scope(), metaEnum.name()))); + if (!streamer) { + streamer = new EnumTypeStreamer(metaEnum); + } } } } @@ -2705,7 +2713,7 @@ QDebug& operator<<(QDebug& debug, const QMetaObject* metaObject) { EnumTypeStreamer::EnumTypeStreamer(const QMetaObject* metaObject, const char* name) : _metaObject(metaObject), _enumName(name), - _name(QByteArray(metaObject->className()) + "::" + name), + _name(getEnumName(metaObject->className(), name)), _bits(-1) { _type = QMetaType::Int; @@ -2713,7 +2721,7 @@ EnumTypeStreamer::EnumTypeStreamer(const QMetaObject* metaObject, const char* na } EnumTypeStreamer::EnumTypeStreamer(const QMetaEnum& metaEnum) : - _name(QByteArray(metaEnum.scope()) + "::" + metaEnum.name()), + _name(getEnumName(metaEnum.scope(), metaEnum.name())), _metaEnum(metaEnum), _bits(-1) { diff --git a/tests/metavoxels/src/MetavoxelTests.cpp b/tests/metavoxels/src/MetavoxelTests.cpp index 85be9a5c8b..4a3010caf4 100644 --- a/tests/metavoxels/src/MetavoxelTests.cpp +++ b/tests/metavoxels/src/MetavoxelTests.cpp @@ -255,8 +255,6 @@ static bool testSerialization(Bitstream::MetadataType metadataType) { jsonWriter << endRead; QByteArray encodedJson = jsonWriter.getDocument().toJson(); - qDebug() << encodedJson; - // and read from JSON JSONReader jsonReader(QJsonDocument::fromJson(encodedJson), Bitstream::ALL_GENERICS); jsonReader >> testObjectReadA; From 5e0cab59ea6a874a91a192a67bfeccef7c3e4d11 Mon Sep 17 00:00:00 2001 From: Andrzej Kapolka Date: Tue, 17 Jun 2014 18:12:06 -0700 Subject: [PATCH 11/16] Some comments. --- libraries/metavoxels/src/Bitstream.h | 83 +++++++++++++++++++++------- 1 file changed, 63 insertions(+), 20 deletions(-) diff --git a/libraries/metavoxels/src/Bitstream.h b/libraries/metavoxels/src/Bitstream.h index 66a42e6991..8af35c86fa 100644 --- a/libraries/metavoxels/src/Bitstream.h +++ b/libraries/metavoxels/src/Bitstream.h @@ -199,12 +199,20 @@ template inline RepeatedValueStreamer& return *this; } -/// A stream for bit-aligned data. +/// A stream for bit-aligned data. Through a combination of code generation, reflection, macros, and templates, provides a +/// serialization mechanism that may be used for both networking and persistent storage. For unreliable networking, the +/// class provides a mapping system that resends mappings for ids until they are acknowledged (and thus persisted). For +/// version-resilient persistence, the class provides a metadata system that maps stored types to local types (or to +/// generic containers), allowing one to add and remove fields to classes without breaking compatibility with previously +/// stored data. class Bitstream : public QObject { Q_OBJECT public: + /// Stores a set of mappings from values to ids written. Typically, one would store these mappings along with the send + /// record of the packet that contained them, persisting the mappings if/when the packet is acknowledged by the remote + /// party. class WriteMappings { public: QHash objectStreamerOffsets; @@ -214,6 +222,9 @@ public: QHash sharedObjectOffsets; }; + /// Stores a set of mappings from ids to values read. Typically, one would store these mappings along with the receive + /// record of the packet that contained them, persisting the mappings if/when the remote party indicates that it + /// has received the local party's acknowledgement of the packet. class ReadMappings { public: QHash objectStreamerValues; @@ -223,11 +234,14 @@ public: QHash sharedObjectValues; }; - /// Registers a metaobject under its name so that instances of it can be streamed. + /// Registers a metaobject under its name so that instances of it can be streamed. Consider using the REGISTER_META_OBJECT + /// at the top level of the source file associated with the class rather than calling this function directly. /// \return zero; the function only returns a value so that it can be used in static initialization static int registerMetaObject(const char* className, const QMetaObject* metaObject); - /// Registers a streamer for the specified Qt-registered type. + /// Registers a streamer for the specified Qt-registered type. Consider using one of the registration macros (such as + /// REGISTER_SIMPLE_TYPE_STREAMER) at the top level of the associated source file rather than calling this function + /// directly. /// \return zero; the function only returns a value so that it can be used in static initialization static int registerTypeStreamer(int type, TypeStreamer* streamer); @@ -237,7 +251,9 @@ public: /// Returns the meta-object registered under the supplied class name, if any. static const QMetaObject* getMetaObject(const QByteArray& className); - /// Returns the list of registered subclasses for the supplied meta-object. + /// Returns the list of registered subclasses for the supplied meta-object. When registered, metaobjects register + /// themselves as subclasses of all of their parents, mostly in order to allow editors to provide lists of available + /// subclasses. static QList getMetaObjectSubClasses(const QMetaObject* metaObject); enum MetadataType { NO_METADATA, HASH_METADATA, FULL_METADATA }; @@ -254,7 +270,8 @@ public: Bitstream(QDataStream& underlying, MetadataType metadataType = NO_METADATA, GenericsMode = NO_GENERICS, QObject* parent = NULL); - /// Substitutes the supplied metaobject for the given class name's default mapping. + /// Substitutes the supplied metaobject for the given class name's default mapping. This is mostly useful for testing the + /// process of mapping between different types, but may in the future be used for permanently renaming classes. void addMetaObjectSubstitution(const QByteArray& className, const QMetaObject* metaObject); /// Substitutes the supplied type for the given type name's default mapping. @@ -773,7 +790,9 @@ template inline Bitstream& Bitstream::operator>>(QHash& return *this; } -/// Tracks state when writing to JSON. +/// Provides a means of writing Bitstream-able data to JSON rather than the usual binary format in a manner that allows it to +/// be manipulated and re-read, converted to binary, etc. To use, create a JSONWriter, stream values in using the << operator, +/// and call getDocument to obtain the JSON data. class JSONWriter { public: @@ -862,10 +881,14 @@ template inline QJsonValue JSONWriter::getData(const QHash> operator. class JSONReader { public: + /// Creates a reader to read from the supplied document. + /// \param genericsMode the generics mode to use: NO_GENERICS to map all types in the document to built-in types, + /// FALLBACK_GENERICS to use generic containers where no matching built-in type is found, or ALL_GENERICS to + /// read all types to generic containers JSONReader(const QJsonDocument& document, Bitstream::GenericsMode genericsMode = Bitstream::NO_GENERICS); void putData(const QJsonValue& data, bool& value); @@ -1049,10 +1072,12 @@ private: Q_DECLARE_METATYPE(const QMetaObject*) -/// Macro for registering streamable meta-objects. +/// Macro for registering streamable meta-objects. Typically, one would use this macro at the top level of the source file +/// associated with the class. #define REGISTER_META_OBJECT(x) static int x##Registration = Bitstream::registerMetaObject(#x, &x::staticMetaObject); -/// Contains a value along with a pointer to its streamer. +/// Contains a value along with a pointer to its streamer. This is stored in QVariants when using fallback generics and +/// no mapping to a built-in type can be found, or when using all generics and the value is any non-simple type. class GenericValue { public: @@ -1071,7 +1096,8 @@ private: Q_DECLARE_METATYPE(GenericValue) -/// Contains a list of property values along with a pointer to their metadata. +/// Contains a list of property values along with a pointer to their metadata. This is stored when using fallback generics +/// and no mapping to a built-in class can be found, or for all QObjects when using all generics. class GenericSharedObject : public SharedObject { Q_OBJECT @@ -1495,16 +1521,19 @@ public: virtual void writeVariant(Bitstream& out, const QVariant& value) const; }; -/// Macro for registering simple type streamers. +/// Macro for registering simple type streamers. Typically, one would use this at the top level of the source file +/// associated with the type. #define REGISTER_SIMPLE_TYPE_STREAMER(X) static int X##Streamer = \ Bitstream::registerTypeStreamer(qMetaTypeId(), new SimpleTypeStreamer()); -/// Macro for registering collection type streamers. +/// Macro for registering collection type (QList, QVector, QSet, QMap) streamers. Typically, one would use this at the top +/// level of the source file associated with the type. #define REGISTER_COLLECTION_TYPE_STREAMER(X) static int x##Streamer = \ Bitstream::registerTypeStreamer(qMetaTypeId(), new CollectionTypeStreamer()); -/// Declares the metatype and the streaming operators. The last lines -/// ensure that the generated file will be included in the link phase. +/// Declares the metatype and the streaming operators. Typically, one would use this immediately after the definition of a +/// type flagged as STREAMABLE in its header file. The last lines ensure that the generated file will be included in the link +/// phase. #ifdef _WIN32 #define DECLARE_STREAMABLE_METATYPE(X) Q_DECLARE_METATYPE(X) \ Bitstream& operator<<(Bitstream& out, const X& obj); \ @@ -1542,6 +1571,9 @@ public: _Pragma(STRINGIFY(unused(_TypePtr##X))) #endif +/// Declares an enum metatype. This is used when one desires to use an enum defined in a QObject outside of that class's +/// direct properties. Typically, one would use this immediately after the definition of the QObject containing the enum +/// in its header file. #define DECLARE_ENUM_METATYPE(S, N) Q_DECLARE_METATYPE(S::N) \ Bitstream& operator<<(Bitstream& out, const S::N& obj); \ Bitstream& operator>>(Bitstream& in, S::N& obj); \ @@ -1550,6 +1582,8 @@ public: template<> inline QJsonValue JSONWriter::getData(const S::N& value) { return (int)value; } \ template<> inline void JSONReader::putData(const QJsonValue& data, S::N& value) { value = (S::N)data.toInt(); } +/// Implements an enum metatype. This performs the implementation of the previous macro, and would normally be used at the +/// top level of the source file associated with the QObject containing the enum. #define IMPLEMENT_ENUM_METATYPE(S, N) \ static int S##N##MetaTypeId = registerEnumMetaType(&S::staticMetaObject, #N); \ Bitstream& operator<<(Bitstream& out, const S::N& obj) { \ @@ -1562,7 +1596,9 @@ public: return in.read(&obj, bits); \ } -/// Registers a simple type and its streamer. +/// Registers a simple type and its streamer. This would typically be used at the top level of the source file associated with +/// the type, and combines the registration of the type with the Qt metatype system with the registration of a simple type +/// streamer. /// \return the metatype id template int registerSimpleMetaType() { int type = qRegisterMetaType(); @@ -1570,7 +1606,8 @@ template int registerSimpleMetaType() { return type; } -/// Registers an enum type and its streamer. +/// Registers an enum type and its streamer. Rather than using this directly, consider using the IMPLEMENT_ENUM_METATYPE +/// macro. /// \return the metatype id template int registerEnumMetaType(const QMetaObject* metaObject, const char* name) { int type = qRegisterMetaType(); @@ -1578,7 +1615,8 @@ template int registerEnumMetaType(const QMetaObject* metaObject, const return type; } -/// Registers a streamable type and its streamer. +/// Registers a streamable type and its streamer. Rather than using this directly, consider using the +/// DECLARE_STREAMABLE_METATYPE macro and the mtc code generation tool. /// \return the metatype id template int registerStreamableMetaType() { int type = qRegisterMetaType(); @@ -1586,7 +1624,9 @@ template int registerStreamableMetaType() { return type; } -/// Registers a collection type and its streamer. +/// Registers a collection type and its streamer. This would typically be used at the top level of the source file associated +/// with the type, and combines the registration of the type with the Qt metatype system with the registration of a collection +/// type streamer. /// \return the metatype id template int registerCollectionMetaType() { int type = qRegisterMetaType(); @@ -1594,7 +1634,9 @@ template int registerCollectionMetaType() { return type; } -/// Flags a class as streamable (use as you would Q_OBJECT). +/// Flags a class as streamable. Use as you would Q_OBJECT: as the first line of a class definition, before any members or +/// access qualifiers. Typically, one would follow the definition with DECLARE_STREAMABLE_METATYPE. The mtc tool looks for +/// this macro in order to generate streaming (etc.) code for types. #define STREAMABLE public: \ static const int Type; \ static const QVector& getMetaFields(); \ @@ -1604,7 +1646,8 @@ template int registerCollectionMetaType() { private: \ static QHash createFieldIndices(); -/// Flags a field or base class as streaming. +/// Flags a (public) field within or base class of a streamable type as streaming. Use before all base classes or fields that +/// you want to stream. #define STREAM #endif // hifi_Bitstream_h From b8e6660e1948f2e80482a8b641b9781ebb68195f Mon Sep 17 00:00:00 2001 From: Andrzej Kapolka Date: Tue, 17 Jun 2014 19:15:29 -0700 Subject: [PATCH 12/16] More commentation. --- libraries/metavoxels/src/Bitstream.cpp | 15 ++++++++-- libraries/metavoxels/src/Bitstream.h | 38 +++++++++++++++++++++++--- 2 files changed, 47 insertions(+), 6 deletions(-) diff --git a/libraries/metavoxels/src/Bitstream.cpp b/libraries/metavoxels/src/Bitstream.cpp index d78b16aab4..ef5b9b7906 100644 --- a/libraries/metavoxels/src/Bitstream.cpp +++ b/libraries/metavoxels/src/Bitstream.cpp @@ -1879,6 +1879,7 @@ QJsonDocument JSONWriter::getDocument() const { } JSONReader::JSONReader(const QJsonDocument& document, Bitstream::GenericsMode genericsMode) { + // create and map the type streamers in order QJsonObject top = document.object(); foreach (const QJsonValue& element, top.value("types").toArray()) { QJsonObject type = element.toObject(); @@ -1889,10 +1890,11 @@ JSONReader::JSONReader(const QJsonDocument& document, Bitstream::GenericsMode ge baseStreamer = Bitstream::getEnumStreamersByName().value(latinName); } if (!baseStreamer && genericsMode == Bitstream::NO_GENERICS) { - continue; + continue; // no built-in type and no generics allowed; we give up } QString category = type.value("category").toString(); if (!baseStreamer || genericsMode == Bitstream::ALL_GENERICS) { + // create a generic streamer TypeStreamerPointer streamer; if (category == "ENUM") { QVector values; @@ -1933,6 +1935,7 @@ JSONReader::JSONReader(const QJsonDocument& document, Bitstream::GenericsMode ge static_cast(streamer.data())->_weakSelf = streamer; continue; } + // create a mapped streamer, determining along the way whether it matches our base if (category == "ENUM") { QHash mappings; int highestValue = 0; @@ -1949,6 +1952,7 @@ JSONReader::JSONReader(const QJsonDocument& document, Bitstream::GenericsMode ge } matches &= (value == mapping); } + // if everything matches our built-in enum, we can use that, which will be faster if (matches) { _typeStreamers.insert(name, baseStreamer->getSelf()); } else { @@ -1967,6 +1971,7 @@ JSONReader::JSONReader(const QJsonDocument& document, Bitstream::GenericsMode ge fields.append(StreamerIndexPair(streamer, index)); matches &= (index == i && streamer == metaFields.at(i).getStreamer()); } + // if everything matches our built-in streamable, we can use that, which will be faster if (matches) { _typeStreamers.insert(name, baseStreamer->getSelf()); } else { @@ -2001,6 +2006,7 @@ JSONReader::JSONReader(const QJsonDocument& document, Bitstream::GenericsMode ge } } + // create and map the object streamers in order foreach (const QJsonValue& element, top.value("classes").toArray()) { QJsonObject clazz = element.toObject(); QString name = clazz.value("name").toString(); @@ -2008,9 +2014,10 @@ JSONReader::JSONReader(const QJsonDocument& document, Bitstream::GenericsMode ge const ObjectStreamer* baseStreamer = Bitstream::getObjectStreamers().value( Bitstream::getMetaObjects().value(latinName)); if (!baseStreamer && genericsMode == Bitstream::NO_GENERICS) { - continue; + continue; // no built-in class and no generics allowed; we give up } if (!baseStreamer || genericsMode == Bitstream::ALL_GENERICS) { + // create a generic streamer QVector properties; foreach (const QJsonValue& property, clazz.value("properties").toArray()) { QJsonObject object = property.toObject(); @@ -2023,6 +2030,7 @@ JSONReader::JSONReader(const QJsonDocument& document, Bitstream::GenericsMode ge static_cast(streamer.data())->_weakSelf = streamer; continue; } + // create a mapped streamer, determining along the way whether it matches our base const QMetaObject* metaObject = baseStreamer->getMetaObject(); const QVector& baseProperties = baseStreamer->getProperties(); QVector properties; @@ -2039,6 +2047,7 @@ JSONReader::JSONReader(const QJsonDocument& document, Bitstream::GenericsMode ge matches &= (typeStreamer == baseProperty.first && metaProperty.propertyIndex() == baseProperty.second.propertyIndex()); } + // if everything matches our built-in type, we can use that directly, which will be faster if (matches) { _objectStreamers.insert(name, baseStreamer->getSelf()); } else { @@ -2046,6 +2055,7 @@ JSONReader::JSONReader(const QJsonDocument& document, Bitstream::GenericsMode ge } } + // create and map the objects in order foreach (const QJsonValue& element, top.value("objects").toArray()) { QJsonObject object = element.toObject(); int id = object.value("id").toInt(); @@ -2056,6 +2066,7 @@ JSONReader::JSONReader(const QJsonDocument& document, Bitstream::GenericsMode ge } } + // prepare the contents for extraction _contents = top.value("contents").toArray(); _contentsIterator = _contents.constBegin(); } diff --git a/libraries/metavoxels/src/Bitstream.h b/libraries/metavoxels/src/Bitstream.h index 8af35c86fa..64765833b9 100644 --- a/libraries/metavoxels/src/Bitstream.h +++ b/libraries/metavoxels/src/Bitstream.h @@ -205,6 +205,36 @@ template inline RepeatedValueStreamer& /// version-resilient persistence, the class provides a metadata system that maps stored types to local types (or to /// generic containers), allowing one to add and remove fields to classes without breaking compatibility with previously /// stored data. +/// +/// The basic usage requires one to create a Bitstream that wraps an underlying QDataStream, specifying the metadata type +/// desired and (for readers) the generics mode. Then, one uses the << or >> operators to write or read values to/from +/// the stream (a stream instance may be used for reading or writing, but not both). For write streams, the flush +/// function should be called on completion to write any partial data. +/// +/// Polymorphic types are supported via the QVariant and QObject*/SharedObjectPointer types. When you write a QVariant or +/// QObject, the type or class name (at minimum) is written to the stream. When you read a QVariant or QObject, the default +/// behavior is to look for a corresponding registered type with the written name. With hash metadata, Bitstream can verify +/// that the local type matches the stream type, throwing the streamed version away if not. With full metadata, Bitstream can +/// create a mapping from the streamed type to the local type that applies the intersection of the two types' fields. +/// +/// To register types for streaming, select from the provided templates and macros. To register a QObject (or SharedObject) +/// subclass, use REGISTER_META_OBJECT in the class's source file. To register a streamable class for use in QVariant, use +/// the STREAMABLE/STREAM/DECLARE_STREAMABLE_METATYPE macros and use the mtc tool to generate the associated implementation +/// code. To register a QObject enum for use outside the QObject's direct properties, use the +/// DECLARE_ENUM_METATYPE/IMPLEMENT_ENUM_METATYPE macro pair. To register a collection type (QList, QVector, QSet, QMap) of +/// streamable types, use the registerCollectionMetaType template function. +/// +/// Delta-streaming is supported through the writeDelta/readDelta functions (analogous to <>), which accept a reference +/// value and stream only the information necessary to turn the reference into the target value. This assumes that the +/// reference value provided when reading is identical to the one used when writing. +/// +/// Special delta handling is provided for objects tracked by SharedObjectPointers. SharedObjects have IDs (representing their +/// unique identity on one system) as well as origin IDs (representing their identity as preserved over the course of +/// mutation). Once a shared object with a given ID is written (and its mapping persisted), that object's state will not be +/// written again; only its ID will be sent. However, if an object with a new ID but the same origin ID is written, that +/// object's state will be encoded as a delta between the previous object and the new one. So, to transmit delta-encoded +/// objects, one should treat each local SharedObject instance as immutable, replacing it when mutated with a cloned instance +/// generated by calling SharedObject::clone(true) and applying the desired changes. class Bitstream : public QObject { Q_OBJECT @@ -1001,7 +1031,7 @@ protected: friend class Bitstream; const QMetaObject* _metaObject; - ObjectStreamerPointer _self; + ObjectStreamerPointer _self; ///< set/used for built-in classes (never deleted), to obtain shared pointers }; /// A streamer that maps to a local class. @@ -1050,7 +1080,7 @@ private: friend class JSONReader; QByteArray _name; - WeakObjectStreamerPointer _weakSelf; + WeakObjectStreamerPointer _weakSelf; ///< promoted to strong references in GenericSharedObject when reading QVector _properties; QByteArray _hash; }; @@ -1184,7 +1214,7 @@ protected: friend class Bitstream; int _type; - TypeStreamerPointer _self; + TypeStreamerPointer _self; ///< set/used for built-in types (never deleted), to obtain shared pointers }; QDebug& operator<<(QDebug& debug, const TypeStreamer* typeStreamer); @@ -1278,7 +1308,7 @@ protected: friend class JSONReader; QByteArray _name; - WeakTypeStreamerPointer _weakSelf; + WeakTypeStreamerPointer _weakSelf; ///< promoted to strong references in GenericValue when reading }; /// A streamer for generic enums. From e0689b14840a2917b88eed3a3e26c347de1c2a7f Mon Sep 17 00:00:00 2001 From: Andrzej Kapolka Date: Wed, 18 Jun 2014 12:04:41 -0700 Subject: [PATCH 13/16] Fixed compile error on Xcode. --- libraries/metavoxels/src/Bitstream.cpp | 14 +++++++------- 1 file changed, 7 insertions(+), 7 deletions(-) diff --git a/libraries/metavoxels/src/Bitstream.cpp b/libraries/metavoxels/src/Bitstream.cpp index 297dfeb079..44342abe33 100644 --- a/libraries/metavoxels/src/Bitstream.cpp +++ b/libraries/metavoxels/src/Bitstream.cpp @@ -1901,9 +1901,9 @@ JSONReader::JSONReader(const QJsonDocument& document, Bitstream::GenericsMode ge int highestValue = 0; foreach (const QJsonValue& value, type.value("values").toArray()) { QJsonObject object = value.toObject(); - int value = object.value("value").toInt(); - highestValue = qMax(value, highestValue); - values.append(NameIntPair(object.value("key").toString().toLatin1(), value)); + int intValue = object.value("value").toInt(); + highestValue = qMax(intValue, highestValue); + values.append(NameIntPair(object.value("key").toString().toLatin1(), intValue)); } streamer = TypeStreamerPointer(new GenericEnumTypeStreamer(latinName, values, getBitsForHighestValue(highestValue), QByteArray())); @@ -1944,13 +1944,13 @@ JSONReader::JSONReader(const QJsonDocument& document, Bitstream::GenericsMode ge bool matches = (array.size() == metaEnum.keyCount()); foreach (const QJsonValue& value, array) { QJsonObject object = value.toObject(); - int value = object.value("value").toInt(); - highestValue = qMax(value, highestValue); + int intValue = object.value("value").toInt(); + highestValue = qMax(intValue, highestValue); int mapping = metaEnum.keyToValue(object.value("key").toString().toLatin1()); if (mapping != -1) { - mappings.insert(value, mapping); + mappings.insert(intValue, mapping); } - matches &= (value == mapping); + matches &= (intValue == mapping); } // if everything matches our built-in enum, we can use that, which will be faster if (matches) { From eea45e98d7895494eec70e5700f9e0713e02264d Mon Sep 17 00:00:00 2001 From: Andrzej Kapolka Date: Wed, 18 Jun 2014 12:22:45 -0700 Subject: [PATCH 14/16] Fix to include GLM. --- tools/bitstream2json/CMakeLists.txt | 6 ++++++ tools/json2bitstream/CMakeLists.txt | 6 ++++++ 2 files changed, 12 insertions(+) diff --git a/tools/bitstream2json/CMakeLists.txt b/tools/bitstream2json/CMakeLists.txt index 6483c024a7..fde80b4d33 100644 --- a/tools/bitstream2json/CMakeLists.txt +++ b/tools/bitstream2json/CMakeLists.txt @@ -9,6 +9,9 @@ set(TARGET_NAME bitstream2json) set(ROOT_DIR ../..) set(MACRO_DIR "${ROOT_DIR}/cmake/macros") +# setup for find modules +set(CMAKE_MODULE_PATH ${CMAKE_MODULE_PATH} "${CMAKE_CURRENT_SOURCE_DIR}/../../cmake/modules/") + find_package(Qt5 COMPONENTS Network Script Widgets) include(${MACRO_DIR}/SetupHifiProject.cmake) @@ -17,4 +20,7 @@ setup_hifi_project(${TARGET_NAME} TRUE) link_hifi_library(metavoxels ${TARGET_NAME} "${ROOT_DIR}") link_hifi_library(shared ${TARGET_NAME} "${ROOT_DIR}") +include(${MACRO_DIR}/IncludeGLM.cmake) +include_glm(${TARGET_NAME} "${ROOT_DIR}") + target_link_libraries(${TARGET_NAME} Qt5::Network Qt5::Widgets Qt5::Script) diff --git a/tools/json2bitstream/CMakeLists.txt b/tools/json2bitstream/CMakeLists.txt index 799f3bcc4c..51382d5858 100644 --- a/tools/json2bitstream/CMakeLists.txt +++ b/tools/json2bitstream/CMakeLists.txt @@ -9,6 +9,9 @@ set(TARGET_NAME json2bitstream) set(ROOT_DIR ../..) set(MACRO_DIR "${ROOT_DIR}/cmake/macros") +# setup for find modules +set(CMAKE_MODULE_PATH ${CMAKE_MODULE_PATH} "${CMAKE_CURRENT_SOURCE_DIR}/../../cmake/modules/") + find_package(Qt5 COMPONENTS Network Script Widgets) include(${MACRO_DIR}/SetupHifiProject.cmake) @@ -17,4 +20,7 @@ setup_hifi_project(${TARGET_NAME} TRUE) link_hifi_library(metavoxels ${TARGET_NAME} "${ROOT_DIR}") link_hifi_library(shared ${TARGET_NAME} "${ROOT_DIR}") +include(${MACRO_DIR}/IncludeGLM.cmake) +include_glm(${TARGET_NAME} "${ROOT_DIR}") + target_link_libraries(${TARGET_NAME} Qt5::Network Qt5::Widgets Qt5::Script) From 26159a05d925236cbe152ecc5479788d27129a7a Mon Sep 17 00:00:00 2001 From: Andrzej Kapolka Date: Wed, 18 Jun 2014 13:09:45 -0700 Subject: [PATCH 15/16] Fix for sporadic test failure due to hash ordering. --- tests/metavoxels/src/MetavoxelTests.cpp | 34 +++++++++++++++---------- 1 file changed, 20 insertions(+), 14 deletions(-) diff --git a/tests/metavoxels/src/MetavoxelTests.cpp b/tests/metavoxels/src/MetavoxelTests.cpp index 4a3010caf4..c9bce27cc3 100644 --- a/tests/metavoxels/src/MetavoxelTests.cpp +++ b/tests/metavoxels/src/MetavoxelTests.cpp @@ -83,7 +83,7 @@ static TestSharedObjectA::TestFlags getRandomTestFlags() { return flags; } -static QScriptValue createRandomScriptValue(bool complex = false) { +static QScriptValue createRandomScriptValue(bool complex = false, bool ensureHashOrder = false) { scriptObjectsCreated++; switch (randIntInRange(0, complex ? 5 : 3)) { case 0: @@ -108,31 +108,37 @@ static QScriptValue createRandomScriptValue(bool complex = false) { } default: { QScriptValue value = ScriptCache::getInstance()->getEngine()->newObject(); - if (randomBoolean()) { + if (ensureHashOrder) { + // we can't depend on the iteration order, so if we need it to be the same (as when comparing bytes), we + // can only have one property value.setProperty("foo", createRandomScriptValue()); - } - if (randomBoolean()) { - value.setProperty("bar", createRandomScriptValue()); - } - if (randomBoolean()) { - value.setProperty("baz", createRandomScriptValue()); - } - if (randomBoolean()) { - value.setProperty("bong", createRandomScriptValue()); + } else { + if (randomBoolean()) { + value.setProperty("foo", createRandomScriptValue()); + } + if (randomBoolean()) { + value.setProperty("bar", createRandomScriptValue()); + } + if (randomBoolean()) { + value.setProperty("baz", createRandomScriptValue()); + } + if (randomBoolean()) { + value.setProperty("bong", createRandomScriptValue()); + } } return value; } } } -static TestMessageC createRandomMessageC() { +static TestMessageC createRandomMessageC(bool ensureHashOrder = false) { TestMessageC message; message.foo = randomBoolean(); message.bar = rand(); message.baz = randFloat(); message.bong.foo = createRandomBytes(); message.bong.baz = getRandomTestEnum(); - message.bizzle = createRandomScriptValue(true); + message.bizzle = createRandomScriptValue(true, ensureHashOrder); return message; } @@ -146,7 +152,7 @@ static bool testSerialization(Bitstream::MetadataType metadataType) { SharedObjectPointer testObjectWrittenB = new TestSharedObjectB(randFloat(), createRandomBytes(), TestSharedObjectB::THIRD_TEST_ENUM, TestSharedObjectB::SECOND_TEST_FLAG); out << testObjectWrittenB; - TestMessageC messageWritten = createRandomMessageC(); + TestMessageC messageWritten = createRandomMessageC(true); out << QVariant::fromValue(messageWritten); QByteArray endWritten = "end"; out << endWritten; From 6ec562931f3a8167920be923c523ab6a1bda23b6 Mon Sep 17 00:00:00 2001 From: Andrzej Kapolka Date: Wed, 18 Jun 2014 14:44:11 -0700 Subject: [PATCH 16/16] Some additional comments for DatagramSequencer. --- libraries/metavoxels/src/DatagramSequencer.h | 44 +++++++++++++++++++- 1 file changed, 43 insertions(+), 1 deletion(-) diff --git a/libraries/metavoxels/src/DatagramSequencer.h b/libraries/metavoxels/src/DatagramSequencer.h index cf6ded74da..5ac88556f0 100644 --- a/libraries/metavoxels/src/DatagramSequencer.h +++ b/libraries/metavoxels/src/DatagramSequencer.h @@ -23,18 +23,50 @@ class ReliableChannel; -/// Performs simple datagram sequencing, packet fragmentation and reassembly. +/// Performs datagram sequencing, packet fragmentation and reassembly. Works with Bitstream to provide methods to send and +/// receive data over UDP with varying reliability and latency characteristics. To use, create a DatagramSequencer with the +/// fixed-size header that will be included with all outgoing datagrams and expected in all incoming ones (the contents of the +/// header are not checked on receive, only skipped over, and may be modified by the party that actually send the +/// datagram--this means that the header may include dynamically generated data, as long as its size remains fixed). Connect +/// the readyToWrite signal to a slot that will actually transmit the datagram to the remote party. When a datagram is +/// received from that party, call receivedDatagram with its contents. +/// +/// A "packet" represents a batch of data sent at one time (split into one or more datagrams sized below the MTU). Packets are +/// received in full and in order or not at all (that is, a packet being assembled is dropped as soon as a fragment from the +/// next packet is received). Packets can be any size, but the larger a packet is, the more likely it is to be dropped--so, +/// it's better to keep packet sizes close to the MTU. To write a packet, call startPacket, write data to the returned +/// Bitstream, then call endPacket (which will result in one or more firings of readyToWrite). Data written in this way is not +/// guaranteed to be received, but if it is received, it will arrive in order. This is a good way to transmit delta state: +/// state that represents the change between the last acknowledged state and the current state (which, if not received, will +/// not be resent as-is; instead, it will be replaced by up-to-date new deltas). +/// +/// There are two methods for sending reliable data. The first, for small messages that require minimum-latency processing, is +/// the high priority messaging system. When you call sendHighPriorityMessage, the message that you send will be included with +/// every outgoing packet until it is acknowledged. When the receiving party first sees the message, it will fire a +/// receivedHighPriorityMessage signal. +/// +/// The second method employs a set of independent reliable channels multiplexed onto the packet stream. These channels are +/// created lazily through the getReliableOutputChannel/getReliableInputChannel functions. Output channels contain buffers +/// to which one may write either arbitrary data (as a QIODevice) or messages (as QVariants), or switch between the two. +/// Each time a packet is sent, data pending for reliable output channels is added, in proportion to their relative priorities, +/// until the packet size limit set by setMaxPacketSize is reached. On the receive side, the streams are reconstructed and +/// (again, depending on whether messages are enabled) either the QIODevice reports that data is available, or, when a complete +/// message is decoded, the receivedMessage signal is fired. class DatagramSequencer : public QObject { Q_OBJECT public: + /// Contains the content of a high-priority message along with the number of the first packet in which it was sent. class HighPriorityMessage { public: QVariant data; int firstPacketNumber; }; + /// Creates a new datagram sequencer. + /// \param datagramHeader the content of the header that will be prepended to each outgoing datagram and whose length + /// will be skipped over in each incoming datagram DatagramSequencer(const QByteArray& datagramHeader = QByteArray(), QObject* parent = NULL); /// Returns a reference to the weak hash mapping remote ids to shared objects. @@ -267,15 +299,24 @@ class ReliableChannel : public QObject { public: + /// Returns the channel's index in the sequencer's channel map. int getIndex() const { return _index; } + /// Returns a reference to the buffer used to write/read data to/from this channel. CircularBuffer& getBuffer() { return _buffer; } + + /// Returns a reference to the data stream created on this channel's buffer. QDataStream& getDataStream() { return _dataStream; } + + /// Returns a reference to the bitstream created on this channel's data stream. Bitstream& getBitstream() { return _bitstream; } + /// Sets the channel priority, which determines how much of this channel's data (in proportion to the other channels) to + /// include in each outgoing packet. void setPriority(float priority) { _priority = priority; } float getPriority() const { return _priority; } + /// Returns the number of bytes available to read from this channel. int getBytesAvailable() const; /// Sets whether we expect to write/read framed messages. @@ -287,6 +328,7 @@ public: signals: + /// Fired when a framed message has been received on this channel. void receivedMessage(const QVariant& message); private slots: