diff --git a/CMakeLists.txt b/CMakeLists.txt index cb1e4224cf..a399e11168 100644 --- a/CMakeLists.txt +++ b/CMakeLists.txt @@ -49,4 +49,5 @@ add_subdirectory(assignment-client) add_subdirectory(domain-server) add_subdirectory(interface) add_subdirectory(tests) +add_subdirectory(tools) add_subdirectory(voxel-edit) diff --git a/cmake/macros/AutoMTC.cmake b/cmake/macros/AutoMTC.cmake index 1682b9cd56..6f0216de7f 100644 --- a/cmake/macros/AutoMTC.cmake +++ b/cmake/macros/AutoMTC.cmake @@ -9,13 +9,9 @@ # macro(AUTO_MTC TARGET ROOT_DIR) - if (NOT TARGET mtc) - add_subdirectory("${ROOT_DIR}/tools/mtc" "${ROOT_DIR}/tools/mtc") - endif () - set(AUTOMTC_SRC ${TARGET}_automtc.cpp) file(GLOB INCLUDE_FILES src/*.h) add_custom_command(OUTPUT ${AUTOMTC_SRC} COMMAND mtc -o ${AUTOMTC_SRC} ${INCLUDE_FILES} DEPENDS mtc ${INCLUDE_FILES}) -endmacro() \ No newline at end of file +endmacro() diff --git a/libraries/metavoxels/src/Bitstream.cpp b/libraries/metavoxels/src/Bitstream.cpp index e5cf34054d..44342abe33 100644 --- a/libraries/metavoxels/src/Bitstream.cpp +++ b/libraries/metavoxels/src/Bitstream.cpp @@ -33,6 +33,7 @@ REGISTER_SIMPLE_TYPE_STREAMER(QByteArray) REGISTER_SIMPLE_TYPE_STREAMER(QColor) REGISTER_SIMPLE_TYPE_STREAMER(QScriptValue) REGISTER_SIMPLE_TYPE_STREAMER(QString) +REGISTER_SIMPLE_TYPE_STREAMER(QVariant) REGISTER_SIMPLE_TYPE_STREAMER(QUrl) REGISTER_SIMPLE_TYPE_STREAMER(QVariantList) REGISTER_SIMPLE_TYPE_STREAMER(QVariantHash) @@ -1178,7 +1179,9 @@ Bitstream& Bitstream::operator>(ObjectStreamerPointer& streamer) { } for (int i = 0; i < localProperties.size(); i++) { const StreamerPropertyPair& property = properties.at(i); - if (localProperties.at(i).first != property.first || property.second.propertyIndex() != i) { + const StreamerPropertyPair& localProperty = localProperties.at(i); + if (property.first != localProperty.first || + property.second.propertyIndex() != localProperty.second.propertyIndex()) { streamer = ObjectStreamerPointer(new MappedObjectStreamer(metaObject, properties)); return *this; } @@ -1618,6 +1621,10 @@ const QHash& Bitstream::getEnumStreamers() { return enumStreamers; } +static QByteArray getEnumName(const char* scope, const char* name) { + return QByteArray(scope) + "::" + name; +} + QHash Bitstream::createEnumStreamers() { QHash enumStreamers; foreach (const QMetaObject* metaObject, getMetaObjects()) { @@ -1625,7 +1632,11 @@ QHash Bitstream::createEnumStreamers() { QMetaEnum metaEnum = metaObject->enumerator(i); const TypeStreamer*& streamer = enumStreamers[ScopeNamePair(metaEnum.scope(), metaEnum.name())]; if (!streamer) { - streamer = new EnumTypeStreamer(metaEnum); + // look for a streamer registered by name + streamer = getTypeStreamers().value(QMetaType::type(getEnumName(metaEnum.scope(), metaEnum.name()))); + if (!streamer) { + streamer = new EnumTypeStreamer(metaEnum); + } } } } @@ -1657,6 +1668,565 @@ const TypeStreamer* Bitstream::createInvalidTypeStreamer() { return streamer; } +QJsonValue JSONWriter::getData(bool value) { + return value; +} + +QJsonValue JSONWriter::getData(int value) { + return value; +} + +QJsonValue JSONWriter::getData(uint value) { + return (int)value; +} + +QJsonValue JSONWriter::getData(float value) { + return (double)value; +} + +QJsonValue JSONWriter::getData(const QByteArray& value) { + return QString(value.toPercentEncoding()); +} + +QJsonValue JSONWriter::getData(const QColor& value) { + return value.name(); +} + +QJsonValue JSONWriter::getData(const QScriptValue& value) { + QJsonObject object; + if (value.isUndefined()) { + object.insert("type", QString("UNDEFINED")); + + } else if (value.isNull()) { + object.insert("type", QString("NULL")); + + } else if (value.isBool()) { + object.insert("type", QString("BOOL")); + object.insert("value", value.toBool()); + + } else if (value.isNumber()) { + object.insert("type", QString("NUMBER")); + object.insert("value", value.toNumber()); + + } else if (value.isString()) { + object.insert("type", QString("STRING")); + object.insert("value", value.toString()); + + } else if (value.isVariant()) { + object.insert("type", QString("VARIANT")); + object.insert("value", getData(value.toVariant())); + + } else if (value.isQObject()) { + object.insert("type", QString("QOBJECT")); + object.insert("value", getData(value.toQObject())); + + } else if (value.isQMetaObject()) { + object.insert("type", QString("QMETAOBJECT")); + object.insert("value", getData(value.toQMetaObject())); + + } else if (value.isDate()) { + object.insert("type", QString("DATE")); + object.insert("value", getData(value.toDateTime())); + + } else if (value.isRegExp()) { + object.insert("type", QString("REGEXP")); + object.insert("value", getData(value.toRegExp())); + + } else if (value.isArray()) { + object.insert("type", QString("ARRAY")); + QJsonArray array; + int length = value.property(ScriptCache::getInstance()->getLengthString()).toInt32(); + for (int i = 0; i < length; i++) { + array.append(getData(value.property(i))); + } + object.insert("value", array); + + } else if (value.isObject()) { + object.insert("type", QString("OBJECT")); + QJsonObject valueObject; + for (QScriptValueIterator it(value); it.hasNext(); ) { + it.next(); + valueObject.insert(it.name(), getData(it.value())); + } + object.insert("value", valueObject); + + } else { + object.insert("type", QString("INVALID")); + } + return object; +} + +QJsonValue JSONWriter::getData(const QString& value) { + return value; +} + +QJsonValue JSONWriter::getData(const QUrl& value) { + return value.toString(); +} + +QJsonValue JSONWriter::getData(const QDateTime& value) { + return (qsreal)value.toMSecsSinceEpoch(); +} + +QJsonValue JSONWriter::getData(const QRegExp& value) { + QJsonObject object; + object.insert("pattern", value.pattern()); + object.insert("caseSensitivity", (int)value.caseSensitivity()); + object.insert("patternSyntax", (int)value.patternSyntax()); + object.insert("minimal", value.isMinimal()); + return object; +} + +QJsonValue JSONWriter::getData(const glm::vec3& value) { + QJsonArray array; + array.append(value.x); + array.append(value.y); + array.append(value.z); + return array; +} + +QJsonValue JSONWriter::getData(const glm::quat& value) { + QJsonArray array; + array.append(value.x); + array.append(value.y); + array.append(value.z); + array.append(value.w); + return array; +} + +QJsonValue JSONWriter::getData(const QMetaObject* metaObject) { + if (!metaObject) { + return QJsonValue(); + } + const ObjectStreamer* streamer = Bitstream::getObjectStreamers().value(metaObject); + addObjectStreamer(streamer); + return QString(streamer->getName()); +} + +QJsonValue JSONWriter::getData(const QVariant& value) { + if (!value.isValid()) { + return QJsonValue(); + } + const TypeStreamer* streamer = Bitstream::getTypeStreamers().value(value.userType()); + if (streamer) { + return streamer->getJSONVariantData(*this, value); + } else { + qWarning() << "Non-streamable type:" << value.typeName(); + return QJsonValue(); + } +} + +QJsonValue JSONWriter::getData(const SharedObjectPointer& object) { + if (object) { + addSharedObject(object); + return object->getID(); + } else { + return 0; + } +} + +QJsonValue JSONWriter::getData(const QObject* object) { + if (!object) { + return QJsonValue(); + } + const QMetaObject* metaObject = object->metaObject(); + const ObjectStreamer* streamer = (metaObject == &GenericSharedObject::staticMetaObject) ? + static_cast(object)->getStreamer().data() : + Bitstream::getObjectStreamers().value(metaObject); + return streamer->getJSONData(*this, object); +} + +QJsonValue JSONWriter::getData(const GenericValue& value) { + return value.getStreamer()->getJSONData(*this, value.getValue()); +} + +void JSONWriter::addTypeStreamer(const TypeStreamer* streamer) { + if (!_typeStreamerNames.contains(streamer->getName())) { + _typeStreamerNames.insert(streamer->getName()); + + QJsonValue metadata = streamer->getJSONMetadata(*this); + if (!metadata.isNull()) { + _typeStreamers.append(metadata); + } + } +} + +void JSONWriter::addObjectStreamer(const ObjectStreamer* streamer) { + if (!_objectStreamerNames.contains(streamer->getName())) { + _objectStreamerNames.insert(streamer->getName()); + _objectStreamers.append(streamer->getJSONMetadata(*this)); + } +} + +void JSONWriter::addSharedObject(const SharedObjectPointer& object) { + if (!_sharedObjectIDs.contains(object->getID())) { + _sharedObjectIDs.insert(object->getID()); + + QJsonObject sharedObject; + sharedObject.insert("id", object->getID()); + sharedObject.insert("data", getData(static_cast(object.data()))); + _sharedObjects.append(sharedObject); + } +} + +QJsonDocument JSONWriter::getDocument() const { + QJsonObject top; + top.insert("contents", _contents); + top.insert("objects", _sharedObjects); + top.insert("classes", _objectStreamers); + top.insert("types", _typeStreamers); + return QJsonDocument(top); +} + +JSONReader::JSONReader(const QJsonDocument& document, Bitstream::GenericsMode genericsMode) { + // create and map the type streamers in order + QJsonObject top = document.object(); + foreach (const QJsonValue& element, top.value("types").toArray()) { + QJsonObject type = element.toObject(); + QString name = type.value("name").toString(); + QByteArray latinName = name.toLatin1(); + const TypeStreamer* baseStreamer = Bitstream::getTypeStreamers().value(QMetaType::type(latinName)); + if (!baseStreamer) { + baseStreamer = Bitstream::getEnumStreamersByName().value(latinName); + } + if (!baseStreamer && genericsMode == Bitstream::NO_GENERICS) { + continue; // no built-in type and no generics allowed; we give up + } + QString category = type.value("category").toString(); + if (!baseStreamer || genericsMode == Bitstream::ALL_GENERICS) { + // create a generic streamer + TypeStreamerPointer streamer; + if (category == "ENUM") { + QVector values; + int highestValue = 0; + foreach (const QJsonValue& value, type.value("values").toArray()) { + QJsonObject object = value.toObject(); + int intValue = object.value("value").toInt(); + highestValue = qMax(intValue, highestValue); + values.append(NameIntPair(object.value("key").toString().toLatin1(), intValue)); + } + streamer = TypeStreamerPointer(new GenericEnumTypeStreamer(latinName, + values, getBitsForHighestValue(highestValue), QByteArray())); + + } else if (category == "STREAMABLE") { + QVector fields; + foreach (const QJsonValue& field, type.value("fields").toArray()) { + QJsonObject object = field.toObject(); + fields.append(StreamerNamePair(getTypeStreamer(object.value("type").toString()), + object.value("name").toString().toLatin1())); + } + streamer = TypeStreamerPointer(new GenericStreamableTypeStreamer(latinName, + fields, QByteArray())); + + } else if (category == "LIST") { + streamer = TypeStreamerPointer(new GenericListTypeStreamer(latinName, + getTypeStreamer(type.value("valueType").toString()))); + + } else if (category == "SET") { + streamer = TypeStreamerPointer(new GenericSetTypeStreamer(latinName, + getTypeStreamer(type.value("valueType").toString()))); + + } else if (category == "MAP") { + streamer = TypeStreamerPointer(new GenericMapTypeStreamer(latinName, + getTypeStreamer(type.value("keyType").toString()), + getTypeStreamer(type.value("valueType").toString()))); + } + _typeStreamers.insert(name, streamer); + static_cast(streamer.data())->_weakSelf = streamer; + continue; + } + // create a mapped streamer, determining along the way whether it matches our base + if (category == "ENUM") { + QHash mappings; + int highestValue = 0; + QMetaEnum metaEnum = baseStreamer->getMetaEnum(); + QJsonArray array = type.value("values").toArray(); + bool matches = (array.size() == metaEnum.keyCount()); + foreach (const QJsonValue& value, array) { + QJsonObject object = value.toObject(); + int intValue = object.value("value").toInt(); + highestValue = qMax(intValue, highestValue); + int mapping = metaEnum.keyToValue(object.value("key").toString().toLatin1()); + if (mapping != -1) { + mappings.insert(intValue, mapping); + } + matches &= (intValue == mapping); + } + // if everything matches our built-in enum, we can use that, which will be faster + if (matches) { + _typeStreamers.insert(name, baseStreamer->getSelf()); + } else { + _typeStreamers.insert(name, TypeStreamerPointer(new MappedEnumTypeStreamer(baseStreamer, + getBitsForHighestValue(highestValue), mappings))); + } + } else if (category == "STREAMABLE") { + QVector fields; + QJsonArray array = type.value("fields").toArray(); + const QVector& metaFields = baseStreamer->getMetaFields(); + bool matches = (array.size() == metaFields.size()); + for (int i = 0; i < array.size(); i++) { + QJsonObject object = array.at(i).toObject(); + TypeStreamerPointer streamer = getTypeStreamer(object.value("type").toString()); + int index = baseStreamer->getFieldIndex(object.value("name").toString().toLatin1()); + fields.append(StreamerIndexPair(streamer, index)); + matches &= (index == i && streamer == metaFields.at(i).getStreamer()); + } + // if everything matches our built-in streamable, we can use that, which will be faster + if (matches) { + _typeStreamers.insert(name, baseStreamer->getSelf()); + } else { + _typeStreamers.insert(name, TypeStreamerPointer(new MappedStreamableTypeStreamer(baseStreamer, fields))); + } + } else if (category == "LIST") { + TypeStreamerPointer valueStreamer = getTypeStreamer(type.value("valueType").toString()); + if (valueStreamer == baseStreamer->getValueStreamer()) { + _typeStreamers.insert(name, baseStreamer->getSelf()); + + } else { + _typeStreamers.insert(name, TypeStreamerPointer(new MappedListTypeStreamer(baseStreamer, valueStreamer))); + } + } else if (category == "SET") { + TypeStreamerPointer valueStreamer = getTypeStreamer(type.value("valueType").toString()); + if (valueStreamer == baseStreamer->getValueStreamer()) { + _typeStreamers.insert(name, baseStreamer->getSelf()); + + } else { + _typeStreamers.insert(name, TypeStreamerPointer(new MappedSetTypeStreamer(baseStreamer, valueStreamer))); + } + } else if (category == "MAP") { + TypeStreamerPointer keyStreamer = getTypeStreamer(type.value("keyType").toString()); + TypeStreamerPointer valueStreamer = getTypeStreamer(type.value("valueType").toString()); + if (keyStreamer == baseStreamer->getKeyStreamer() && valueStreamer == baseStreamer->getValueStreamer()) { + _typeStreamers.insert(name, baseStreamer->getSelf()); + + } else { + _typeStreamers.insert(name, TypeStreamerPointer(new MappedMapTypeStreamer( + baseStreamer, keyStreamer, valueStreamer))); + } + } + } + + // create and map the object streamers in order + foreach (const QJsonValue& element, top.value("classes").toArray()) { + QJsonObject clazz = element.toObject(); + QString name = clazz.value("name").toString(); + QByteArray latinName = name.toLatin1(); + const ObjectStreamer* baseStreamer = Bitstream::getObjectStreamers().value( + Bitstream::getMetaObjects().value(latinName)); + if (!baseStreamer && genericsMode == Bitstream::NO_GENERICS) { + continue; // no built-in class and no generics allowed; we give up + } + if (!baseStreamer || genericsMode == Bitstream::ALL_GENERICS) { + // create a generic streamer + QVector properties; + foreach (const QJsonValue& property, clazz.value("properties").toArray()) { + QJsonObject object = property.toObject(); + properties.append(StreamerNamePair(getTypeStreamer(object.value("type").toString()), + object.value("name").toString().toLatin1())); + } + ObjectStreamerPointer streamer = ObjectStreamerPointer(new GenericObjectStreamer( + latinName, properties, QByteArray())); + _objectStreamers.insert(name, streamer); + static_cast(streamer.data())->_weakSelf = streamer; + continue; + } + // create a mapped streamer, determining along the way whether it matches our base + const QMetaObject* metaObject = baseStreamer->getMetaObject(); + const QVector& baseProperties = baseStreamer->getProperties(); + QVector properties; + QJsonArray propertyArray = clazz.value("properties").toArray(); + bool matches = (baseProperties.size() == propertyArray.size()); + for (int i = 0; i < propertyArray.size(); i++) { + QJsonObject object = propertyArray.at(i).toObject(); + TypeStreamerPointer typeStreamer = getTypeStreamer(object.value("type").toString()); + QMetaProperty metaProperty = metaObject->property(metaObject->indexOfProperty( + object.value("name").toString().toLatin1())); + properties.append(StreamerPropertyPair(typeStreamer, metaProperty)); + + const StreamerPropertyPair& baseProperty = baseProperties.at(i); + matches &= (typeStreamer == baseProperty.first && + metaProperty.propertyIndex() == baseProperty.second.propertyIndex()); + } + // if everything matches our built-in type, we can use that directly, which will be faster + if (matches) { + _objectStreamers.insert(name, baseStreamer->getSelf()); + } else { + _objectStreamers.insert(name, ObjectStreamerPointer(new MappedObjectStreamer(metaObject, properties))); + } + } + + // create and map the objects in order + foreach (const QJsonValue& element, top.value("objects").toArray()) { + QJsonObject object = element.toObject(); + int id = object.value("id").toInt(); + QObject* qObject; + putData(object.value("data"), qObject); + if (qObject) { + _sharedObjects.insert(id, static_cast(qObject)); + } + } + + // prepare the contents for extraction + _contents = top.value("contents").toArray(); + _contentsIterator = _contents.constBegin(); +} + +void JSONReader::putData(const QJsonValue& data, bool& value) { + value = data.toBool(); +} + +void JSONReader::putData(const QJsonValue& data, int& value) { + value = data.toInt(); +} + +void JSONReader::putData(const QJsonValue& data, uint& value) { + value = data.toInt(); +} + +void JSONReader::putData(const QJsonValue& data, float& value) { + value = data.toDouble(); +} + +void JSONReader::putData(const QJsonValue& data, QByteArray& value) { + value = QByteArray::fromPercentEncoding(data.toString().toLatin1()); +} + +void JSONReader::putData(const QJsonValue& data, QColor& value) { + value.setNamedColor(data.toString()); +} + +void JSONReader::putData(const QJsonValue& data, QScriptValue& value) { + QJsonObject object = data.toObject(); + QString type = object.value("type").toString(); + if (type == "UNDEFINED") { + value = QScriptValue(QScriptValue::UndefinedValue); + + } else if (type == "NULL") { + value = QScriptValue(QScriptValue::NullValue); + + } else if (type == "BOOL") { + value = QScriptValue(object.value("value").toBool()); + + } else if (type == "NUMBER") { + value = QScriptValue(object.value("value").toDouble()); + + } else if (type == "STRING") { + value = QScriptValue(object.value("value").toString()); + + } else if (type == "VARIANT") { + QVariant variant; + putData(object.value("value"), variant); + value = ScriptCache::getInstance()->getEngine()->newVariant(variant); + + } else if (type == "QOBJECT") { + QObject* qObject; + putData(object.value("value"), qObject); + value = ScriptCache::getInstance()->getEngine()->newQObject(qObject, QScriptEngine::ScriptOwnership); + + } else if (type == "QMETAOBJECT") { + const QMetaObject* metaObject; + putData(object.value("value"), metaObject); + value = ScriptCache::getInstance()->getEngine()->newQMetaObject(metaObject); + + } else if (type == "DATE") { + QDateTime dateTime; + putData(object.value("value"), dateTime); + value = ScriptCache::getInstance()->getEngine()->newDate(dateTime); + + } else if (type == "REGEXP") { + QRegExp regExp; + putData(object.value("value"), regExp); + value = ScriptCache::getInstance()->getEngine()->newRegExp(regExp); + + } else if (type == "ARRAY") { + QJsonArray array = object.value("value").toArray(); + value = ScriptCache::getInstance()->getEngine()->newArray(array.size()); + for (int i = 0; i < array.size(); i++) { + QScriptValue element; + putData(array.at(i), element); + value.setProperty(i, element); + } + } else if (type == "OBJECT") { + QJsonObject jsonObject = object.value("value").toObject(); + value = ScriptCache::getInstance()->getEngine()->newObject(); + for (QJsonObject::const_iterator it = jsonObject.constBegin(); it != jsonObject.constEnd(); it++) { + QScriptValue element; + putData(it.value(), element); + value.setProperty(it.key(), element); + } + } else { + value = QScriptValue(); + } +} + +void JSONReader::putData(const QJsonValue& data, QString& value) { + value = data.toString(); +} + +void JSONReader::putData(const QJsonValue& data, QUrl& value) { + value = data.toString(); +} + +void JSONReader::putData(const QJsonValue& data, QDateTime& value) { + value.setMSecsSinceEpoch((qint64)data.toDouble()); +} + +void JSONReader::putData(const QJsonValue& data, QRegExp& value) { + QJsonObject object = data.toObject(); + value = QRegExp(object.value("pattern").toString(), (Qt::CaseSensitivity)object.value("caseSensitivity").toInt(), + (QRegExp::PatternSyntax)object.value("patternSyntax").toInt()); + value.setMinimal(object.value("minimal").toBool()); +} + +void JSONReader::putData(const QJsonValue& data, glm::vec3& value) { + QJsonArray array = data.toArray(); + value = glm::vec3(array.at(0).toDouble(), array.at(1).toDouble(), array.at(2).toDouble()); +} + +void JSONReader::putData(const QJsonValue& data, glm::quat& value) { + QJsonArray array = data.toArray(); + value = glm::quat(array.at(0).toDouble(), array.at(1).toDouble(), array.at(2).toDouble(), array.at(3).toDouble()); +} + +void JSONReader::putData(const QJsonValue& data, const QMetaObject*& value) { + ObjectStreamerPointer streamer = _objectStreamers.value(data.toString()); + value = streamer ? streamer->getMetaObject() : NULL; +} + +void JSONReader::putData(const QJsonValue& data, QVariant& value) { + QJsonObject object = data.toObject(); + QString type = object.value("type").toString(); + TypeStreamerPointer streamer = getTypeStreamer(type); + if (streamer) { + streamer->putJSONVariantData(*this, object.value("value"), value); + } else { + value = QVariant(); + } +} + +void JSONReader::putData(const QJsonValue& data, SharedObjectPointer& value) { + value = _sharedObjects.value(data.toInt()); +} + +void JSONReader::putData(const QJsonValue& data, QObject*& value) { + QJsonObject object = data.toObject(); + ObjectStreamerPointer streamer = _objectStreamers.value(object.value("class").toString()); + value = streamer ? streamer->putJSONData(*this, object) : NULL; +} + +TypeStreamerPointer JSONReader::getTypeStreamer(const QString& name) const { + TypeStreamerPointer streamer = _typeStreamers.value(name); + if (!streamer) { + const TypeStreamer* defaultStreamer = Bitstream::getTypeStreamers().value(QMetaType::type(name.toLatin1())); + if (defaultStreamer) { + streamer = defaultStreamer->getSelf(); + } else { + qWarning() << "Unknown type:" << name; + } + } + return streamer; +} + ObjectStreamer::ObjectStreamer(const QMetaObject* metaObject) : _metaObject(metaObject) { } @@ -1702,6 +2272,50 @@ void MappedObjectStreamer::writeMetadata(Bitstream& out, bool full) const { } } +QJsonObject MappedObjectStreamer::getJSONMetadata(JSONWriter& writer) const { + QJsonObject metadata; + metadata.insert("name", QString(_metaObject->className())); + QJsonArray properties; + foreach (const StreamerPropertyPair& property, _properties) { + QJsonObject object; + writer.addTypeStreamer(property.first.data()); + object.insert("type", QString(property.first->getName())); + object.insert("name", QString(property.second.name())); + properties.append(object); + } + metadata.insert("properties", properties); + return metadata; +} + +QJsonObject MappedObjectStreamer::getJSONData(JSONWriter& writer, const QObject* object) const { + QJsonObject data; + writer.addObjectStreamer(this); + data.insert("class", QString(_metaObject->className())); + QJsonArray properties; + foreach (const StreamerPropertyPair& property, _properties) { + properties.append(property.first->getJSONData(writer, property.second.read(object))); + } + data.insert("properties", properties); + return data; +} + +QObject* MappedObjectStreamer::putJSONData(JSONReader& reader, const QJsonObject& jsonObject) const { + if (!_metaObject) { + return NULL; + } + QObject* object = _metaObject->newInstance(); + QJsonArray properties = jsonObject.value("properties").toArray(); + for (int i = 0; i < _properties.size(); i++) { + const StreamerPropertyPair& property = _properties.at(i); + if (property.second.isValid()) { + QVariant value; + property.first->putJSONData(reader, properties.at(i), value); + property.second.write(object, value); + } + } + return object; +} + void MappedObjectStreamer::write(Bitstream& out, const QObject* object) const { foreach (const StreamerPropertyPair& property, _properties) { property.first->write(out, property.second.read(object)); @@ -1778,6 +2392,47 @@ void GenericObjectStreamer::writeMetadata(Bitstream& out, bool full) const { } } +QJsonObject GenericObjectStreamer::getJSONMetadata(JSONWriter& writer) const { + QJsonObject metadata; + metadata.insert("name", QString(_name)); + QJsonArray properties; + foreach (const StreamerNamePair& property, _properties) { + QJsonObject object; + writer.addTypeStreamer(property.first.data()); + object.insert("type", QString(property.first->getName())); + object.insert("name", QString(property.second)); + properties.append(object); + } + metadata.insert("properties", properties); + return metadata; +} + +QJsonObject GenericObjectStreamer::getJSONData(JSONWriter& writer, const QObject* object) const { + QJsonObject data; + writer.addObjectStreamer(this); + data.insert("class", QString(_name)); + QJsonArray properties; + const QVariantList& values = static_cast(object)->getValues(); + for (int i = 0; i < _properties.size(); i++) { + properties.append(_properties.at(i).first->getJSONData(writer, values.at(i))); + } + data.insert("properties", properties); + return data; +} + +QObject* GenericObjectStreamer::putJSONData(JSONReader& reader, const QJsonObject& jsonObject) const { + GenericSharedObject* object = new GenericSharedObject(_weakSelf); + QJsonArray properties = jsonObject.value("properties").toArray(); + QVariantList values; + for (int i = 0; i < _properties.size(); i++) { + QVariant value; + _properties.at(i).first->putJSONData(reader, properties.at(i), value); + values.append(value); + } + object->setValues(values); + return object; +} + void GenericObjectStreamer::write(Bitstream& out, const QObject* object) const { const QVariantList& values = static_cast(object)->getValues(); for (int i = 0; i < _properties.size(); i++) { @@ -1879,6 +2534,71 @@ void TypeStreamer::writeMetadata(Bitstream& out, bool full) const { } } +QJsonValue TypeStreamer::getJSONMetadata(JSONWriter& writer) const { + Category category = getCategory(); + switch (category) { + case STREAMABLE_CATEGORY: { + QJsonObject metadata; + metadata.insert("name", QString(getName())); + metadata.insert("category", QString("STREAMABLE")); + QJsonArray fields; + foreach (const MetaField& metaField, getMetaFields()) { + QJsonObject field; + writer.addTypeStreamer(metaField.getStreamer()); + field.insert("type", QString(metaField.getStreamer()->getName())); + field.insert("name", QString(metaField.getName())); + fields.append(field); + } + metadata.insert("fields", fields); + return metadata; + } + case LIST_CATEGORY: + case SET_CATEGORY: { + QJsonObject metadata; + metadata.insert("name", QString(getName())); + metadata.insert("category", QString(category == LIST_CATEGORY ? "LIST" : "SET")); + const TypeStreamer* valueStreamer = getValueStreamer(); + writer.addTypeStreamer(valueStreamer); + metadata.insert("valueType", QString(valueStreamer->getName())); + return metadata; + } + case MAP_CATEGORY: { + QJsonObject metadata; + metadata.insert("name", QString(getName())); + metadata.insert("category", QString("MAP")); + const TypeStreamer* keyStreamer = getKeyStreamer(); + writer.addTypeStreamer(keyStreamer); + metadata.insert("keyType", QString(keyStreamer->getName())); + const TypeStreamer* valueStreamer = getValueStreamer(); + writer.addTypeStreamer(valueStreamer); + metadata.insert("valueType", QString(valueStreamer->getName())); + return metadata; + } + default: + return QJsonValue(); + } +} + +QJsonValue TypeStreamer::getJSONData(JSONWriter& writer, const QVariant& value) const { + return QJsonValue(); +} + +QJsonValue TypeStreamer::getJSONVariantData(JSONWriter& writer, const QVariant& value) const { + writer.addTypeStreamer(this); + QJsonObject data; + data.insert("type", QString(getName())); + data.insert("value", getJSONData(writer, value)); + return data; +} + +void TypeStreamer::putJSONData(JSONReader& reader, const QJsonValue& data, QVariant& value) const { + value = QVariant(); +} + +void TypeStreamer::putJSONVariantData(JSONReader& reader, const QJsonValue& data, QVariant& value) const { + putJSONData(reader, data, value); +} + bool TypeStreamer::equal(const QVariant& first, const QVariant& second) const { return first == second; } @@ -2007,7 +2727,7 @@ QDebug& operator<<(QDebug& debug, const QMetaObject* metaObject) { EnumTypeStreamer::EnumTypeStreamer(const QMetaObject* metaObject, const char* name) : _metaObject(metaObject), _enumName(name), - _name(QByteArray(metaObject->className()) + "::" + name), + _name(getEnumName(metaObject->className(), name)), _bits(-1) { _type = QMetaType::Int; @@ -2015,7 +2735,7 @@ EnumTypeStreamer::EnumTypeStreamer(const QMetaObject* metaObject, const char* na } EnumTypeStreamer::EnumTypeStreamer(const QMetaEnum& metaEnum) : - _name(QByteArray(metaEnum.scope()) + "::" + metaEnum.name()), + _name(getEnumName(metaEnum.scope(), metaEnum.name())), _metaEnum(metaEnum), _bits(-1) { @@ -2048,6 +2768,30 @@ void EnumTypeStreamer::writeMetadata(Bitstream& out, bool full) const { } } +QJsonValue EnumTypeStreamer::getJSONMetadata(JSONWriter& writer) const { + QJsonObject metadata; + metadata.insert("name", QString(getName())); + metadata.insert("category", QString("ENUM")); + QJsonArray values; + QMetaEnum metaEnum = getMetaEnum(); + for (int i = 0; i < metaEnum.keyCount(); i++) { + QJsonObject value; + value.insert("key", QString(metaEnum.key(i))); + value.insert("value", metaEnum.value(i)); + values.append(value); + } + metadata.insert("values", values); + return metadata; +} + +QJsonValue EnumTypeStreamer::getJSONData(JSONWriter& writer, const QVariant& value) const { + return value.toInt(); +} + +void EnumTypeStreamer::putJSONData(JSONReader& reader, const QJsonValue& data, QVariant& value) const { + value = data.toInt(); +} + TypeStreamer::Category EnumTypeStreamer::getCategory() const { return ENUM_CATEGORY; } @@ -2140,6 +2884,11 @@ MappedEnumTypeStreamer::MappedEnumTypeStreamer(const TypeStreamer* baseStreamer, _mappings(mappings) { } +void MappedEnumTypeStreamer::putJSONData(JSONReader& reader, const QJsonValue& data, QVariant& value) const { + value = QVariant(_baseStreamer->getType(), 0); + _baseStreamer->setEnumValue(value, data.toInt(), _mappings); +} + QVariant MappedEnumTypeStreamer::read(Bitstream& in) const { QVariant object = QVariant(_baseStreamer->getType(), 0); int value = 0; @@ -2162,6 +2911,12 @@ const char* GenericTypeStreamer::getName() const { return _name.constData(); } +void GenericTypeStreamer::putJSONVariantData(JSONReader& reader, const QJsonValue& data, QVariant& value) const { + QVariant containedValue; + putJSONData(reader, data, containedValue); + value = QVariant::fromValue(GenericValue(_weakSelf, containedValue)); +} + QVariant GenericTypeStreamer::readVariant(Bitstream& in) const { return QVariant::fromValue(GenericValue(_weakSelf, read(in))); } @@ -2197,6 +2952,29 @@ void GenericEnumTypeStreamer::writeMetadata(Bitstream& out, bool full) const { } } +QJsonValue GenericEnumTypeStreamer::getJSONMetadata(JSONWriter& writer) const { + QJsonObject metadata; + metadata.insert("name", QString(getName())); + metadata.insert("category", QString("ENUM")); + QJsonArray values; + foreach (const NameIntPair& value, _values) { + QJsonObject object; + object.insert("key", QString(value.first)); + object.insert("value", value.second); + values.append(object); + } + metadata.insert("values", values); + return metadata; +} + +QJsonValue GenericEnumTypeStreamer::getJSONData(JSONWriter& writer, const QVariant& value) const { + return value.toInt(); +} + +void GenericEnumTypeStreamer::putJSONData(JSONReader& reader, const QJsonValue& data, QVariant& value) const { + value = data.toInt(); +} + void GenericEnumTypeStreamer::write(Bitstream& out, const QVariant& value) const { int intValue = value.toInt(); out.write(&intValue, _bits); @@ -2218,6 +2996,19 @@ MappedStreamableTypeStreamer::MappedStreamableTypeStreamer(const TypeStreamer* b _fields(fields) { } +void MappedStreamableTypeStreamer::putJSONData(JSONReader& reader, const QJsonValue& data, QVariant& value) const { + value = QVariant(_baseStreamer->getType(), 0); + QJsonArray array = data.toArray(); + for (int i = 0; i < _fields.size(); i++) { + const StreamerIndexPair& pair = _fields.at(i); + if (pair.second != -1) { + QVariant element; + pair.first->putJSONData(reader, array.at(i), element); + _baseStreamer->setField(value, pair.second, element); + } + } +} + QVariant MappedStreamableTypeStreamer::read(Bitstream& in) const { QVariant object = QVariant(_baseStreamer->getType(), 0); foreach (const StreamerIndexPair& pair, _fields) { @@ -2273,6 +3064,42 @@ void GenericStreamableTypeStreamer::writeMetadata(Bitstream& out, bool full) con } } +QJsonValue GenericStreamableTypeStreamer::getJSONMetadata(JSONWriter& writer) const { + QJsonObject metadata; + metadata.insert("name", QString(getName())); + metadata.insert("category", QString("STREAMABLE")); + QJsonArray fields; + foreach (const StreamerNamePair& field, _fields) { + QJsonObject object; + writer.addTypeStreamer(field.first.data()); + object.insert("type", QString(field.first->getName())); + object.insert("name", QString(field.second)); + fields.append(object); + } + metadata.insert("fields", fields); + return metadata; +} + +QJsonValue GenericStreamableTypeStreamer::getJSONData(JSONWriter& writer, const QVariant& value) const { + QVariantList values = value.toList(); + QJsonArray array; + for (int i = 0; i < _fields.size(); i++) { + array.append(_fields.at(i).first->getJSONData(writer, values.at(i))); + } + return array; +} + +void GenericStreamableTypeStreamer::putJSONData(JSONReader& reader, const QJsonValue& data, QVariant& value) const { + QJsonArray array = data.toArray(); + QVariantList values; + for (int i = 0; i < _fields.size(); i++) { + QVariant element; + _fields.at(i).first->putJSONData(reader, array.at(i), element); + values.append(element); + } + value = values; +} + void GenericStreamableTypeStreamer::write(Bitstream& out, const QVariant& value) const { QVariantList values = value.toList(); for (int i = 0; i < _fields.size(); i++) { @@ -2297,6 +3124,15 @@ MappedListTypeStreamer::MappedListTypeStreamer(const TypeStreamer* baseStreamer, _valueStreamer(valueStreamer) { } +void MappedListTypeStreamer::putJSONData(JSONReader& reader, const QJsonValue& data, QVariant& value) const { + value = QVariant(_baseStreamer->getType(), 0); + foreach (const QJsonValue& element, data.toArray()) { + QVariant elementValue; + _valueStreamer->putJSONData(reader, element, elementValue); + _baseStreamer->insert(value, elementValue); + } +} + QVariant MappedListTypeStreamer::read(Bitstream& in) const { QVariant object = QVariant(_baseStreamer->getType(), 0); int size; @@ -2337,6 +3173,35 @@ void GenericListTypeStreamer::writeMetadata(Bitstream& out, bool full) const { out << _valueStreamer.data(); } +QJsonValue GenericListTypeStreamer::getJSONMetadata(JSONWriter& writer) const { + QJsonObject metadata; + metadata.insert("name", QString(getName())); + metadata.insert("category", QString("LIST")); + writer.addTypeStreamer(_valueStreamer.data()); + metadata.insert("valueType", QString(_valueStreamer->getName())); + return metadata; +} + +QJsonValue GenericListTypeStreamer::getJSONData(JSONWriter& writer, const QVariant& value) const { + QVariantList values = value.toList(); + QJsonArray array; + foreach (const QVariant& element, values) { + array.append(_valueStreamer->getJSONData(writer, element)); + } + return array; +} + +void GenericListTypeStreamer::putJSONData(JSONReader& reader, const QJsonValue& data, QVariant& value) const { + QJsonArray array = data.toArray(); + QVariantList values; + foreach (const QJsonValue& element, array) { + QVariant elementValue; + _valueStreamer->putJSONData(reader, element, elementValue); + values.append(elementValue); + } + value = values; +} + void GenericListTypeStreamer::write(Bitstream& out, const QVariant& value) const { QVariantList values = value.toList(); out << values.size(); @@ -2379,6 +3244,15 @@ GenericSetTypeStreamer::GenericSetTypeStreamer(const QByteArray& name, const Typ GenericListTypeStreamer(name, valueStreamer) { } +QJsonValue GenericSetTypeStreamer::getJSONMetadata(JSONWriter& writer) const { + QJsonObject metadata; + metadata.insert("name", QString(getName())); + metadata.insert("category", QString("SET")); + writer.addTypeStreamer(_valueStreamer.data()); + metadata.insert("valueType", QString(_valueStreamer->getName())); + return metadata; +} + TypeStreamer::Category GenericSetTypeStreamer::getCategory() const { return SET_CATEGORY; } @@ -2390,6 +3264,18 @@ MappedMapTypeStreamer::MappedMapTypeStreamer(const TypeStreamer* baseStreamer, c _valueStreamer(valueStreamer) { } +void MappedMapTypeStreamer::putJSONData(JSONReader& reader, const QJsonValue& data, QVariant& value) const { + value = QVariant(_baseStreamer->getType(), 0); + foreach (const QJsonValue& element, data.toArray()) { + QJsonArray pair = element.toArray(); + QVariant elementKey; + _keyStreamer->putJSONData(reader, pair.at(0), elementKey); + QVariant elementValue; + _valueStreamer->putJSONData(reader, pair.at(1), elementValue); + _baseStreamer->insert(value, elementKey, elementValue); + } +} + QVariant MappedMapTypeStreamer::read(Bitstream& in) const { QVariant object = QVariant(_baseStreamer->getType(), 0); int size; @@ -2440,6 +3326,43 @@ void GenericMapTypeStreamer::writeMetadata(Bitstream& out, bool full) const { out << _keyStreamer.data() << _valueStreamer.data(); } +QJsonValue GenericMapTypeStreamer::getJSONMetadata(JSONWriter& writer) const { + QJsonObject metadata; + metadata.insert("name", QString(getName())); + metadata.insert("category", QString("MAP")); + writer.addTypeStreamer(_keyStreamer.data()); + metadata.insert("keyType", QString(_keyStreamer->getName())); + writer.addTypeStreamer(_valueStreamer.data()); + metadata.insert("valueType", QString(_valueStreamer->getName())); + return metadata; +} + +QJsonValue GenericMapTypeStreamer::getJSONData(JSONWriter& writer, const QVariant& value) const { + QVariantPairList values = value.value(); + QJsonArray array; + foreach (const QVariantPair& pair, values) { + QJsonArray pairArray; + pairArray.append(_keyStreamer->getJSONData(writer, pair.first)); + pairArray.append(_valueStreamer->getJSONData(writer, pair.second)); + array.append(pairArray); + } + return array; +} + +void GenericMapTypeStreamer::putJSONData(JSONReader& reader, const QJsonValue& data, QVariant& value) const { + QJsonArray array = data.toArray(); + QVariantPairList values; + foreach (const QJsonValue& element, array) { + QJsonArray pair = element.toArray(); + QVariant elementKey; + _keyStreamer->putJSONData(reader, pair.at(0), elementKey); + QVariant elementValue; + _valueStreamer->putJSONData(reader, pair.at(1), elementValue); + values.append(QVariantPair(elementKey, elementValue)); + } + value = QVariant::fromValue(values); +} + void GenericMapTypeStreamer::write(Bitstream& out, const QVariant& value) const { QVariantPairList values = value.value(); out << values.size(); @@ -2465,6 +3388,15 @@ TypeStreamer::Category GenericMapTypeStreamer::getCategory() const { return MAP_CATEGORY; } +QJsonValue GenericValueStreamer::getJSONVariantData(JSONWriter& writer, const QVariant& value) const { + GenericValue genericValue = value.value(); + writer.addTypeStreamer(genericValue.getStreamer().data()); + QJsonObject data; + data.insert("type", QString(genericValue.getStreamer()->getName())); + data.insert("value", genericValue.getStreamer()->getJSONData(writer, genericValue.getValue())); + return data; +} + void GenericValueStreamer::writeVariant(Bitstream& out, const QVariant& value) const { GenericValue genericValue = value.value(); out << genericValue.getStreamer().data(); diff --git a/libraries/metavoxels/src/Bitstream.h b/libraries/metavoxels/src/Bitstream.h index 5dc2fcc35d..0d9e516640 100644 --- a/libraries/metavoxels/src/Bitstream.h +++ b/libraries/metavoxels/src/Bitstream.h @@ -13,6 +13,9 @@ #define hifi_Bitstream_h #include +#include +#include +#include #include #include #include @@ -36,6 +39,7 @@ class Attribute; class AttributeValue; class Bitstream; class GenericValue; +class JSONWriter; class ObjectReader; class ObjectStreamer; class OwnedAttributeValue; @@ -195,12 +199,50 @@ template inline RepeatedValueStreamer& return *this; } -/// A stream for bit-aligned data. +/// A stream for bit-aligned data. Through a combination of code generation, reflection, macros, and templates, provides a +/// serialization mechanism that may be used for both networking and persistent storage. For unreliable networking, the +/// class provides a mapping system that resends mappings for ids until they are acknowledged (and thus persisted). For +/// version-resilient persistence, the class provides a metadata system that maps stored types to local types (or to +/// generic containers), allowing one to add and remove fields to classes without breaking compatibility with previously +/// stored data. +/// +/// The basic usage requires one to create a Bitstream that wraps an underlying QDataStream, specifying the metadata type +/// desired and (for readers) the generics mode. Then, one uses the << or >> operators to write or read values to/from +/// the stream (a stream instance may be used for reading or writing, but not both). For write streams, the flush +/// function should be called on completion to write any partial data. +/// +/// Polymorphic types are supported via the QVariant and QObject*/SharedObjectPointer types. When you write a QVariant or +/// QObject, the type or class name (at minimum) is written to the stream. When you read a QVariant or QObject, the default +/// behavior is to look for a corresponding registered type with the written name. With hash metadata, Bitstream can verify +/// that the local type matches the stream type, throwing the streamed version away if not. With full metadata, Bitstream can +/// create a mapping from the streamed type to the local type that applies the intersection of the two types' fields. +/// +/// To register types for streaming, select from the provided templates and macros. To register a QObject (or SharedObject) +/// subclass, use REGISTER_META_OBJECT in the class's source file. To register a streamable class for use in QVariant, use +/// the STREAMABLE/STREAM/DECLARE_STREAMABLE_METATYPE macros and use the mtc tool to generate the associated implementation +/// code. To register a QObject enum for use outside the QObject's direct properties, use the +/// DECLARE_ENUM_METATYPE/IMPLEMENT_ENUM_METATYPE macro pair. To register a collection type (QList, QVector, QSet, QMap) of +/// streamable types, use the registerCollectionMetaType template function. +/// +/// Delta-streaming is supported through the writeDelta/readDelta functions (analogous to <>), which accept a reference +/// value and stream only the information necessary to turn the reference into the target value. This assumes that the +/// reference value provided when reading is identical to the one used when writing. +/// +/// Special delta handling is provided for objects tracked by SharedObjectPointers. SharedObjects have IDs (representing their +/// unique identity on one system) as well as origin IDs (representing their identity as preserved over the course of +/// mutation). Once a shared object with a given ID is written (and its mapping persisted), that object's state will not be +/// written again; only its ID will be sent. However, if an object with a new ID but the same origin ID is written, that +/// object's state will be encoded as a delta between the previous object and the new one. So, to transmit delta-encoded +/// objects, one should treat each local SharedObject instance as immutable, replacing it when mutated with a cloned instance +/// generated by calling SharedObject::clone(true) and applying the desired changes. class Bitstream : public QObject { Q_OBJECT public: + /// Stores a set of mappings from values to ids written. Typically, one would store these mappings along with the send + /// record of the packet that contained them, persisting the mappings if/when the packet is acknowledged by the remote + /// party. class WriteMappings { public: QHash objectStreamerOffsets; @@ -210,6 +252,9 @@ public: QHash sharedObjectOffsets; }; + /// Stores a set of mappings from ids to values read. Typically, one would store these mappings along with the receive + /// record of the packet that contained them, persisting the mappings if/when the remote party indicates that it + /// has received the local party's acknowledgement of the packet. class ReadMappings { public: QHash objectStreamerValues; @@ -219,11 +264,14 @@ public: QHash sharedObjectValues; }; - /// Registers a metaobject under its name so that instances of it can be streamed. + /// Registers a metaobject under its name so that instances of it can be streamed. Consider using the REGISTER_META_OBJECT + /// at the top level of the source file associated with the class rather than calling this function directly. /// \return zero; the function only returns a value so that it can be used in static initialization static int registerMetaObject(const char* className, const QMetaObject* metaObject); - /// Registers a streamer for the specified Qt-registered type. + /// Registers a streamer for the specified Qt-registered type. Consider using one of the registration macros (such as + /// REGISTER_SIMPLE_TYPE_STREAMER) at the top level of the associated source file rather than calling this function + /// directly. /// \return zero; the function only returns a value so that it can be used in static initialization static int registerTypeStreamer(int type, TypeStreamer* streamer); @@ -233,7 +281,9 @@ public: /// Returns the meta-object registered under the supplied class name, if any. static const QMetaObject* getMetaObject(const QByteArray& className); - /// Returns the list of registered subclasses for the supplied meta-object. + /// Returns the list of registered subclasses for the supplied meta-object. When registered, metaobjects register + /// themselves as subclasses of all of their parents, mostly in order to allow editors to provide lists of available + /// subclasses. static QList getMetaObjectSubClasses(const QMetaObject* metaObject); enum MetadataType { NO_METADATA, HASH_METADATA, FULL_METADATA }; @@ -250,7 +300,8 @@ public: Bitstream(QDataStream& underlying, MetadataType metadataType = NO_METADATA, GenericsMode = NO_GENERICS, QObject* parent = NULL); - /// Substitutes the supplied metaobject for the given class name's default mapping. + /// Substitutes the supplied metaobject for the given class name's default mapping. This is mostly useful for testing the + /// process of mapping between different types, but may in the future be used for permanently renaming classes. void addMetaObjectSubstitution(const QByteArray& className, const QMetaObject* metaObject); /// Substitutes the supplied type for the given type name's default mapping. @@ -445,6 +496,9 @@ private slots: private: + friend class JSONReader; + friend class JSONWriter; + ObjectStreamerPointer readGenericObjectStreamer(const QByteArray& name); TypeStreamerPointer readGenericTypeStreamer(const QByteArray& name, int category); @@ -471,7 +525,7 @@ private: static QHash& getMetaObjects(); static QMultiHash& getMetaObjectSubClasses(); static QHash& getTypeStreamers(); - + static const QHash& getObjectStreamers(); static QHash createObjectStreamers(); @@ -766,6 +820,189 @@ template inline Bitstream& Bitstream::operator>>(QHash& return *this; } +/// Provides a means of writing Bitstream-able data to JSON rather than the usual binary format in a manner that allows it to +/// be manipulated and re-read, converted to binary, etc. To use, create a JSONWriter, stream values in using the << operator, +/// and call getDocument to obtain the JSON data. +class JSONWriter { +public: + + QJsonValue getData(bool value); + QJsonValue getData(int value); + QJsonValue getData(uint value); + QJsonValue getData(float value); + QJsonValue getData(const QByteArray& value); + QJsonValue getData(const QColor& value); + QJsonValue getData(const QScriptValue& value); + QJsonValue getData(const QString& value); + QJsonValue getData(const QUrl& value); + QJsonValue getData(const QDateTime& value); + QJsonValue getData(const QRegExp& value); + QJsonValue getData(const glm::vec3& value); + QJsonValue getData(const glm::quat& value); + QJsonValue getData(const QMetaObject* value); + QJsonValue getData(const QVariant& value); + QJsonValue getData(const SharedObjectPointer& value); + QJsonValue getData(const QObject* value); + QJsonValue getData(const GenericValue& value); + + template QJsonValue getData(const T& value) { return QJsonValue(); } + + template QJsonValue getData(const QList& list); + template QJsonValue getData(const QVector& list); + template QJsonValue getData(const QSet& set); + template QJsonValue getData(const QHash& hash); + + template JSONWriter& operator<<(const T& value) { _contents.append(getData(value)); return *this; } + + void appendToContents(const QJsonValue& value) { _contents.append(value); } + + void addSharedObject(const SharedObjectPointer& object); + void addObjectStreamer(const ObjectStreamer* streamer); + void addTypeStreamer(const TypeStreamer* streamer); + + QJsonDocument getDocument() const; + +private: + + QJsonArray _contents; + + QSet _sharedObjectIDs; + QJsonArray _sharedObjects; + + QSet _objectStreamerNames; + QJsonArray _objectStreamers; + + QSet _typeStreamerNames; + QJsonArray _typeStreamers; +}; + +template inline QJsonValue JSONWriter::getData(const QList& list) { + QJsonArray array; + foreach (const T& value, list) { + array.append(getData(value)); + } + return array; +} + +template inline QJsonValue JSONWriter::getData(const QVector& vector) { + QJsonArray array; + foreach (const T& value, vector) { + array.append(getData(value)); + } + return array; +} + +template inline QJsonValue JSONWriter::getData(const QSet& set) { + QJsonArray array; + foreach (const T& value, set) { + array.append(getData(value)); + } + return array; +} + +template inline QJsonValue JSONWriter::getData(const QHash& hash) { + QJsonArray array; + for (typename QHash::const_iterator it = hash.constBegin(); it != hash.constEnd(); it++) { + QJsonArray pair; + pair.append(getData(it.key())); + pair.append(getData(it.value())); + array.append(pair); + } + return array; +} + +/// Reads a document written by JSONWriter. To use, create a JSONReader and stream values out using the >> operator. +class JSONReader { +public: + + /// Creates a reader to read from the supplied document. + /// \param genericsMode the generics mode to use: NO_GENERICS to map all types in the document to built-in types, + /// FALLBACK_GENERICS to use generic containers where no matching built-in type is found, or ALL_GENERICS to + /// read all types to generic containers + JSONReader(const QJsonDocument& document, Bitstream::GenericsMode genericsMode = Bitstream::NO_GENERICS); + + void putData(const QJsonValue& data, bool& value); + void putData(const QJsonValue& data, int& value); + void putData(const QJsonValue& data, uint& value); + void putData(const QJsonValue& data, float& value); + void putData(const QJsonValue& data, QByteArray& value); + void putData(const QJsonValue& data, QColor& value); + void putData(const QJsonValue& data, QScriptValue& value); + void putData(const QJsonValue& data, QString& value); + void putData(const QJsonValue& data, QUrl& value); + void putData(const QJsonValue& data, QDateTime& value); + void putData(const QJsonValue& data, QRegExp& value); + void putData(const QJsonValue& data, glm::vec3& value); + void putData(const QJsonValue& data, glm::quat& value); + void putData(const QJsonValue& data, const QMetaObject*& value); + void putData(const QJsonValue& data, QVariant& value); + void putData(const QJsonValue& data, SharedObjectPointer& value); + void putData(const QJsonValue& data, QObject*& value); + + template void putData(const QJsonValue& data, T& value) { value = T(); } + + template void putData(const QJsonValue& data, QList& list); + template void putData(const QJsonValue& data, QVector& list); + template void putData(const QJsonValue& data, QSet& set); + template void putData(const QJsonValue& data, QHash& hash); + + template JSONReader& operator>>(T& value) { putData(*_contentsIterator++, value); return *this; } + + QJsonValue retrieveNextFromContents() { return *_contentsIterator++; } + + TypeStreamerPointer getTypeStreamer(const QString& name) const; + ObjectStreamerPointer getObjectStreamer(const QString& name) const { return _objectStreamers.value(name); } + SharedObjectPointer getSharedObject(int id) const { return _sharedObjects.value(id); } + +private: + + QJsonArray _contents; + QJsonArray::const_iterator _contentsIterator; + + QHash _typeStreamers; + QHash _objectStreamers; + QHash _sharedObjects; +}; + +template inline void JSONReader::putData(const QJsonValue& data, QList& list) { + list.clear(); + foreach (const QJsonValue& element, data.toArray()) { + T value; + putData(element, value); + list.append(value); + } +} + +template inline void JSONReader::putData(const QJsonValue& data, QVector& list) { + list.clear(); + foreach (const QJsonValue& element, data.toArray()) { + T value; + putData(element, value); + list.append(value); + } +} + +template inline void JSONReader::putData(const QJsonValue& data, QSet& set) { + set.clear(); + foreach (const QJsonValue& element, data.toArray()) { + T value; + putData(element, value); + set.insert(value); + } +} + +template inline void JSONReader::putData(const QJsonValue& data, QHash& hash) { + hash.clear(); + foreach (const QJsonValue& element, data.toArray()) { + QJsonArray pair = element.toArray(); + K key; + putData(pair.at(0), key); + V value; + putData(pair.at(1), value); + hash.insert(key, value); + } +} + typedef QPair StreamerPropertyPair; /// Contains the information required to stream an object. @@ -781,6 +1018,10 @@ public: virtual const char* getName() const = 0; virtual const QVector& getProperties() const; virtual void writeMetadata(Bitstream& out, bool full) const = 0; + virtual QJsonObject getJSONMetadata(JSONWriter& writer) const = 0; + virtual QJsonObject getJSONData(JSONWriter& writer, const QObject* object) const = 0; + virtual QObject* putJSONData(JSONReader& reader, const QJsonObject& jsonObject) const = 0; + virtual void write(Bitstream& out, const QObject* object) const = 0; virtual void writeRawDelta(Bitstream& out, const QObject* object, const QObject* reference) const = 0; virtual QObject* read(Bitstream& in, QObject* object = NULL) const = 0; @@ -791,7 +1032,7 @@ protected: friend class Bitstream; const QMetaObject* _metaObject; - ObjectStreamerPointer _self; + ObjectStreamerPointer _self; ///< set/used for built-in classes (never deleted), to obtain shared pointers }; /// A streamer that maps to a local class. @@ -803,6 +1044,9 @@ public: virtual const char* getName() const; virtual const QVector& getProperties() const; virtual void writeMetadata(Bitstream& out, bool full) const; + virtual QJsonObject getJSONMetadata(JSONWriter& writer) const; + virtual QJsonObject getJSONData(JSONWriter& writer, const QObject* object) const; + virtual QObject* putJSONData(JSONReader& reader, const QJsonObject& jsonObject) const; virtual void write(Bitstream& out, const QObject* object) const; virtual void writeRawDelta(Bitstream& out, const QObject* object, const QObject* reference) const; virtual QObject* read(Bitstream& in, QObject* object = NULL) const; @@ -823,6 +1067,9 @@ public: virtual const char* getName() const; virtual void writeMetadata(Bitstream& out, bool full) const; + virtual QJsonObject getJSONMetadata(JSONWriter& writer) const; + virtual QJsonObject getJSONData(JSONWriter& writer, const QObject* object) const; + virtual QObject* putJSONData(JSONReader& reader, const QJsonObject& jsonObject) const; virtual void write(Bitstream& out, const QObject* object) const; virtual void writeRawDelta(Bitstream& out, const QObject* object, const QObject* reference) const; virtual QObject* read(Bitstream& in, QObject* object = NULL) const; @@ -831,9 +1078,10 @@ public: private: friend class Bitstream; + friend class JSONReader; QByteArray _name; - WeakObjectStreamerPointer _weakSelf; + WeakObjectStreamerPointer _weakSelf; ///< promoted to strong references in GenericSharedObject when reading QVector _properties; QByteArray _hash; }; @@ -855,10 +1103,12 @@ private: Q_DECLARE_METATYPE(const QMetaObject*) -/// Macro for registering streamable meta-objects. +/// Macro for registering streamable meta-objects. Typically, one would use this macro at the top level of the source file +/// associated with the class. #define REGISTER_META_OBJECT(x) static int x##Registration = Bitstream::registerMetaObject(#x, &x::staticMetaObject); -/// Contains a value along with a pointer to its streamer. +/// Contains a value along with a pointer to its streamer. This is stored in QVariants when using fallback generics and +/// no mapping to a built-in type can be found, or when using all generics and the value is any non-simple type. class GenericValue { public: @@ -877,7 +1127,8 @@ private: Q_DECLARE_METATYPE(GenericValue) -/// Contains a list of property values along with a pointer to their metadata. +/// Contains a list of property values along with a pointer to their metadata. This is stored when using fallback generics +/// and no mapping to a built-in class can be found, or for all QObjects when using all generics. class GenericSharedObject : public SharedObject { Q_OBJECT @@ -914,6 +1165,12 @@ public: virtual void writeMetadata(Bitstream& out, bool full) const; + virtual QJsonValue getJSONMetadata(JSONWriter& writer) const; + virtual QJsonValue getJSONData(JSONWriter& writer, const QVariant& value) const; + virtual QJsonValue getJSONVariantData(JSONWriter& writer, const QVariant& value) const; + virtual void putJSONData(JSONReader& reader, const QJsonValue& data, QVariant& value) const; + virtual void putJSONVariantData(JSONReader& reader, const QJsonValue& data, QVariant& value) const; + virtual bool equal(const QVariant& first, const QVariant& second) const; virtual void write(Bitstream& out, const QVariant& value) const; @@ -958,7 +1215,7 @@ protected: friend class Bitstream; int _type; - TypeStreamerPointer _self; + TypeStreamerPointer _self; ///< set/used for built-in types (never deleted), to obtain shared pointers }; QDebug& operator<<(QDebug& debug, const TypeStreamer* typeStreamer); @@ -969,6 +1226,10 @@ QDebug& operator<<(QDebug& debug, const QMetaObject* metaObject); template class SimpleTypeStreamer : public TypeStreamer { public: + virtual QJsonValue getJSONData(JSONWriter& writer, const QVariant& value) const { + return writer.getData(value.value()); } + virtual void putJSONData(JSONReader& reader, const QJsonValue& data, QVariant& value) const { + T rawValue; reader.putData(data, rawValue); value = QVariant::fromValue(rawValue); } virtual bool equal(const QVariant& first, const QVariant& second) const { return first.value() == second.value(); } virtual void write(Bitstream& out, const QVariant& value) const { out << value.value(); } virtual QVariant read(Bitstream& in) const { T value; in >> value; return QVariant::fromValue(value); } @@ -991,6 +1252,9 @@ public: virtual const char* getName() const; virtual void writeMetadata(Bitstream& out, bool full) const; + virtual QJsonValue getJSONMetadata(JSONWriter& writer) const; + virtual QJsonValue getJSONData(JSONWriter& writer, const QVariant& value) const; + virtual void putJSONData(JSONReader& reader, const QJsonValue& data, QVariant& value) const; virtual Category getCategory() const; virtual int getBits() const; virtual QMetaEnum getMetaEnum() const; @@ -1018,6 +1282,7 @@ public: MappedEnumTypeStreamer(const TypeStreamer* baseStreamer, int bits, const QHash& mappings); + virtual void putJSONData(JSONReader& reader, const QJsonValue& data, QVariant& value) const; virtual QVariant read(Bitstream& in) const; virtual void readRawDelta(Bitstream& in, QVariant& object, const QVariant& reference) const; @@ -1035,14 +1300,16 @@ public: GenericTypeStreamer(const QByteArray& name); virtual const char* getName() const; + virtual void putJSONVariantData(JSONReader& reader, const QJsonValue& data, QVariant& value) const; virtual QVariant readVariant(Bitstream& in) const; protected: friend class Bitstream; + friend class JSONReader; QByteArray _name; - WeakTypeStreamerPointer _weakSelf; + WeakTypeStreamerPointer _weakSelf; ///< promoted to strong references in GenericValue when reading }; /// A streamer for generic enums. @@ -1052,6 +1319,9 @@ public: GenericEnumTypeStreamer(const QByteArray& name, const QVector& values, int bits, const QByteArray& hash); virtual void writeMetadata(Bitstream& out, bool full) const; + virtual QJsonValue getJSONMetadata(JSONWriter& writer) const; + virtual QJsonValue getJSONData(JSONWriter& writer, const QVariant& value) const; + virtual void putJSONData(JSONReader& reader, const QJsonValue& data, QVariant& value) const; virtual void write(Bitstream& out, const QVariant& value) const; virtual QVariant read(Bitstream& in) const; virtual Category getCategory() const; @@ -1084,6 +1354,7 @@ public: MappedStreamableTypeStreamer(const TypeStreamer* baseStreamer, const QVector& fields); + virtual void putJSONData(JSONReader& reader, const QJsonValue& data, QVariant& value) const; virtual QVariant read(Bitstream& in) const; virtual void readRawDelta(Bitstream& in, QVariant& object, const QVariant& reference) const; @@ -1100,6 +1371,9 @@ public: GenericStreamableTypeStreamer(const QByteArray& name, const QVector& fields, const QByteArray& hash); virtual void writeMetadata(Bitstream& out, bool full) const; + virtual QJsonValue getJSONMetadata(JSONWriter& writer) const; + virtual QJsonValue getJSONData(JSONWriter& writer, const QVariant& value) const; + virtual void putJSONData(JSONReader& reader, const QJsonValue& data, QVariant& value) const; virtual void write(Bitstream& out, const QVariant& value) const; virtual QVariant read(Bitstream& in) const; virtual Category getCategory() const; @@ -1154,6 +1428,7 @@ public: MappedListTypeStreamer(const TypeStreamer* baseStreamer, const TypeStreamerPointer& valueStreamer); + virtual void putJSONData(JSONReader& reader, const QJsonValue& data, QVariant& value) const; virtual QVariant read(Bitstream& in) const; virtual void readRawDelta(Bitstream& in, QVariant& object, const QVariant& reference) const; @@ -1170,11 +1445,14 @@ public: GenericListTypeStreamer(const QByteArray& name, const TypeStreamerPointer& valueStreamer); virtual void writeMetadata(Bitstream& out, bool full) const; + virtual QJsonValue getJSONMetadata(JSONWriter& writer) const; + virtual QJsonValue getJSONData(JSONWriter& writer, const QVariant& value) const; + virtual void putJSONData(JSONReader& reader, const QJsonValue& data, QVariant& value) const; virtual void write(Bitstream& out, const QVariant& value) const; virtual QVariant read(Bitstream& in) const; virtual Category getCategory() const; -private: +protected: TypeStreamerPointer _valueStreamer; }; @@ -1207,6 +1485,7 @@ public: GenericSetTypeStreamer(const QByteArray& name, const TypeStreamerPointer& valueStreamer); + virtual QJsonValue getJSONMetadata(JSONWriter& writer) const; virtual Category getCategory() const; }; @@ -1233,6 +1512,7 @@ public: MappedMapTypeStreamer(const TypeStreamer* baseStreamer, const TypeStreamerPointer& keyStreamer, const TypeStreamerPointer& valueStreamer); + virtual void putJSONData(JSONReader& reader, const QJsonValue& data, QVariant& value) const; virtual QVariant read(Bitstream& in) const; virtual void readRawDelta(Bitstream& in, QVariant& object, const QVariant& reference) const; @@ -1251,6 +1531,9 @@ public: const TypeStreamerPointer& valueStreamer); virtual void writeMetadata(Bitstream& out, bool full) const; + virtual QJsonValue getJSONMetadata(JSONWriter& writer) const; + virtual QJsonValue getJSONData(JSONWriter& writer, const QVariant& value) const; + virtual void putJSONData(JSONReader& reader, const QJsonValue& data, QVariant& value) const; virtual void write(Bitstream& out, const QVariant& value) const; virtual QVariant read(Bitstream& in) const; virtual Category getCategory() const; @@ -1265,25 +1548,31 @@ private: class GenericValueStreamer : public SimpleTypeStreamer { public: + QJsonValue getJSONVariantData(JSONWriter& writer, const QVariant& value) const; virtual void writeVariant(Bitstream& out, const QVariant& value) const; }; -/// Macro for registering simple type streamers. +/// Macro for registering simple type streamers. Typically, one would use this at the top level of the source file +/// associated with the type. #define REGISTER_SIMPLE_TYPE_STREAMER(X) static int X##Streamer = \ Bitstream::registerTypeStreamer(qMetaTypeId(), new SimpleTypeStreamer()); -/// Macro for registering collection type streamers. +/// Macro for registering collection type (QList, QVector, QSet, QMap) streamers. Typically, one would use this at the top +/// level of the source file associated with the type. #define REGISTER_COLLECTION_TYPE_STREAMER(X) static int x##Streamer = \ Bitstream::registerTypeStreamer(qMetaTypeId(), new CollectionTypeStreamer()); -/// Declares the metatype and the streaming operators. The last lines -/// ensure that the generated file will be included in the link phase. +/// Declares the metatype and the streaming operators. Typically, one would use this immediately after the definition of a +/// type flagged as STREAMABLE in its header file. The last lines ensure that the generated file will be included in the link +/// phase. #ifdef _WIN32 #define DECLARE_STREAMABLE_METATYPE(X) Q_DECLARE_METATYPE(X) \ Bitstream& operator<<(Bitstream& out, const X& obj); \ Bitstream& operator>>(Bitstream& in, X& obj); \ template<> void Bitstream::writeRawDelta(const X& value, const X& reference); \ template<> void Bitstream::readRawDelta(X& value, const X& reference); \ + template<> QJsonValue JSONWriter::getData(const X& value); \ + template<> void JSONReader::putData(const QJsonValue& data, X& value); \ bool operator==(const X& first, const X& second); \ bool operator!=(const X& first, const X& second); \ static const int* _TypePtr##X = &X::Type; @@ -1293,6 +1582,8 @@ public: Bitstream& operator>>(Bitstream& in, X& obj); \ template<> void Bitstream::writeRawDelta(const X& value, const X& reference); \ template<> void Bitstream::readRawDelta(X& value, const X& reference); \ + template<> QJsonValue JSONWriter::getData(const X& value); \ + template<> void JSONReader::putData(const QJsonValue& data, X& value); \ bool operator==(const X& first, const X& second); \ bool operator!=(const X& first, const X& second); \ __attribute__((unused)) static const int* _TypePtr##X = &X::Type; @@ -1303,18 +1594,27 @@ public: Bitstream& operator>>(Bitstream& in, X& obj); \ template<> void Bitstream::writeRawDelta(const X& value, const X& reference); \ template<> void Bitstream::readRawDelta(X& value, const X& reference); \ + template<> QJsonValue JSONWriter::getData(const X& value); \ + template<> void JSONReader::putData(const QJsonValue& data, X& value); \ bool operator==(const X& first, const X& second); \ bool operator!=(const X& first, const X& second); \ static const int* _TypePtr##X = &X::Type; \ _Pragma(STRINGIFY(unused(_TypePtr##X))) #endif +/// Declares an enum metatype. This is used when one desires to use an enum defined in a QObject outside of that class's +/// direct properties. Typically, one would use this immediately after the definition of the QObject containing the enum +/// in its header file. #define DECLARE_ENUM_METATYPE(S, N) Q_DECLARE_METATYPE(S::N) \ Bitstream& operator<<(Bitstream& out, const S::N& obj); \ Bitstream& operator>>(Bitstream& in, S::N& obj); \ template<> inline void Bitstream::writeRawDelta(const S::N& value, const S::N& reference) { *this << value; } \ - template<> inline void Bitstream::readRawDelta(S::N& value, const S::N& reference) { *this >> value; } + template<> inline void Bitstream::readRawDelta(S::N& value, const S::N& reference) { *this >> value; } \ + template<> inline QJsonValue JSONWriter::getData(const S::N& value) { return (int)value; } \ + template<> inline void JSONReader::putData(const QJsonValue& data, S::N& value) { value = (S::N)data.toInt(); } +/// Implements an enum metatype. This performs the implementation of the previous macro, and would normally be used at the +/// top level of the source file associated with the QObject containing the enum. #define IMPLEMENT_ENUM_METATYPE(S, N) \ static int S##N##MetaTypeId = registerEnumMetaType(&S::staticMetaObject, #N); \ Bitstream& operator<<(Bitstream& out, const S::N& obj) { \ @@ -1327,7 +1627,9 @@ public: return in.read(&obj, bits); \ } -/// Registers a simple type and its streamer. +/// Registers a simple type and its streamer. This would typically be used at the top level of the source file associated with +/// the type, and combines the registration of the type with the Qt metatype system with the registration of a simple type +/// streamer. /// \return the metatype id template int registerSimpleMetaType() { int type = qRegisterMetaType(); @@ -1335,7 +1637,8 @@ template int registerSimpleMetaType() { return type; } -/// Registers an enum type and its streamer. +/// Registers an enum type and its streamer. Rather than using this directly, consider using the IMPLEMENT_ENUM_METATYPE +/// macro. /// \return the metatype id template int registerEnumMetaType(const QMetaObject* metaObject, const char* name) { int type = qRegisterMetaType(); @@ -1343,7 +1646,8 @@ template int registerEnumMetaType(const QMetaObject* metaObject, const return type; } -/// Registers a streamable type and its streamer. +/// Registers a streamable type and its streamer. Rather than using this directly, consider using the +/// DECLARE_STREAMABLE_METATYPE macro and the mtc code generation tool. /// \return the metatype id template int registerStreamableMetaType() { int type = qRegisterMetaType(); @@ -1351,7 +1655,9 @@ template int registerStreamableMetaType() { return type; } -/// Registers a collection type and its streamer. +/// Registers a collection type and its streamer. This would typically be used at the top level of the source file associated +/// with the type, and combines the registration of the type with the Qt metatype system with the registration of a collection +/// type streamer. /// \return the metatype id template int registerCollectionMetaType() { int type = qRegisterMetaType(); @@ -1359,7 +1665,9 @@ template int registerCollectionMetaType() { return type; } -/// Flags a class as streamable (use as you would Q_OBJECT). +/// Flags a class as streamable. Use as you would Q_OBJECT: as the first line of a class definition, before any members or +/// access qualifiers. Typically, one would follow the definition with DECLARE_STREAMABLE_METATYPE. The mtc tool looks for +/// this macro in order to generate streaming (etc.) code for types. #define STREAMABLE public: \ static const int Type; \ static const QVector& getMetaFields(); \ @@ -1369,7 +1677,8 @@ template int registerCollectionMetaType() { private: \ static QHash createFieldIndices(); -/// Flags a field or base class as streaming. +/// Flags a (public) field within or base class of a streamable type as streaming. Use before all base classes or fields that +/// you want to stream. #define STREAM #endif // hifi_Bitstream_h diff --git a/libraries/metavoxels/src/DatagramSequencer.h b/libraries/metavoxels/src/DatagramSequencer.h index cf6ded74da..5ac88556f0 100644 --- a/libraries/metavoxels/src/DatagramSequencer.h +++ b/libraries/metavoxels/src/DatagramSequencer.h @@ -23,18 +23,50 @@ class ReliableChannel; -/// Performs simple datagram sequencing, packet fragmentation and reassembly. +/// Performs datagram sequencing, packet fragmentation and reassembly. Works with Bitstream to provide methods to send and +/// receive data over UDP with varying reliability and latency characteristics. To use, create a DatagramSequencer with the +/// fixed-size header that will be included with all outgoing datagrams and expected in all incoming ones (the contents of the +/// header are not checked on receive, only skipped over, and may be modified by the party that actually send the +/// datagram--this means that the header may include dynamically generated data, as long as its size remains fixed). Connect +/// the readyToWrite signal to a slot that will actually transmit the datagram to the remote party. When a datagram is +/// received from that party, call receivedDatagram with its contents. +/// +/// A "packet" represents a batch of data sent at one time (split into one or more datagrams sized below the MTU). Packets are +/// received in full and in order or not at all (that is, a packet being assembled is dropped as soon as a fragment from the +/// next packet is received). Packets can be any size, but the larger a packet is, the more likely it is to be dropped--so, +/// it's better to keep packet sizes close to the MTU. To write a packet, call startPacket, write data to the returned +/// Bitstream, then call endPacket (which will result in one or more firings of readyToWrite). Data written in this way is not +/// guaranteed to be received, but if it is received, it will arrive in order. This is a good way to transmit delta state: +/// state that represents the change between the last acknowledged state and the current state (which, if not received, will +/// not be resent as-is; instead, it will be replaced by up-to-date new deltas). +/// +/// There are two methods for sending reliable data. The first, for small messages that require minimum-latency processing, is +/// the high priority messaging system. When you call sendHighPriorityMessage, the message that you send will be included with +/// every outgoing packet until it is acknowledged. When the receiving party first sees the message, it will fire a +/// receivedHighPriorityMessage signal. +/// +/// The second method employs a set of independent reliable channels multiplexed onto the packet stream. These channels are +/// created lazily through the getReliableOutputChannel/getReliableInputChannel functions. Output channels contain buffers +/// to which one may write either arbitrary data (as a QIODevice) or messages (as QVariants), or switch between the two. +/// Each time a packet is sent, data pending for reliable output channels is added, in proportion to their relative priorities, +/// until the packet size limit set by setMaxPacketSize is reached. On the receive side, the streams are reconstructed and +/// (again, depending on whether messages are enabled) either the QIODevice reports that data is available, or, when a complete +/// message is decoded, the receivedMessage signal is fired. class DatagramSequencer : public QObject { Q_OBJECT public: + /// Contains the content of a high-priority message along with the number of the first packet in which it was sent. class HighPriorityMessage { public: QVariant data; int firstPacketNumber; }; + /// Creates a new datagram sequencer. + /// \param datagramHeader the content of the header that will be prepended to each outgoing datagram and whose length + /// will be skipped over in each incoming datagram DatagramSequencer(const QByteArray& datagramHeader = QByteArray(), QObject* parent = NULL); /// Returns a reference to the weak hash mapping remote ids to shared objects. @@ -267,15 +299,24 @@ class ReliableChannel : public QObject { public: + /// Returns the channel's index in the sequencer's channel map. int getIndex() const { return _index; } + /// Returns a reference to the buffer used to write/read data to/from this channel. CircularBuffer& getBuffer() { return _buffer; } + + /// Returns a reference to the data stream created on this channel's buffer. QDataStream& getDataStream() { return _dataStream; } + + /// Returns a reference to the bitstream created on this channel's data stream. Bitstream& getBitstream() { return _bitstream; } + /// Sets the channel priority, which determines how much of this channel's data (in proportion to the other channels) to + /// include in each outgoing packet. void setPriority(float priority) { _priority = priority; } float getPriority() const { return _priority; } + /// Returns the number of bytes available to read from this channel. int getBytesAvailable() const; /// Sets whether we expect to write/read framed messages. @@ -287,6 +328,7 @@ public: signals: + /// Fired when a framed message has been received on this channel. void receivedMessage(const QVariant& message); private slots: diff --git a/tests/metavoxels/src/MetavoxelTests.cpp b/tests/metavoxels/src/MetavoxelTests.cpp index 6bd99a6c82..c9bce27cc3 100644 --- a/tests/metavoxels/src/MetavoxelTests.cpp +++ b/tests/metavoxels/src/MetavoxelTests.cpp @@ -83,7 +83,7 @@ static TestSharedObjectA::TestFlags getRandomTestFlags() { return flags; } -static QScriptValue createRandomScriptValue(bool complex = false) { +static QScriptValue createRandomScriptValue(bool complex = false, bool ensureHashOrder = false) { scriptObjectsCreated++; switch (randIntInRange(0, complex ? 5 : 3)) { case 0: @@ -108,31 +108,37 @@ static QScriptValue createRandomScriptValue(bool complex = false) { } default: { QScriptValue value = ScriptCache::getInstance()->getEngine()->newObject(); - if (randomBoolean()) { + if (ensureHashOrder) { + // we can't depend on the iteration order, so if we need it to be the same (as when comparing bytes), we + // can only have one property value.setProperty("foo", createRandomScriptValue()); - } - if (randomBoolean()) { - value.setProperty("bar", createRandomScriptValue()); - } - if (randomBoolean()) { - value.setProperty("baz", createRandomScriptValue()); - } - if (randomBoolean()) { - value.setProperty("bong", createRandomScriptValue()); + } else { + if (randomBoolean()) { + value.setProperty("foo", createRandomScriptValue()); + } + if (randomBoolean()) { + value.setProperty("bar", createRandomScriptValue()); + } + if (randomBoolean()) { + value.setProperty("baz", createRandomScriptValue()); + } + if (randomBoolean()) { + value.setProperty("bong", createRandomScriptValue()); + } } return value; } } } -static TestMessageC createRandomMessageC() { +static TestMessageC createRandomMessageC(bool ensureHashOrder = false) { TestMessageC message; message.foo = randomBoolean(); message.bar = rand(); message.baz = randFloat(); message.bong.foo = createRandomBytes(); message.bong.baz = getRandomTestEnum(); - message.bizzle = createRandomScriptValue(true); + message.bizzle = createRandomScriptValue(true, ensureHashOrder); return message; } @@ -146,7 +152,7 @@ static bool testSerialization(Bitstream::MetadataType metadataType) { SharedObjectPointer testObjectWrittenB = new TestSharedObjectB(randFloat(), createRandomBytes(), TestSharedObjectB::THIRD_TEST_ENUM, TestSharedObjectB::SECOND_TEST_FLAG); out << testObjectWrittenB; - TestMessageC messageWritten = createRandomMessageC(); + TestMessageC messageWritten = createRandomMessageC(true); out << QVariant::fromValue(messageWritten); QByteArray endWritten = "end"; out << endWritten; @@ -243,6 +249,74 @@ static bool testSerialization(Bitstream::MetadataType metadataType) { return true; } + if (metadataType != Bitstream::FULL_METADATA) { + return false; + } + + // now write to JSON + JSONWriter jsonWriter; + jsonWriter << testObjectReadA; + jsonWriter << testObjectReadB; + jsonWriter << messageRead; + jsonWriter << endRead; + QByteArray encodedJson = jsonWriter.getDocument().toJson(); + + // and read from JSON + JSONReader jsonReader(QJsonDocument::fromJson(encodedJson), Bitstream::ALL_GENERICS); + jsonReader >> testObjectReadA; + jsonReader >> testObjectReadB; + jsonReader >> messageRead; + jsonReader >> endRead; + + // reassign the ids + testObjectReadA->setID(testObjectWrittenA->getID()); + testObjectReadA->setOriginID(testObjectWrittenA->getOriginID()); + testObjectReadB->setID(testObjectWrittenB->getID()); + testObjectReadB->setOriginID(testObjectWrittenB->getOriginID()); + + // and back to binary + QByteArray secondCompareArray; + QDataStream secondCompareOutStream(&secondCompareArray, QIODevice::WriteOnly); + Bitstream secondCompareOut(secondCompareOutStream, Bitstream::FULL_METADATA); + secondCompareOut << testObjectReadA; + secondCompareOut << testObjectReadB; + secondCompareOut << messageRead; + secondCompareOut << endRead; + secondCompareOut.flush(); + + if (compareArray != secondCompareArray) { + qDebug() << "Mismatch between written/JSON streams (generics)."; + return true; + } + + // once more, with mapping! + JSONReader secondJSONReader(QJsonDocument::fromJson(encodedJson)); + secondJSONReader >> testObjectReadA; + secondJSONReader >> testObjectReadB; + secondJSONReader >> messageRead; + secondJSONReader >> endRead; + + // reassign the ids + testObjectReadA->setID(testObjectWrittenA->getID()); + testObjectReadA->setOriginID(testObjectWrittenA->getOriginID()); + testObjectReadB->setID(testObjectWrittenB->getID()); + testObjectReadB->setOriginID(testObjectWrittenB->getOriginID()); + + // and back to binary + QByteArray thirdCompareArray; + QDataStream thirdCompareOutStream(&thirdCompareArray, QIODevice::WriteOnly); + Bitstream thirdCompareOut(thirdCompareOutStream, Bitstream::FULL_METADATA); + thirdCompareOut << testObjectReadA; + thirdCompareOut << testObjectReadB; + thirdCompareOut << messageRead; + thirdCompareOut << endRead; + thirdCompareOut.flush(); + + if (compareArray != thirdCompareArray) { + qDebug() << "Mismatch between written/JSON streams (mapped)."; + return true; + } + return false; } diff --git a/tools/CMakeLists.txt b/tools/CMakeLists.txt new file mode 100644 index 0000000000..79db82e90f --- /dev/null +++ b/tools/CMakeLists.txt @@ -0,0 +1,6 @@ +cmake_minimum_required(VERSION 2.8) + +# add the tool directories +add_subdirectory(bitstream2json) +add_subdirectory(json2bitstream) +add_subdirectory(mtc) diff --git a/tools/bitstream2json/CMakeLists.txt b/tools/bitstream2json/CMakeLists.txt new file mode 100644 index 0000000000..fde80b4d33 --- /dev/null +++ b/tools/bitstream2json/CMakeLists.txt @@ -0,0 +1,26 @@ +cmake_minimum_required(VERSION 2.8) + +if (WIN32) + cmake_policy (SET CMP0020 NEW) +endif (WIN32) + +set(TARGET_NAME bitstream2json) + +set(ROOT_DIR ../..) +set(MACRO_DIR "${ROOT_DIR}/cmake/macros") + +# setup for find modules +set(CMAKE_MODULE_PATH ${CMAKE_MODULE_PATH} "${CMAKE_CURRENT_SOURCE_DIR}/../../cmake/modules/") + +find_package(Qt5 COMPONENTS Network Script Widgets) + +include(${MACRO_DIR}/SetupHifiProject.cmake) +setup_hifi_project(${TARGET_NAME} TRUE) + +link_hifi_library(metavoxels ${TARGET_NAME} "${ROOT_DIR}") +link_hifi_library(shared ${TARGET_NAME} "${ROOT_DIR}") + +include(${MACRO_DIR}/IncludeGLM.cmake) +include_glm(${TARGET_NAME} "${ROOT_DIR}") + +target_link_libraries(${TARGET_NAME} Qt5::Network Qt5::Widgets Qt5::Script) diff --git a/tools/bitstream2json/src/main.cpp b/tools/bitstream2json/src/main.cpp new file mode 100644 index 0000000000..0f299527b0 --- /dev/null +++ b/tools/bitstream2json/src/main.cpp @@ -0,0 +1,70 @@ +// +// main.cpp +// tools/bitstream2json/src +// +// Created by Andrzej Kapolka on 6/17/14. +// Copyright 2014 High Fidelity, Inc. +// +// Distributed under the Apache License, Version 2.0. +// See the accompanying file LICENSE or http://www.apache.org/licenses/LICENSE-2.0.html + +#include + +#include +#include +#include + +#include + +using namespace std; + +int main (int argc, char** argv) { + // need the core application for the script engine + QCoreApplication app(argc, argv); + + if (argc < 3) { + cerr << "Usage: bitstream2json inputfile outputfile [types...]" << endl; + return 0; + } + QFile inputFile(argv[1]); + if (!inputFile.open(QIODevice::ReadOnly)) { + cerr << "Failed to open input file: " << inputFile.errorString().toLatin1().constData() << endl; + return 1; + } + QDataStream inputData(&inputFile); + Bitstream input(inputData, Bitstream::FULL_METADATA, Bitstream::ALL_GENERICS); + + QFile outputFile(argv[2]); + if (!outputFile.open(QIODevice::WriteOnly)) { + cerr << "Failed to open output file: " << outputFile.errorString().toLatin1().constData() << endl; + return 1; + } + JSONWriter output; + + if (argc < 4) { + // default type is a single QVariant + QVariant value; + input >> value; + output << value; + + } else { + for (int i = 3; i < argc; i++) { + int type = QMetaType::type(argv[i]); + if (type == QMetaType::UnknownType) { + cerr << "Unknown type: " << argv[i] << endl; + return 1; + } + const TypeStreamer* streamer = Bitstream::getTypeStreamer(type); + if (!streamer) { + cerr << "Non-streamable type: " << argv[i] << endl; + return 1; + } + QVariant value = streamer->read(input); + output.appendToContents(streamer->getJSONData(output, value)); + } + } + + outputFile.write(output.getDocument().toJson()); + + return 0; +} diff --git a/tools/json2bitstream/CMakeLists.txt b/tools/json2bitstream/CMakeLists.txt new file mode 100644 index 0000000000..51382d5858 --- /dev/null +++ b/tools/json2bitstream/CMakeLists.txt @@ -0,0 +1,26 @@ +cmake_minimum_required(VERSION 2.8) + +if (WIN32) + cmake_policy (SET CMP0020 NEW) +endif (WIN32) + +set(TARGET_NAME json2bitstream) + +set(ROOT_DIR ../..) +set(MACRO_DIR "${ROOT_DIR}/cmake/macros") + +# setup for find modules +set(CMAKE_MODULE_PATH ${CMAKE_MODULE_PATH} "${CMAKE_CURRENT_SOURCE_DIR}/../../cmake/modules/") + +find_package(Qt5 COMPONENTS Network Script Widgets) + +include(${MACRO_DIR}/SetupHifiProject.cmake) +setup_hifi_project(${TARGET_NAME} TRUE) + +link_hifi_library(metavoxels ${TARGET_NAME} "${ROOT_DIR}") +link_hifi_library(shared ${TARGET_NAME} "${ROOT_DIR}") + +include(${MACRO_DIR}/IncludeGLM.cmake) +include_glm(${TARGET_NAME} "${ROOT_DIR}") + +target_link_libraries(${TARGET_NAME} Qt5::Network Qt5::Widgets Qt5::Script) diff --git a/tools/json2bitstream/src/main.cpp b/tools/json2bitstream/src/main.cpp new file mode 100644 index 0000000000..ff09bcfdc6 --- /dev/null +++ b/tools/json2bitstream/src/main.cpp @@ -0,0 +1,76 @@ +// +// main.cpp +// tools/json2bitstream/src +// +// Created by Andrzej Kapolka on 6/17/14. +// Copyright 2014 High Fidelity, Inc. +// +// Distributed under the Apache License, Version 2.0. +// See the accompanying file LICENSE or http://www.apache.org/licenses/LICENSE-2.0.html + +#include + +#include +#include +#include + +#include + +using namespace std; + +int main (int argc, char** argv) { + // need the core application for the script engine + QCoreApplication app(argc, argv); + + if (argc < 3) { + cerr << "Usage: bitstream2json inputfile outputfile [types...]" << endl; + return 0; + } + QFile inputFile(argv[1]); + if (!inputFile.open(QIODevice::ReadOnly)) { + cerr << "Failed to open input file: " << inputFile.errorString().toLatin1().constData() << endl; + return 1; + } + QJsonParseError error; + QJsonDocument document = QJsonDocument::fromJson(inputFile.readAll(), &error); + if (error.error != QJsonParseError::NoError) { + cerr << "Failed to read input file: " << error.errorString().toLatin1().constData() << endl; + return 1; + } + JSONReader input(document, Bitstream::ALL_GENERICS); + + QFile outputFile(argv[2]); + if (!outputFile.open(QIODevice::WriteOnly)) { + cerr << "Failed to open output file: " << outputFile.errorString().toLatin1().constData() << endl; + return 1; + } + QDataStream outputData(&outputFile); + Bitstream output(outputData, Bitstream::FULL_METADATA); + + if (argc < 4) { + // default type is a single QVariant + QVariant value; + input >> value; + output << value; + + } else { + for (int i = 3; i < argc; i++) { + int type = QMetaType::type(argv[i]); + if (type == QMetaType::UnknownType) { + cerr << "Unknown type: " << argv[i] << endl; + return 1; + } + const TypeStreamer* streamer = Bitstream::getTypeStreamer(type); + if (!streamer) { + cerr << "Non-streamable type: " << argv[i] << endl; + return 1; + } + QVariant value; + streamer->putJSONData(input, input.retrieveNextFromContents(), value); + streamer->write(output, value); + } + } + output.flush(); + + return 0; +} diff --git a/tools/mtc/src/main.cpp b/tools/mtc/src/main.cpp index 096ade4625..dd0ab837a5 100644 --- a/tools/mtc/src/main.cpp +++ b/tools/mtc/src/main.cpp @@ -123,11 +123,6 @@ void generateOutput (QTextStream& out, const QList& streamables) { out << "QHash " << name << "::createFieldIndices() {\n"; out << " QHash indices;\n"; out << " int index = 1;\n"; - foreach (const QString& base, str.clazz.bases) { - out << " foreach (const MetaField& field, " << base << "::getMetaFields()) {\n"; - out << " indices.insert(field.getName(), index++);\n"; - out << " }\n"; - } out << " foreach (const MetaField& field, getMetaFields()) {\n"; out << " indices.insert(field.getName(), index++);\n"; out << " }\n"; @@ -217,6 +212,36 @@ void generateOutput (QTextStream& out, const QList& streamables) { } out << "}\n"; + out << "template<> QJsonValue JSONWriter::getData(const " << name << "& value) {\n"; + out << " QJsonArray array;\n"; + foreach (const QString& base, str.clazz.bases) { + out << " foreach (const QJsonValue& element, getData(static_cast(value)).toArray()) {\n"; + out << " array.append(element);\n"; + out << " }\n"; + } + foreach (const Field& field, str.fields) { + out << " array.append(getData(value." << field.name << "));\n"; + } + out << " return array;\n"; + out << "}\n"; + + out << "template<> void JSONReader::putData(const QJsonValue& data, " << name << "& value) {\n"; + if (!(str.clazz.bases.isEmpty() && str.fields.isEmpty())) { + out << " QJsonArray array = data.toArray(), subarray;\n"; + out << " QJsonArray::const_iterator it = array.constBegin();\n"; + foreach (const QString& base, str.clazz.bases) { + out << " subarray = QJsonArray();\n"; + out << " for (int i = 0; i < " << base << "::getMetaFields().size(); i++) {\n"; + out << " subarray.append(*it++);\n"; + out << " }\n"; + out << " putData(subarray, static_cast<" << base << "&>(value));\n"; + } + foreach (const Field& field, str.fields) { + out << " putData(*it++, value." << field.name << ");\n"; + } + } + out << "}\n"; + out << "bool operator==(const " << name << "& first, const " << name << "& second) {\n"; if (str.clazz.bases.isEmpty() && str.fields.isEmpty()) { out << " return true";