From 9c3624d0d970f95b1c60f71d7a59031b5e806a6b Mon Sep 17 00:00:00 2001 From: Andrzej Kapolka Date: Fri, 7 Feb 2014 14:25:48 -0800 Subject: [PATCH 01/17] Fix for texture on jellyrob_blue_blender.fbx; apparently you can map textures, as well as materials, by polygon. Closes #1844. --- interface/src/renderer/FBXReader.cpp | 102 ++++++++++++++++++--------- 1 file changed, 68 insertions(+), 34 deletions(-) diff --git a/interface/src/renderer/FBXReader.cpp b/interface/src/renderer/FBXReader.cpp index 5a27b51b70..e4dfd7dd84 100644 --- a/interface/src/renderer/FBXReader.cpp +++ b/interface/src/renderer/FBXReader.cpp @@ -601,6 +601,7 @@ public: FBXMesh mesh; QMultiHash newIndices; QVector > blendshapeIndexMaps; + QVector > partMaterialTextures; }; class MeshData { @@ -667,6 +668,7 @@ void appendIndex(MeshData& data, QVector& indices, int index) { ExtractedMesh extractMesh(const FBXNode& object) { MeshData data; QVector materials; + QVector textures; foreach (const FBXNode& child, object.children) { if (child.name == "Vertices") { data.vertices = createVec3Vector(getDoubleVector(child.properties, 0)); @@ -703,19 +705,32 @@ ExtractedMesh extractMesh(const FBXNode& object) { materials = getIntVector(subdata.properties, 0); } } + } else if (child.name == "LayerElementTexture") { + foreach (const FBXNode& subdata, child.children) { + if (subdata.name == "TextureId") { + textures = getIntVector(subdata.properties, 0); + } + } } } // convert the polygons to quads and triangles int polygonIndex = 0; + QHash, int> materialTextureParts; for (int beginIndex = 0; beginIndex < data.polygonIndices.size(); polygonIndex++) { int endIndex = beginIndex; while (data.polygonIndices.at(endIndex++) >= 0); - int materialIndex = (polygonIndex < materials.size()) ? materials.at(polygonIndex) : 0; - data.extracted.mesh.parts.resize(max(data.extracted.mesh.parts.size(), materialIndex + 1)); - FBXMeshPart& part = data.extracted.mesh.parts[materialIndex]; - + QPair materialTexture((polygonIndex < materials.size()) ? materials.at(polygonIndex) : 0, + (polygonIndex < textures.size()) ? textures.at(polygonIndex) : 0); + int& partIndex = materialTextureParts[materialTexture]; + if (partIndex == 0) { + data.extracted.partMaterialTextures.append(materialTexture); + data.extracted.mesh.parts.resize(data.extracted.mesh.parts.size() + 1); + partIndex = data.extracted.mesh.parts.size(); + } + FBXMeshPart& part = data.extracted.mesh.parts[partIndex - 1]; + if (endIndex - beginIndex == 4) { appendIndex(data, part.quadIndices, beginIndex++); appendIndex(data, part.quadIndices, beginIndex++); @@ -1265,41 +1280,60 @@ FBXGeometry extractFBXGeometry(const FBXNode& node, const QVariantHash& mapping) glm::mat4 modelTransform = getGlobalTransform(parentMap, models, modelID); // look for textures, material properties - int partIndex = extracted.mesh.parts.size() - 1; + int materialIndex = 0; + int textureIndex = 0; bool generateTangents = false; - foreach (const QString& childID, childMap.values(modelID)) { - if (partIndex < 0) { - break; - } - FBXMeshPart& part = extracted.mesh.parts[partIndex]; - if (textureFilenames.contains(childID)) { - part.diffuseFilename = textureFilenames.value(childID); - continue; - } - if (!materials.contains(childID)) { - continue; - } - Material material = materials.value(childID); - part.diffuseColor = material.diffuse; - part.specularColor = material.specular; - part.shininess = material.shininess; - QString diffuseTextureID = diffuseTextures.value(childID); - if (!diffuseTextureID.isNull()) { - part.diffuseFilename = textureFilenames.value(diffuseTextureID); + QList children = childMap.values(modelID); + for (int i = children.size() - 1; i >= 0; i--) { + const QString& childID = children.at(i); + if (materials.contains(childID)) { + Material material = materials.value(childID); + + QByteArray diffuseFilename; + QString diffuseTextureID = diffuseTextures.value(childID); + if (!diffuseTextureID.isNull()) { + diffuseFilename = textureFilenames.value(diffuseTextureID); - // FBX files generated by 3DSMax have an intermediate texture parent, apparently - foreach (const QString& childTextureID, childMap.values(diffuseTextureID)) { - if (textureFilenames.contains(childTextureID)) { - part.diffuseFilename = textureFilenames.value(childTextureID); + // FBX files generated by 3DSMax have an intermediate texture parent, apparently + foreach (const QString& childTextureID, childMap.values(diffuseTextureID)) { + if (textureFilenames.contains(childTextureID)) { + diffuseFilename = textureFilenames.value(childTextureID); + } } } + + QByteArray normalFilename; + QString bumpTextureID = bumpTextures.value(childID); + if (!bumpTextureID.isNull()) { + normalFilename = textureFilenames.value(bumpTextureID); + generateTangents = true; + } + + for (int j = 0; j < extracted.partMaterialTextures.size(); j++) { + if (extracted.partMaterialTextures.at(j).first == materialIndex) { + FBXMeshPart& part = extracted.mesh.parts[j]; + part.diffuseColor = material.diffuse; + part.specularColor = material.specular; + part.shininess = material.shininess; + if (!diffuseFilename.isNull()) { + part.diffuseFilename = diffuseFilename; + } + if (!normalFilename.isNull()) { + part.normalFilename = normalFilename; + } + } + } + materialIndex++; + + } else if (textureFilenames.contains(childID)) { + QByteArray filename = textureFilenames.value(childID); + for (int j = 0; j < extracted.partMaterialTextures.size(); j++) { + if (extracted.partMaterialTextures.at(j).second == textureIndex) { + extracted.mesh.parts[j].diffuseFilename = filename; + } + } + textureIndex++; } - QString bumpTextureID = bumpTextures.value(childID); - if (!bumpTextureID.isNull()) { - part.normalFilename = textureFilenames.value(bumpTextureID); - generateTangents = true; - } - partIndex--; } // if we have a normal map (and texture coordinates), we must compute tangents From 524ceb4ed2f788ce9ebf012f27a835c148bb12fe Mon Sep 17 00:00:00 2001 From: Andrzej Kapolka Date: Sun, 9 Feb 2014 16:26:53 -0800 Subject: [PATCH 02/17] Working on tests for datagram sequencing, added autogenerated ==/!= operators to mtc. --- CMakeLists.txt | 3 +- .../metavoxels/src/AttributeRegistry.cpp | 8 + libraries/metavoxels/src/AttributeRegistry.h | 3 + libraries/metavoxels/src/Bitstream.h | 4 + tests/CMakeLists.txt | 10 + tests/metavoxels/CMakeLists.txt | 32 ++++ tests/metavoxels/src/MetavoxelTests.cpp | 178 ++++++++++++++++++ tests/metavoxels/src/MetavoxelTests.h | 109 +++++++++++ tests/metavoxels/src/main.cpp | 14 ++ tools/mtc/src/main.cpp | 56 +++++- 10 files changed, 414 insertions(+), 3 deletions(-) create mode 100644 tests/CMakeLists.txt create mode 100644 tests/metavoxels/CMakeLists.txt create mode 100644 tests/metavoxels/src/MetavoxelTests.cpp create mode 100644 tests/metavoxels/src/MetavoxelTests.h create mode 100644 tests/metavoxels/src/main.cpp diff --git a/CMakeLists.txt b/CMakeLists.txt index a583d7d951..f20142d698 100644 --- a/CMakeLists.txt +++ b/CMakeLists.txt @@ -39,4 +39,5 @@ add_subdirectory(assignment-client) add_subdirectory(domain-server) add_subdirectory(interface) add_subdirectory(pairing-server) -add_subdirectory(voxel-edit) \ No newline at end of file +add_subdirectory(tests) +add_subdirectory(voxel-edit) diff --git a/libraries/metavoxels/src/AttributeRegistry.cpp b/libraries/metavoxels/src/AttributeRegistry.cpp index 431eaf62c9..9595d96e1f 100644 --- a/libraries/metavoxels/src/AttributeRegistry.cpp +++ b/libraries/metavoxels/src/AttributeRegistry.cpp @@ -82,6 +82,14 @@ bool AttributeValue::operator==(void* other) const { return _attribute && _attribute->equal(_value, other); } +bool AttributeValue::operator!=(const AttributeValue& other) const { + return _attribute != other._attribute || (_attribute && !_attribute->equal(_value, other._value)); +} + +bool AttributeValue::operator!=(void* other) const { + return !_attribute || !_attribute->equal(_value, other); +} + OwnedAttributeValue::OwnedAttributeValue(const AttributePointer& attribute, void* value) : AttributeValue(attribute, value) { } diff --git a/libraries/metavoxels/src/AttributeRegistry.h b/libraries/metavoxels/src/AttributeRegistry.h index db5e54cc4a..56cff7eeb4 100644 --- a/libraries/metavoxels/src/AttributeRegistry.h +++ b/libraries/metavoxels/src/AttributeRegistry.h @@ -105,6 +105,9 @@ public: bool operator==(const AttributeValue& other) const; bool operator==(void* other) const; + bool operator!=(const AttributeValue& other) const; + bool operator!=(void* other) const; + protected: AttributePointer _attribute; diff --git a/libraries/metavoxels/src/Bitstream.h b/libraries/metavoxels/src/Bitstream.h index 5a79b10766..cc776a742a 100644 --- a/libraries/metavoxels/src/Bitstream.h +++ b/libraries/metavoxels/src/Bitstream.h @@ -415,12 +415,16 @@ public: #define DECLARE_STREAMABLE_METATYPE(X) Q_DECLARE_METATYPE(X) \ Bitstream& operator<<(Bitstream& out, const X& obj); \ Bitstream& operator>>(Bitstream& in, X& obj); \ + bool operator==(const X& first, const X& second); \ + bool operator!=(const X& first, const X& second); \ static const int* _TypePtr##X = &X::Type; #else #define STRINGIFY(x) #x #define DECLARE_STREAMABLE_METATYPE(X) Q_DECLARE_METATYPE(X) \ Bitstream& operator<<(Bitstream& out, const X& obj); \ Bitstream& operator>>(Bitstream& in, X& obj); \ + bool operator==(const X& first, const X& second); \ + bool operator!=(const X& first, const X& second); \ static const int* _TypePtr##X = &X::Type; \ _Pragma(STRINGIFY(unused(_TypePtr##X))) #endif diff --git a/tests/CMakeLists.txt b/tests/CMakeLists.txt new file mode 100644 index 0000000000..e817ffe506 --- /dev/null +++ b/tests/CMakeLists.txt @@ -0,0 +1,10 @@ +cmake_minimum_required(VERSION 2.8) + +# add the test directories +file(GLOB TEST_SUBDIRS RELATIVE ${CMAKE_CURRENT_SOURCE_DIR} ${CMAKE_CURRENT_SOURCE_DIR}/*) +foreach(DIR ${TEST_SUBDIRS}) + if(IS_DIRECTORY ${CMAKE_CURRENT_SOURCE_DIR}/${DIR}) + add_subdirectory(${DIR}) + endif() +endforeach() + diff --git a/tests/metavoxels/CMakeLists.txt b/tests/metavoxels/CMakeLists.txt new file mode 100644 index 0000000000..a21dc2f316 --- /dev/null +++ b/tests/metavoxels/CMakeLists.txt @@ -0,0 +1,32 @@ +cmake_minimum_required(VERSION 2.8) + +set(TARGET_NAME metavoxel-tests) + +set(ROOT_DIR ../..) +set(MACRO_DIR ${ROOT_DIR}/cmake/macros) + +# setup for find modules +set(CMAKE_MODULE_PATH ${CMAKE_MODULE_PATH} "${CMAKE_CURRENT_SOURCE_DIR}/../../cmake/modules/") + +find_package(Qt5Network REQUIRED) +find_package(Qt5Script REQUIRED) +find_package(Qt5Widgets REQUIRED) + +include(${MACRO_DIR}/SetupHifiProject.cmake) +setup_hifi_project(${TARGET_NAME} TRUE) + +include(${MACRO_DIR}/AutoMTC.cmake) +auto_mtc(${TARGET_NAME} ${ROOT_DIR}) + +qt5_use_modules(${TARGET_NAME} Network Script Widgets) + +#include glm +include(${MACRO_DIR}/IncludeGLM.cmake) +include_glm(${TARGET_NAME} ${ROOT_DIR}) + +# link in the shared libraries +include(${MACRO_DIR}/LinkHifiLibrary.cmake) +link_hifi_library(metavoxels ${TARGET_NAME} ${ROOT_DIR}) +link_hifi_library(shared ${TARGET_NAME} ${ROOT_DIR}) + + diff --git a/tests/metavoxels/src/MetavoxelTests.cpp b/tests/metavoxels/src/MetavoxelTests.cpp new file mode 100644 index 0000000000..ec8b474f33 --- /dev/null +++ b/tests/metavoxels/src/MetavoxelTests.cpp @@ -0,0 +1,178 @@ +// +// MetavoxelTests.cpp +// metavoxel-tests +// +// Created by Andrzej Kapolka on 2/7/14. +// Copyright (c) 2014 High Fidelity, Inc. All rights reserved. +// + +#include + +#include +#include + +#include "MetavoxelTests.h" + +MetavoxelTests::MetavoxelTests(int& argc, char** argv) : + QCoreApplication(argc, argv) { +} + +static int highPriorityMessagesSent = 0; +static int unreliableMessagesSent = 0; +static int unreliableMessagesReceived = 0; + +bool MetavoxelTests::run() { + + qDebug() << "Running metavoxel tests..."; + + // seed the random number generator so that our tests are reproducible + srand(0xBAAAAABE); + + // create two endpoints with the same header + QByteArray datagramHeader("testheader"); + Endpoint alice(datagramHeader), bob(datagramHeader); + + alice.setOther(&bob); + bob.setOther(&alice); + + // perform a large number of simulation iterations + const int SIMULATION_ITERATIONS = 100000; + for (int i = 0; i < SIMULATION_ITERATIONS; i++) { + if (alice.simulate(i) || bob.simulate(i)) { + return true; + } + } + + qDebug() << "Sent " << highPriorityMessagesSent << " high priority messages"; + + qDebug() << "Sent " << unreliableMessagesSent << " unreliable messages, received " << unreliableMessagesReceived; + + qDebug() << "All tests passed!"; + + return false; +} + +Endpoint::Endpoint(const QByteArray& datagramHeader) : + _sequencer(new DatagramSequencer(datagramHeader)), + _highPriorityMessagesToSend(0.0f) { + + connect(_sequencer, SIGNAL(readyToWrite(const QByteArray&)), SLOT(sendDatagram(const QByteArray&))); + connect(_sequencer, SIGNAL(readyToRead(Bitstream&)), SLOT(readMessage(Bitstream&))); + connect(_sequencer, SIGNAL(receivedHighPriorityMessage(const QVariant&)), + SLOT(handleHighPriorityMessage(const QVariant&))); +} + +static QByteArray createRandomBytes() { + const int MIN_BYTES = 4; + const int MAX_BYTES = 16; + QByteArray bytes(randIntInRange(MIN_BYTES, MAX_BYTES), 0); + for (int i = 0; i < bytes.size(); i++) { + bytes[i] = rand(); + } + return bytes; +} + +static QVariant createRandomMessage() { + switch (randIntInRange(0, 2)) { + case 0: { + TestMessageA message = { randomBoolean(), rand(), randFloat() }; + return QVariant::fromValue(message); + } + case 1: { + TestMessageB message = { createRandomBytes() }; + return QVariant::fromValue(message); + } + case 2: + default: { + TestMessageC message; + message.foo = randomBoolean(); + message.bar = rand(); + message.baz = randFloat(); + message.bong.foo = createRandomBytes(); + return QVariant::fromValue(message); + } + } +} + +static bool messagesEqual(const QVariant& firstMessage, const QVariant& secondMessage) { + int type = firstMessage.userType(); + if (secondMessage.userType() != type) { + return false; + } + if (type == TestMessageA::Type) { + return firstMessage.value() == secondMessage.value(); + } else if (type == TestMessageB::Type) { + return firstMessage.value() == secondMessage.value(); + } else if (type == TestMessageC::Type) { + return firstMessage.value() == secondMessage.value(); + } else { + return firstMessage == secondMessage; + } +} + +bool Endpoint::simulate(int iterationNumber) { + // enqueue some number of high priority messages + const float MIN_HIGH_PRIORITY_MESSAGES = 0.0f; + const float MAX_HIGH_PRIORITY_MESSAGES = 2.0f; + _highPriorityMessagesToSend += randFloatInRange(MIN_HIGH_PRIORITY_MESSAGES, MAX_HIGH_PRIORITY_MESSAGES); + while (_highPriorityMessagesToSend >= 1.0f) { + QVariant message = createRandomMessage(); + _highPriorityMessagesSent.append(message); + _sequencer->sendHighPriorityMessage(message); + highPriorityMessagesSent++; + _highPriorityMessagesToSend -= 1.0f; + } + + // send a packet + try { + Bitstream& out = _sequencer->startPacket(); + SequencedTestMessage message = { iterationNumber, createRandomMessage() }; + _unreliableMessagesSent.append(message); + unreliableMessagesSent++; + out << message; + _sequencer->endPacket(); + + } catch (const QString& message) { + qDebug() << message; + return true; + } + + return false; +} + +void Endpoint::sendDatagram(const QByteArray& datagram) { + // some datagrams are dropped + const float DROP_PROBABILITY = 0.1f; + if (randFloat() < DROP_PROBABILITY) { + return; + } + _other->_sequencer->receivedDatagram(datagram); +} + +void Endpoint::handleHighPriorityMessage(const QVariant& message) { + if (_other->_highPriorityMessagesSent.isEmpty()) { + throw QString("Received unsent/already sent high priority message."); + } + QVariant sentMessage = _other->_highPriorityMessagesSent.takeFirst(); + if (!messagesEqual(message, sentMessage)) { + throw QString("Sent/received high priority message mismatch."); + } +} + +void Endpoint::readMessage(Bitstream& in) { + SequencedTestMessage message; + in >> message; + + for (QList::iterator it = _other->_unreliableMessagesSent.begin(); + it != _other->_unreliableMessagesSent.end(); it++) { + if (it->sequenceNumber == message.sequenceNumber) { + if (!messagesEqual(it->submessage, message.submessage)) { + throw QString("Sent/received unreliable message mismatch."); + } + _other->_unreliableMessagesSent.erase(_other->_unreliableMessagesSent.begin(), it + 1); + unreliableMessagesReceived++; + return; + } + } + throw QString("Received unsent/already sent unreliable message."); +} diff --git a/tests/metavoxels/src/MetavoxelTests.h b/tests/metavoxels/src/MetavoxelTests.h new file mode 100644 index 0000000000..fb47d9f463 --- /dev/null +++ b/tests/metavoxels/src/MetavoxelTests.h @@ -0,0 +1,109 @@ +// +// MetavoxelTests.h +// metavoxel-tests +// +// Created by Andrzej Kapolka on 2/7/14. +// Copyright (c) 2014 High Fidelity, Inc. All rights reserved. +// + +#ifndef __interface__MetavoxelTests__ +#define __interface__MetavoxelTests__ + +#include +#include + +#include + +class DatagramSequencer; +class SequencedTestMessage; + +/// Tests various aspects of the metavoxel library. +class MetavoxelTests : public QCoreApplication { + Q_OBJECT + +public: + + MetavoxelTests(int& argc, char** argv); + + /// Performs our various tests. + /// \return true if any of the tests failed. + bool run(); +}; + +/// Represents a simulated endpoint. +class Endpoint : public QObject { + Q_OBJECT + +public: + + Endpoint(const QByteArray& datagramHeader); + + void setOther(Endpoint* other) { _other = other; } + + /// Perform a simulation step. + /// \return true if failure was detected + bool simulate(int iterationNumber); + +private slots: + + void sendDatagram(const QByteArray& datagram); + void handleHighPriorityMessage(const QVariant& message); + void readMessage(Bitstream& in); + +private: + + DatagramSequencer* _sequencer; + Endpoint* _other; + float _highPriorityMessagesToSend; + QVariantList _highPriorityMessagesSent; + QList _unreliableMessagesSent; +}; + +/// A simple test message. +class TestMessageA { + STREAMABLE + +public: + + STREAM bool foo; + STREAM int bar; + STREAM float baz; +}; + +DECLARE_STREAMABLE_METATYPE(TestMessageA) + +// Another simple test message. +class TestMessageB { + STREAMABLE + +public: + + STREAM QByteArray foo; +}; + +DECLARE_STREAMABLE_METATYPE(TestMessageB) + +// A test message that demonstrates inheritance and composition. +class TestMessageC : public TestMessageA { + STREAMABLE + +public: + + STREAM TestMessageB bong; +}; + +DECLARE_STREAMABLE_METATYPE(TestMessageC) + +/// Combines a sequence number with a submessage; used for testing unreliable transport. +class SequencedTestMessage { + STREAMABLE + +public: + + STREAM int sequenceNumber; + STREAM QVariant submessage; +}; + +DECLARE_STREAMABLE_METATYPE(SequencedTestMessage) + +#endif /* defined(__interface__MetavoxelTests__) */ diff --git a/tests/metavoxels/src/main.cpp b/tests/metavoxels/src/main.cpp new file mode 100644 index 0000000000..10bf786957 --- /dev/null +++ b/tests/metavoxels/src/main.cpp @@ -0,0 +1,14 @@ +// +// main.cpp +// metavoxel-tests +// +// Created by Andrzej Kapolka on 2/7/14. +// Copyright (c) 2014 High Fidelity, Inc. All rights reserved. + +#include + +#include "MetavoxelTests.h" + +int main(int argc, char** argv) { + return MetavoxelTests(argc, argv).run(); +} diff --git a/tools/mtc/src/main.cpp b/tools/mtc/src/main.cpp index 77f0a069b5..050fe0e418 100644 --- a/tools/mtc/src/main.cpp +++ b/tools/mtc/src/main.cpp @@ -90,7 +90,7 @@ void generateOutput (QTextStream& out, const QList& streamables) { foreach (const Streamable& str, streamables) { const QString& name = str.clazz.name; - out << "Bitstream& operator<< (Bitstream& out, const " << name << "& obj) {\n"; + out << "Bitstream& operator<<(Bitstream& out, const " << name << "& obj) {\n"; foreach (const QString& base, str.clazz.bases) { out << " out << static_cast(obj);\n"; } @@ -100,7 +100,7 @@ void generateOutput (QTextStream& out, const QList& streamables) { out << " return out;\n"; out << "}\n"; - out << "Bitstream& operator>> (Bitstream& in, " << name << "& obj) {\n"; + out << "Bitstream& operator>>(Bitstream& in, " << name << "& obj) {\n"; foreach (const QString& base, str.clazz.bases) { out << " in >> static_cast<" << base << "&>(obj);\n"; } @@ -110,6 +110,58 @@ void generateOutput (QTextStream& out, const QList& streamables) { out << " return in;\n"; out << "}\n"; + out << "bool operator==(const " << name << "& first, const " << name << "& second) {\n"; + if (str.clazz.bases.isEmpty() && str.fields.isEmpty()) { + out << " return true"; + } else { + out << " return "; + bool first = true; + foreach (const QString& base, str.clazz.bases) { + if (!first) { + out << " &&\n"; + out << " "; + } + out << "static_cast<" << base << "&>(first) == static_cast<" << base << "&>(second)"; + first = false; + } + foreach (const QString& field, str.fields) { + if (!first) { + out << " &&\n"; + out << " "; + } + out << "first." << field << " == second." << field; + first = false; + } + } + out << ";\n"; + out << "}\n"; + + out << "bool operator!=(const " << name << "& first, const " << name << "& second) {\n"; + if (str.clazz.bases.isEmpty() && str.fields.isEmpty()) { + out << " return false"; + } else { + out << " return "; + bool first = true; + foreach (const QString& base, str.clazz.bases) { + if (!first) { + out << " ||\n"; + out << " "; + } + out << "static_cast<" << base << "&>(first) != static_cast<" << base << "&>(second)"; + first = false; + } + foreach (const QString& field, str.fields) { + if (!first) { + out << " ||\n"; + out << " "; + } + out << "first." << field << " != second." << field; + first = false; + } + } + out << ";\n"; + out << "}\n"; + out << "const int " << name << "::Type = registerStreamableMetaType<" << name << ">();\n\n"; } } From 8c9f06ceecacf57209a09c8917f770e5c064a325 Mon Sep 17 00:00:00 2001 From: Andrzej Kapolka Date: Sun, 9 Feb 2014 17:43:18 -0800 Subject: [PATCH 03/17] Fix/tests for streaming. --- .../metavoxels/src/DatagramSequencer.cpp | 5 +- libraries/metavoxels/src/DatagramSequencer.h | 3 +- tests/metavoxels/src/MetavoxelTests.cpp | 84 ++++++++++++++++--- tests/metavoxels/src/MetavoxelTests.h | 4 + 4 files changed, 79 insertions(+), 17 deletions(-) diff --git a/libraries/metavoxels/src/DatagramSequencer.cpp b/libraries/metavoxels/src/DatagramSequencer.cpp index af271d97ba..9c300113c6 100644 --- a/libraries/metavoxels/src/DatagramSequencer.cpp +++ b/libraries/metavoxels/src/DatagramSequencer.cpp @@ -195,7 +195,7 @@ void DatagramSequencer::receivedDatagram(const QByteArray& datagram) { for (int i = 0; i < reliableChannels; i++) { quint32 channelIndex; _incomingPacketStream >> channelIndex; - getReliableOutputChannel(channelIndex)->readData(_incomingPacketStream); + getReliableInputChannel(channelIndex)->readData(_incomingPacketStream); } _incomingPacketStream.device()->seek(0); @@ -369,7 +369,7 @@ int SpanList::set(int offset, int length) { int SpanList::setSpans(QList::iterator it, int length) { int remainingLength = length; int totalRemoved = 0; - for (; it != _spans.end(); it++) { + for (; it != _spans.end(); it = _spans.erase(it)) { if (remainingLength < it->unset) { it->unset -= remainingLength; totalRemoved += remainingLength; @@ -378,7 +378,6 @@ int SpanList::setSpans(QList::iterator it, int length) { int combined = it->unset + it->set; remainingLength = qMax(remainingLength - combined, 0); totalRemoved += combined; - it = _spans.erase(it); _totalSet -= it->set; } return qMax(length, totalRemoved); diff --git a/libraries/metavoxels/src/DatagramSequencer.h b/libraries/metavoxels/src/DatagramSequencer.h index 44d3c83116..0d9fc0d2b7 100644 --- a/libraries/metavoxels/src/DatagramSequencer.h +++ b/libraries/metavoxels/src/DatagramSequencer.h @@ -58,7 +58,7 @@ public: /// Returns the output channel at the specified index, creating it if necessary. ReliableChannel* getReliableOutputChannel(int index = 0); - /// Returns the intput channel at the + /// Returns the intput channel at the specified index, creating it if necessary. ReliableChannel* getReliableInputChannel(int index = 0); /// Starts a new packet for transmission. @@ -208,6 +208,7 @@ public: int getIndex() const { return _index; } + QBuffer& getBuffer() { return _buffer; } QDataStream& getDataStream() { return _dataStream; } Bitstream& getBitstream() { return _bitstream; } diff --git a/tests/metavoxels/src/MetavoxelTests.cpp b/tests/metavoxels/src/MetavoxelTests.cpp index ec8b474f33..edfc45ac0d 100644 --- a/tests/metavoxels/src/MetavoxelTests.cpp +++ b/tests/metavoxels/src/MetavoxelTests.cpp @@ -17,9 +17,13 @@ MetavoxelTests::MetavoxelTests(int& argc, char** argv) : QCoreApplication(argc, argv) { } +static int datagramsSent = 0; +static int datagramsReceived = 0; static int highPriorityMessagesSent = 0; static int unreliableMessagesSent = 0; static int unreliableMessagesReceived = 0; +static int streamedBytesSent = 0; +static int lowPriorityStreamedBytesSent = 0; bool MetavoxelTests::run() { @@ -43,15 +47,31 @@ bool MetavoxelTests::run() { } } - qDebug() << "Sent " << highPriorityMessagesSent << " high priority messages"; - - qDebug() << "Sent " << unreliableMessagesSent << " unreliable messages, received " << unreliableMessagesReceived; + qDebug() << "Sent" << highPriorityMessagesSent << "high priority messages"; + qDebug() << "Sent" << unreliableMessagesSent << "unreliable messages, received" << unreliableMessagesReceived; + qDebug() << "Sent" << streamedBytesSent << "streamed bytes"; + qDebug() << "Sent" << lowPriorityStreamedBytesSent << "low-priority streamed bytes"; + qDebug() << "Sent" << datagramsSent << "datagrams, received" << datagramsReceived; qDebug() << "All tests passed!"; return false; } +static QByteArray createRandomBytes(int minimumSize, int maximumSize) { + QByteArray bytes(randIntInRange(minimumSize, maximumSize), 0); + for (int i = 0; i < bytes.size(); i++) { + bytes[i] = rand(); + } + return bytes; +} + +static QByteArray createRandomBytes() { + const int MIN_BYTES = 4; + const int MAX_BYTES = 16; + return createRandomBytes(MIN_BYTES, MAX_BYTES); +} + Endpoint::Endpoint(const QByteArray& datagramHeader) : _sequencer(new DatagramSequencer(datagramHeader)), _highPriorityMessagesToSend(0.0f) { @@ -60,16 +80,17 @@ Endpoint::Endpoint(const QByteArray& datagramHeader) : connect(_sequencer, SIGNAL(readyToRead(Bitstream&)), SLOT(readMessage(Bitstream&))); connect(_sequencer, SIGNAL(receivedHighPriorityMessage(const QVariant&)), SLOT(handleHighPriorityMessage(const QVariant&))); -} - -static QByteArray createRandomBytes() { - const int MIN_BYTES = 4; - const int MAX_BYTES = 16; - QByteArray bytes(randIntInRange(MIN_BYTES, MAX_BYTES), 0); - for (int i = 0; i < bytes.size(); i++) { - bytes[i] = rand(); - } - return bytes; + connect(&_sequencer->getReliableInputChannel()->getBuffer(), SIGNAL(readyRead()), SLOT(readReliableChannel())); + connect(&_sequencer->getReliableInputChannel(1)->getBuffer(), SIGNAL(readyRead()), SLOT(readLowPriorityReliableChannel())); + + // enqueue a large amount of data in a low-priority channel + ReliableChannel* output = _sequencer->getReliableOutputChannel(1); + output->setPriority(0.25f); + const int MIN_LOW_PRIORITY_DATA = 100000; + const int MAX_LOW_PRIORITY_DATA = 200000; + _lowPriorityDataStreamed = createRandomBytes(MIN_LOW_PRIORITY_DATA, MAX_LOW_PRIORITY_DATA); + output->getBuffer().write(_lowPriorityDataStreamed); + lowPriorityStreamedBytesSent += _lowPriorityDataStreamed.size(); } static QVariant createRandomMessage() { @@ -123,6 +144,14 @@ bool Endpoint::simulate(int iterationNumber) { _highPriorityMessagesToSend -= 1.0f; } + // stream some random data + const int MIN_BYTES_TO_STREAM = 10; + const int MAX_BYTES_TO_STREAM = 100; + QByteArray bytes = createRandomBytes(MIN_BYTES_TO_STREAM, MAX_BYTES_TO_STREAM); + _dataStreamed.append(bytes); + streamedBytesSent += bytes.size(); + _sequencer->getReliableOutputChannel()->getDataStream().writeRawData(bytes.constData(), bytes.size()); + // send a packet try { Bitstream& out = _sequencer->startPacket(); @@ -141,12 +170,15 @@ bool Endpoint::simulate(int iterationNumber) { } void Endpoint::sendDatagram(const QByteArray& datagram) { + datagramsSent++; + // some datagrams are dropped const float DROP_PROBABILITY = 0.1f; if (randFloat() < DROP_PROBABILITY) { return; } _other->_sequencer->receivedDatagram(datagram); + datagramsReceived++; } void Endpoint::handleHighPriorityMessage(const QVariant& message) { @@ -176,3 +208,29 @@ void Endpoint::readMessage(Bitstream& in) { } throw QString("Received unsent/already sent unreliable message."); } + +void Endpoint::readReliableChannel() { + QByteArray bytes = _sequencer->getReliableInputChannel()->getBuffer().readAll(); + if (_other->_dataStreamed.size() < bytes.size()) { + throw QString("Received unsent/already sent streamed data."); + } + QByteArray compare = _other->_dataStreamed; + _other->_dataStreamed = _other->_dataStreamed.mid(bytes.size()); + compare.truncate(bytes.size()); + if (compare != bytes) { + throw QString("Sent/received streamed data mismatch."); + } +} + +void Endpoint::readLowPriorityReliableChannel() { + QByteArray bytes = _sequencer->getReliableInputChannel(1)->getBuffer().readAll(); + if (_other->_lowPriorityDataStreamed.size() < bytes.size()) { + throw QString("Received unsent/already sent low-priority streamed data."); + } + QByteArray compare = _other->_lowPriorityDataStreamed; + _other->_lowPriorityDataStreamed = _other->_lowPriorityDataStreamed.mid(bytes.size()); + compare.truncate(bytes.size()); + if (compare != bytes) { + throw QString("Sent/received low-priority streamed data mismatch."); + } +} diff --git a/tests/metavoxels/src/MetavoxelTests.h b/tests/metavoxels/src/MetavoxelTests.h index fb47d9f463..7c2bd29b2f 100644 --- a/tests/metavoxels/src/MetavoxelTests.h +++ b/tests/metavoxels/src/MetavoxelTests.h @@ -49,6 +49,8 @@ private slots: void sendDatagram(const QByteArray& datagram); void handleHighPriorityMessage(const QVariant& message); void readMessage(Bitstream& in); + void readReliableChannel(); + void readLowPriorityReliableChannel(); private: @@ -57,6 +59,8 @@ private: float _highPriorityMessagesToSend; QVariantList _highPriorityMessagesSent; QList _unreliableMessagesSent; + QByteArray _dataStreamed; + QByteArray _lowPriorityDataStreamed; }; /// A simple test message. From f81a9d9fe8978094a00c03e91ff9379933b78e8a Mon Sep 17 00:00:00 2001 From: Andrzej Kapolka Date: Sun, 9 Feb 2014 22:29:23 -0800 Subject: [PATCH 04/17] Circular buffer work. --- .../metavoxels/src/DatagramSequencer.cpp | 204 ++++++++++++++++-- libraries/metavoxels/src/DatagramSequencer.h | 55 ++++- tests/metavoxels/src/MetavoxelTests.cpp | 23 +- tests/metavoxels/src/MetavoxelTests.h | 7 +- 4 files changed, 253 insertions(+), 36 deletions(-) diff --git a/libraries/metavoxels/src/DatagramSequencer.cpp b/libraries/metavoxels/src/DatagramSequencer.cpp index 9c300113c6..26354bc9f5 100644 --- a/libraries/metavoxels/src/DatagramSequencer.cpp +++ b/libraries/metavoxels/src/DatagramSequencer.cpp @@ -311,6 +311,174 @@ void DatagramSequencer::handleHighPriorityMessage(const QVariant& data) { } } +const int INITIAL_CIRCULAR_BUFFER_CAPACITY = 16; + +CircularBuffer::CircularBuffer(QObject* parent) : + QIODevice(parent), + _data(INITIAL_CIRCULAR_BUFFER_CAPACITY, 0), + _position(0), + _size(0), + _offset(0) { +} + +void CircularBuffer::append(const char* data, int length) { + // resize to fit + int oldSize = _size; + resize(_size + length); + + // write our data in up to two segments: one from the position to the end, one from the beginning + int end = (_position + oldSize) % _data.size(); + int firstSegment = qMin(length, _data.size() - end); + memcpy(_data.data() + end, data, firstSegment); + int secondSegment = length - firstSegment; + if (secondSegment > 0) { + memcpy(_data.data(), data + firstSegment, secondSegment); + } +} + +void CircularBuffer::remove(int length) { + _position = (_position + length) % _data.size(); + _size -= length; +} + +QByteArray CircularBuffer::readBytes(int offset, int length) const { + // write in up to two segments + QByteArray array; + int start = (_position + offset) % _data.size(); + int firstSegment = qMin(length, _data.size() - start); + array.append(_data.constData() + start, firstSegment); + int secondSegment = length - firstSegment; + if (secondSegment > 0) { + array.append(_data.constData(), secondSegment); + } + return array; +} + +void CircularBuffer::writeToStream(int offset, int length, QDataStream& out) const { + // write in up to two segments + int start = (_position + offset) % _data.size(); + int firstSegment = qMin(length, _data.size() - start); + out.writeRawData(_data.constData() + start, firstSegment); + int secondSegment = length - firstSegment; + if (secondSegment > 0) { + out.writeRawData(_data.constData(), secondSegment); + } +} + +void CircularBuffer::readFromStream(int offset, int length, QDataStream& in) { + // resize to fit + int requiredSize = offset + length; + if (requiredSize > _size) { + resize(requiredSize); + } + + // read in up to two segments + int start = (_position + offset) % _data.size(); + int firstSegment = qMin(length, _data.size() - start); + in.readRawData(_data.data() + start, firstSegment); + int secondSegment = length - firstSegment; + if (secondSegment > 0) { + in.readRawData(_data.data(), secondSegment); + } +} + +void CircularBuffer::appendToBuffer(int offset, int length, CircularBuffer& buffer) const { + // append in up to two segments + int start = (_position + offset) % _data.size(); + int firstSegment = qMin(length, _data.size() - start); + buffer.append(_data.constData() + start, firstSegment); + int secondSegment = length - firstSegment; + if (secondSegment > 0) { + buffer.append(_data.constData(), secondSegment); + } +} + +bool CircularBuffer::atEnd() const { + return _offset >= _size; +} + +bool CircularBuffer::canReadLine() const { + for (int offset = _offset; offset < _size; offset++) { + if (_data.at((_position + offset) % _data.size()) == '\n') { + return true; + } + } + return false; +} + +bool CircularBuffer::open(OpenMode flags) { + return QIODevice::open(flags | QIODevice::Unbuffered); +} + +qint64 CircularBuffer::pos() const { + return _offset; +} + +bool CircularBuffer::seek(qint64 pos) { + if (pos < 0 || pos > _size) { + return false; + } + _offset = pos; + return true; +} + +qint64 CircularBuffer::size() const { + return _size; +} + +qint64 CircularBuffer::readData(char* data, qint64 length) { + int readable = qMin((int)length, _size - _offset); + + // read in up to two segments + int start = (_position + _offset) % _data.size(); + int firstSegment = qMin((int)length, _data.size() - start); + memcpy(data, _data.constData() + start, firstSegment); + int secondSegment = length - firstSegment; + if (secondSegment > 0) { + memcpy(data + firstSegment, _data.constData(), secondSegment); + } + _offset += readable; + return readable; +} + +qint64 CircularBuffer::writeData(const char* data, qint64 length) { + // resize to fit + int requiredSize = _offset + length; + if (requiredSize > _size) { + resize(requiredSize); + } + + // write in up to two segments + int start = (_position + _offset) % _data.size(); + int firstSegment = qMin((int)length, _data.size() - start); + memcpy(_data.data() + start, data, firstSegment); + int secondSegment = length - firstSegment; + if (secondSegment > 0) { + memcpy(_data.data(), data + firstSegment, secondSegment); + } + _offset += length; + return length; +} + +void CircularBuffer::resize(int size) { + if (size > _data.size()) { + // double our capacity until we can fit the desired length + int newCapacity = _data.size(); + do { + newCapacity *= 2; + } while (size > newCapacity); + + int oldCapacity = _data.size(); + _data.resize(newCapacity); + + int trailing = _position + _size - oldCapacity; + if (trailing > 0) { + memcpy(_data.data() + oldCapacity, _data.constData(), trailing); + } + } + _size = size; +} + SpanList::SpanList() : _totalSet(0) { } @@ -472,16 +640,16 @@ int ReliableChannel::writeSpan(QDataStream& out, bool& first, int position, int spans.append(span); out << (quint32)span.offset; out << (quint32)length; - out.writeRawData(_buffer.data().constData() + position, length); + _buffer.writeToStream(position, length, out); return length; } void ReliableChannel::spanAcknowledged(const DatagramSequencer::ChannelSpan& span) { int advancement = _acknowledged.set(span.offset - _offset, span.length); if (advancement > 0) { - // TODO: better way of pruning buffer - _buffer.buffer() = _buffer.buffer().right(_buffer.size() - advancement); + _buffer.remove(advancement); _buffer.seek(_buffer.size()); + _offset += advancement; _writePosition = qMax(_writePosition - advancement, 0); } @@ -490,38 +658,40 @@ void ReliableChannel::spanAcknowledged(const DatagramSequencer::ChannelSpan& spa void ReliableChannel::readData(QDataStream& in) { quint32 segments; in >> segments; + bool readSome = false; for (int i = 0; i < segments; i++) { quint32 offset, size; in >> offset >> size; int position = offset - _offset; int end = position + size; - if (_assemblyBuffer.size() < end) { - _assemblyBuffer.resize(end); - } if (end <= 0) { in.skipRawData(size); + } else if (position < 0) { in.skipRawData(-position); - in.readRawData(_assemblyBuffer.data(), size + position); + _assemblyBuffer.readFromStream(0, end, in); + } else { - in.readRawData(_assemblyBuffer.data() + position, size); + _assemblyBuffer.readFromStream(position, size, in); } int advancement = _acknowledged.set(position, size); if (advancement > 0) { - // TODO: better way of pruning buffer - _buffer.buffer().append(_assemblyBuffer.constData(), advancement); - emit _buffer.readyRead(); - _assemblyBuffer = _assemblyBuffer.right(_assemblyBuffer.size() - advancement); + _assemblyBuffer.appendToBuffer(0, advancement, _buffer); + _assemblyBuffer.remove(advancement); _offset += advancement; + readSome = true; } } - // when the read head is sufficiently advanced into the buffer, prune it off. this along - // with other buffer usages should be replaced with a circular buffer - const int PRUNE_SIZE = 8192; - if (_buffer.pos() > PRUNE_SIZE) { - _buffer.buffer() = _buffer.buffer().right(_buffer.size() - _buffer.pos()); + // let listeners know that there's data to read + if (readSome) { + emit _buffer.readyRead(); + } + + // prune any read data from the buffer + if (_buffer.pos() > 0) { + _buffer.remove((int)_buffer.pos()); _buffer.seek(0); } } diff --git a/libraries/metavoxels/src/DatagramSequencer.h b/libraries/metavoxels/src/DatagramSequencer.h index 0d9fc0d2b7..5afd5175a2 100644 --- a/libraries/metavoxels/src/DatagramSequencer.h +++ b/libraries/metavoxels/src/DatagramSequencer.h @@ -167,6 +167,55 @@ private: QHash _reliableInputChannels; }; +/// A circular buffer, where one may efficiently append data to the end or remove data from the beginning. +class CircularBuffer : public QIODevice { +public: + + CircularBuffer(QObject* parent = NULL); + + /// Appends data to the end of the buffer. + void append(const QByteArray& data) { append(data.constData(), data.size()); } + + /// Appends data to the end of the buffer. + void append(const char* data, int length); + + /// Removes data from the beginning of the buffer. + void remove(int length); + + /// Reads part of the data from the buffer. + QByteArray readBytes(int offset, int length) const; + + /// Writes part of the buffer to the supplied stream. + void writeToStream(int offset, int length, QDataStream& out) const; + + /// Reads part of the buffer from the supplied stream. + void readFromStream(int offset, int length, QDataStream& in); + + /// Appends part of the buffer to the supplied other buffer. + void appendToBuffer(int offset, int length, CircularBuffer& buffer) const; + + virtual bool atEnd() const; + virtual bool canReadLine() const; + virtual bool open(OpenMode flags); + virtual qint64 pos() const; + virtual bool seek(qint64 pos); + virtual qint64 size() const; + +protected: + + virtual qint64 readData(char* data, qint64 length); + virtual qint64 writeData(const char* data, qint64 length); + +private: + + void resize(int size); + + QByteArray _data; + int _position; + int _size; + int _offset; +}; + /// A list of contiguous spans, alternating between set and unset. Conceptually, the list is preceeded by a set /// span of infinite length and followed by an unset span of infinite length. Within those bounds, it alternates /// between unset and set. @@ -208,7 +257,7 @@ public: int getIndex() const { return _index; } - QBuffer& getBuffer() { return _buffer; } + CircularBuffer& getBuffer() { return _buffer; } QDataStream& getDataStream() { return _dataStream; } Bitstream& getBitstream() { return _bitstream; } @@ -238,8 +287,8 @@ private: void readData(QDataStream& in); int _index; - QBuffer _buffer; - QByteArray _assemblyBuffer; + CircularBuffer _buffer; + CircularBuffer _assemblyBuffer; QDataStream _dataStream; Bitstream _bitstream; float _priority; diff --git a/tests/metavoxels/src/MetavoxelTests.cpp b/tests/metavoxels/src/MetavoxelTests.cpp index edfc45ac0d..7c09e60ef9 100644 --- a/tests/metavoxels/src/MetavoxelTests.cpp +++ b/tests/metavoxels/src/MetavoxelTests.cpp @@ -8,7 +8,6 @@ #include -#include #include #include "MetavoxelTests.h" @@ -88,9 +87,9 @@ Endpoint::Endpoint(const QByteArray& datagramHeader) : output->setPriority(0.25f); const int MIN_LOW_PRIORITY_DATA = 100000; const int MAX_LOW_PRIORITY_DATA = 200000; - _lowPriorityDataStreamed = createRandomBytes(MIN_LOW_PRIORITY_DATA, MAX_LOW_PRIORITY_DATA); - output->getBuffer().write(_lowPriorityDataStreamed); - lowPriorityStreamedBytesSent += _lowPriorityDataStreamed.size(); + _lowPriorityDataStreamed.append(createRandomBytes(MIN_LOW_PRIORITY_DATA, MAX_LOW_PRIORITY_DATA)); + //output->getBuffer().write(_lowPriorityDataStreamed); + //lowPriorityStreamedBytesSent += _lowPriorityDataStreamed.size(); } static QVariant createRandomMessage() { @@ -210,26 +209,26 @@ void Endpoint::readMessage(Bitstream& in) { } void Endpoint::readReliableChannel() { - QByteArray bytes = _sequencer->getReliableInputChannel()->getBuffer().readAll(); + CircularBuffer& buffer = _sequencer->getReliableInputChannel()->getBuffer(); + QByteArray bytes = buffer.read(buffer.bytesAvailable()); if (_other->_dataStreamed.size() < bytes.size()) { throw QString("Received unsent/already sent streamed data."); } - QByteArray compare = _other->_dataStreamed; - _other->_dataStreamed = _other->_dataStreamed.mid(bytes.size()); - compare.truncate(bytes.size()); + QByteArray compare = _other->_dataStreamed.readBytes(0, bytes.size()); + _other->_dataStreamed.remove(bytes.size()); if (compare != bytes) { throw QString("Sent/received streamed data mismatch."); } } void Endpoint::readLowPriorityReliableChannel() { - QByteArray bytes = _sequencer->getReliableInputChannel(1)->getBuffer().readAll(); + CircularBuffer& buffer = _sequencer->getReliableInputChannel(1)->getBuffer(); + QByteArray bytes = buffer.read(buffer.bytesAvailable()); if (_other->_lowPriorityDataStreamed.size() < bytes.size()) { throw QString("Received unsent/already sent low-priority streamed data."); } - QByteArray compare = _other->_lowPriorityDataStreamed; - _other->_lowPriorityDataStreamed = _other->_lowPriorityDataStreamed.mid(bytes.size()); - compare.truncate(bytes.size()); + QByteArray compare = _other->_lowPriorityDataStreamed.readBytes(0, bytes.size()); + _other->_lowPriorityDataStreamed.remove(bytes.size()); if (compare != bytes) { throw QString("Sent/received low-priority streamed data mismatch."); } diff --git a/tests/metavoxels/src/MetavoxelTests.h b/tests/metavoxels/src/MetavoxelTests.h index 7c2bd29b2f..372718afc4 100644 --- a/tests/metavoxels/src/MetavoxelTests.h +++ b/tests/metavoxels/src/MetavoxelTests.h @@ -12,9 +12,8 @@ #include #include -#include +#include -class DatagramSequencer; class SequencedTestMessage; /// Tests various aspects of the metavoxel library. @@ -59,8 +58,8 @@ private: float _highPriorityMessagesToSend; QVariantList _highPriorityMessagesSent; QList _unreliableMessagesSent; - QByteArray _dataStreamed; - QByteArray _lowPriorityDataStreamed; + CircularBuffer _dataStreamed; + CircularBuffer _lowPriorityDataStreamed; }; /// A simple test message. From ee2984ce6d3c5fc40622076e6d79e0cd938cbf7e Mon Sep 17 00:00:00 2001 From: Andrzej Kapolka Date: Sun, 9 Feb 2014 23:38:33 -0800 Subject: [PATCH 05/17] Streaming fixes. --- .../metavoxels/src/DatagramSequencer.cpp | 10 ++++----- tests/metavoxels/src/MetavoxelTests.cpp | 22 +++++++++++++------ 2 files changed, 20 insertions(+), 12 deletions(-) diff --git a/libraries/metavoxels/src/DatagramSequencer.cpp b/libraries/metavoxels/src/DatagramSequencer.cpp index 26354bc9f5..adaefc9af1 100644 --- a/libraries/metavoxels/src/DatagramSequencer.cpp +++ b/libraries/metavoxels/src/DatagramSequencer.cpp @@ -591,14 +591,13 @@ void ReliableChannel::writeData(QDataStream& out, int bytes, QVector 0 && leftover > 0) { spanCount++; - remainingBytes -= getBytesToWrite(first, leftover); + remainingBytes -= getBytesToWrite(first, qMin(remainingBytes, leftover)); } } @@ -615,8 +614,9 @@ void ReliableChannel::writeData(QDataStream& out, int bytes, QVector 0 && position < _buffer.pos()) { - remainingBytes -= writeSpan(out, first, position, qMin(remainingBytes, (int)(_buffer.pos() - position)), spans); + int leftover = _buffer.pos() - position; + if (remainingBytes > 0 && leftover > 0) { + remainingBytes -= writeSpan(out, first, position, qMin(remainingBytes, leftover), spans); } } } diff --git a/tests/metavoxels/src/MetavoxelTests.cpp b/tests/metavoxels/src/MetavoxelTests.cpp index 7c09e60ef9..4d7569f2d6 100644 --- a/tests/metavoxels/src/MetavoxelTests.cpp +++ b/tests/metavoxels/src/MetavoxelTests.cpp @@ -19,10 +19,13 @@ MetavoxelTests::MetavoxelTests(int& argc, char** argv) : static int datagramsSent = 0; static int datagramsReceived = 0; static int highPriorityMessagesSent = 0; +static int highPriorityMessagesReceived = 0; static int unreliableMessagesSent = 0; static int unreliableMessagesReceived = 0; static int streamedBytesSent = 0; +static int streamedBytesReceived = 0; static int lowPriorityStreamedBytesSent = 0; +static int lowPriorityStreamedBytesReceived = 0; bool MetavoxelTests::run() { @@ -46,10 +49,11 @@ bool MetavoxelTests::run() { } } - qDebug() << "Sent" << highPriorityMessagesSent << "high priority messages"; + qDebug() << "Sent" << highPriorityMessagesSent << "high priority messages, received" << highPriorityMessagesReceived; qDebug() << "Sent" << unreliableMessagesSent << "unreliable messages, received" << unreliableMessagesReceived; - qDebug() << "Sent" << streamedBytesSent << "streamed bytes"; - qDebug() << "Sent" << lowPriorityStreamedBytesSent << "low-priority streamed bytes"; + qDebug() << "Sent" << streamedBytesSent << "streamed bytes, received" << streamedBytesReceived; + qDebug() << "Sent" << lowPriorityStreamedBytesSent << "low-priority streamed bytes, received" << + lowPriorityStreamedBytesReceived; qDebug() << "Sent" << datagramsSent << "datagrams, received" << datagramsReceived; qDebug() << "All tests passed!"; @@ -87,9 +91,10 @@ Endpoint::Endpoint(const QByteArray& datagramHeader) : output->setPriority(0.25f); const int MIN_LOW_PRIORITY_DATA = 100000; const int MAX_LOW_PRIORITY_DATA = 200000; - _lowPriorityDataStreamed.append(createRandomBytes(MIN_LOW_PRIORITY_DATA, MAX_LOW_PRIORITY_DATA)); - //output->getBuffer().write(_lowPriorityDataStreamed); - //lowPriorityStreamedBytesSent += _lowPriorityDataStreamed.size(); + QByteArray bytes = createRandomBytes(MIN_LOW_PRIORITY_DATA, MAX_LOW_PRIORITY_DATA); + _lowPriorityDataStreamed.append(bytes); + output->getBuffer().write(bytes); + lowPriorityStreamedBytesSent += bytes.size(); } static QVariant createRandomMessage() { @@ -172,7 +177,7 @@ void Endpoint::sendDatagram(const QByteArray& datagram) { datagramsSent++; // some datagrams are dropped - const float DROP_PROBABILITY = 0.1f; + const float DROP_PROBABILITY = 0.25f; if (randFloat() < DROP_PROBABILITY) { return; } @@ -188,6 +193,7 @@ void Endpoint::handleHighPriorityMessage(const QVariant& message) { if (!messagesEqual(message, sentMessage)) { throw QString("Sent/received high priority message mismatch."); } + highPriorityMessagesReceived++; } void Endpoint::readMessage(Bitstream& in) { @@ -219,6 +225,7 @@ void Endpoint::readReliableChannel() { if (compare != bytes) { throw QString("Sent/received streamed data mismatch."); } + streamedBytesReceived += bytes.size(); } void Endpoint::readLowPriorityReliableChannel() { @@ -232,4 +239,5 @@ void Endpoint::readLowPriorityReliableChannel() { if (compare != bytes) { throw QString("Sent/received low-priority streamed data mismatch."); } + lowPriorityStreamedBytesReceived += bytes.size(); } From e9579feef5d9dc2979d2e281b025d1745b42c96c Mon Sep 17 00:00:00 2001 From: Andrzej Kapolka Date: Mon, 10 Feb 2014 00:41:15 -0800 Subject: [PATCH 06/17] Reordering testing. --- .../metavoxels/src/DatagramSequencer.cpp | 7 +++- libraries/metavoxels/src/DatagramSequencer.h | 3 +- tests/metavoxels/src/MetavoxelTests.cpp | 33 +++++++++++++++++-- tests/metavoxels/src/MetavoxelTests.h | 1 + 4 files changed, 40 insertions(+), 4 deletions(-) diff --git a/libraries/metavoxels/src/DatagramSequencer.cpp b/libraries/metavoxels/src/DatagramSequencer.cpp index adaefc9af1..1d56f45c9e 100644 --- a/libraries/metavoxels/src/DatagramSequencer.cpp +++ b/libraries/metavoxels/src/DatagramSequencer.cpp @@ -20,7 +20,8 @@ const int MAX_DATAGRAM_SIZE = MAX_PACKET_SIZE; const int DEFAULT_MAX_PACKET_SIZE = 3000; -DatagramSequencer::DatagramSequencer(const QByteArray& datagramHeader) : +DatagramSequencer::DatagramSequencer(const QByteArray& datagramHeader, QObject* parent) : + QObject(parent), _outgoingPacketStream(&_outgoingPacketData, QIODevice::WriteOnly), _outputStream(_outgoingPacketStream), _incomingDatagramStream(&_incomingDatagramBuffer), @@ -397,6 +398,10 @@ bool CircularBuffer::atEnd() const { return _offset >= _size; } +qint64 CircularBuffer::bytesAvailable() const { + return _size - _offset + QIODevice::bytesAvailable(); +} + bool CircularBuffer::canReadLine() const { for (int offset = _offset; offset < _size; offset++) { if (_data.at((_position + offset) % _data.size()) == '\n') { diff --git a/libraries/metavoxels/src/DatagramSequencer.h b/libraries/metavoxels/src/DatagramSequencer.h index 5afd5175a2..27a4f05379 100644 --- a/libraries/metavoxels/src/DatagramSequencer.h +++ b/libraries/metavoxels/src/DatagramSequencer.h @@ -32,7 +32,7 @@ public: int firstPacketNumber; }; - DatagramSequencer(const QByteArray& datagramHeader = QByteArray()); + DatagramSequencer(const QByteArray& datagramHeader = QByteArray(), QObject* parent = NULL); /// Returns the packet number of the last packet sent. int getOutgoingPacketNumber() const { return _outgoingPacketNumber; } @@ -195,6 +195,7 @@ public: void appendToBuffer(int offset, int length, CircularBuffer& buffer) const; virtual bool atEnd() const; + virtual qint64 bytesAvailable() const; virtual bool canReadLine() const; virtual bool open(OpenMode flags); virtual qint64 pos() const; diff --git a/tests/metavoxels/src/MetavoxelTests.cpp b/tests/metavoxels/src/MetavoxelTests.cpp index 4d7569f2d6..15d7463742 100644 --- a/tests/metavoxels/src/MetavoxelTests.cpp +++ b/tests/metavoxels/src/MetavoxelTests.cpp @@ -76,7 +76,7 @@ static QByteArray createRandomBytes() { } Endpoint::Endpoint(const QByteArray& datagramHeader) : - _sequencer(new DatagramSequencer(datagramHeader)), + _sequencer(new DatagramSequencer(datagramHeader, this)), _highPriorityMessagesToSend(0.0f) { connect(_sequencer, SIGNAL(readyToWrite(const QByteArray&)), SLOT(sendDatagram(const QByteArray&))); @@ -136,6 +136,18 @@ static bool messagesEqual(const QVariant& firstMessage, const QVariant& secondMe } bool Endpoint::simulate(int iterationNumber) { + // update/send our delayed datagrams + for (QList >::iterator it = _delayedDatagrams.begin(); it != _delayedDatagrams.end(); ) { + if (it->second-- == 1) { + _other->_sequencer->receivedDatagram(it->first); + datagramsReceived++; + it = _delayedDatagrams.erase(it); + + } else { + it++; + } + } + // enqueue some number of high priority messages const float MIN_HIGH_PRIORITY_MESSAGES = 0.0f; const float MAX_HIGH_PRIORITY_MESSAGES = 2.0f; @@ -177,10 +189,27 @@ void Endpoint::sendDatagram(const QByteArray& datagram) { datagramsSent++; // some datagrams are dropped - const float DROP_PROBABILITY = 0.25f; + const float DROP_PROBABILITY = 0.1f; if (randFloat() < DROP_PROBABILITY) { return; } + + // some are received out of order + const float REORDER_PROBABILITY = 0.1f; + if (randFloat() < REORDER_PROBABILITY) { + const int MIN_DELAY = 1; + const int MAX_DELAY = 5; + // have to copy the datagram; the one we're passed is a reference to a shared buffer + _delayedDatagrams.append(QPair(QByteArray(datagram.constData(), datagram.size()), + randIntInRange(MIN_DELAY, MAX_DELAY))); + + // and some are duplicated + const float DUPLICATE_PROBABILITY = 0.01f; + if (randFloat() > DUPLICATE_PROBABILITY) { + return; + } + } + _other->_sequencer->receivedDatagram(datagram); datagramsReceived++; } diff --git a/tests/metavoxels/src/MetavoxelTests.h b/tests/metavoxels/src/MetavoxelTests.h index 372718afc4..b73f7eb07e 100644 --- a/tests/metavoxels/src/MetavoxelTests.h +++ b/tests/metavoxels/src/MetavoxelTests.h @@ -55,6 +55,7 @@ private: DatagramSequencer* _sequencer; Endpoint* _other; + QList > _delayedDatagrams; float _highPriorityMessagesToSend; QVariantList _highPriorityMessagesSent; QList _unreliableMessagesSent; From c39f9f627159649fe7b00503a1a3a5cb58c923e0 Mon Sep 17 00:00:00 2001 From: Andrzej Kapolka Date: Mon, 10 Feb 2014 00:59:51 -0800 Subject: [PATCH 07/17] Fix (hopefully) for Windows runtime error. --- libraries/metavoxels/src/MetavoxelUtil.cpp | 31 ++++++++++++++++++---- 1 file changed, 26 insertions(+), 5 deletions(-) diff --git a/libraries/metavoxels/src/MetavoxelUtil.cpp b/libraries/metavoxels/src/MetavoxelUtil.cpp index e59cf0fe21..c6f8f5451e 100644 --- a/libraries/metavoxels/src/MetavoxelUtil.cpp +++ b/libraries/metavoxels/src/MetavoxelUtil.cpp @@ -11,12 +11,12 @@ #include #include #include +#include #include #include #include #include #include -#include #include #include @@ -76,27 +76,48 @@ static QItemEditorFactory* getItemEditorFactory() { return factory; } +/// Because Windows doesn't necessarily have the staticMetaObject available when we want to create, +/// this class simply delays the value property name lookup until actually requested. +template class LazyItemEditorCreator : public QItemEditorCreatorBase { +public: + + virtual QWidget* createWidget(QWidget* parent) const { return new T(parent); } + + virtual QByteArray valuePropertyName() const; + +protected: + + QByteArray _valuePropertyName; +}; + +template QByteArray LazyItemEditorCreator::valuePropertyName() const { + if (_valuePropertyName.isNull()) { + const_cast*>(this)->_valuePropertyName = T::staticMetaObject.userProperty().name(); + } + return _valuePropertyName; +} + static QItemEditorCreatorBase* createDoubleEditorCreator() { - QItemEditorCreatorBase* creator = new QStandardItemEditorCreator(); + QItemEditorCreatorBase* creator = new LazyItemEditorCreator(); getItemEditorFactory()->registerEditor(qMetaTypeId(), creator); getItemEditorFactory()->registerEditor(qMetaTypeId(), creator); return creator; } static QItemEditorCreatorBase* createQColorEditorCreator() { - QItemEditorCreatorBase* creator = new QStandardItemEditorCreator(); + QItemEditorCreatorBase* creator = new LazyItemEditorCreator(); getItemEditorFactory()->registerEditor(qMetaTypeId(), creator); return creator; } static QItemEditorCreatorBase* createVec3EditorCreator() { - QItemEditorCreatorBase* creator = new QStandardItemEditorCreator(); + QItemEditorCreatorBase* creator = new LazyItemEditorCreator(); getItemEditorFactory()->registerEditor(qMetaTypeId(), creator); return creator; } static QItemEditorCreatorBase* createParameterizedURLEditorCreator() { - QItemEditorCreatorBase* creator = new QStandardItemEditorCreator(); + QItemEditorCreatorBase* creator = new LazyItemEditorCreator(); getItemEditorFactory()->registerEditor(qMetaTypeId(), creator); return creator; } From 200a2d77c255d0d77db68f605510356a25d2f7c3 Mon Sep 17 00:00:00 2001 From: Andrzej Kapolka Date: Mon, 10 Feb 2014 10:13:52 -0800 Subject: [PATCH 08/17] Incorporated fixes for warnings that Brad reported on Windows. --- libraries/metavoxels/src/DatagramSequencer.cpp | 8 ++++---- libraries/metavoxels/src/MetavoxelUtil.cpp | 9 +++++++-- libraries/metavoxels/src/MetavoxelUtil.h | 3 +++ libraries/metavoxels/src/SharedObject.cpp | 4 ++-- 4 files changed, 16 insertions(+), 8 deletions(-) diff --git a/libraries/metavoxels/src/DatagramSequencer.cpp b/libraries/metavoxels/src/DatagramSequencer.cpp index 1d56f45c9e..3cbf4a3ac9 100644 --- a/libraries/metavoxels/src/DatagramSequencer.cpp +++ b/libraries/metavoxels/src/DatagramSequencer.cpp @@ -175,10 +175,10 @@ void DatagramSequencer::receivedDatagram(const QByteArray& datagram) { } // read and dispatch the high-priority messages - int highPriorityMessageCount; + quint32 highPriorityMessageCount; _incomingPacketStream >> highPriorityMessageCount; int newHighPriorityMessages = highPriorityMessageCount - _receivedHighPriorityMessages; - for (int i = 0; i < highPriorityMessageCount; i++) { + for (quint32 i = 0; i < highPriorityMessageCount; i++) { QVariant data; _inputStream >> data; if (i >= _receivedHighPriorityMessages) { @@ -193,7 +193,7 @@ void DatagramSequencer::receivedDatagram(const QByteArray& datagram) { // read the reliable data, if any quint32 reliableChannels; _incomingPacketStream >> reliableChannels; - for (int i = 0; i < reliableChannels; i++) { + for (quint32 i = 0; i < reliableChannels; i++) { quint32 channelIndex; _incomingPacketStream >> channelIndex; getReliableInputChannel(channelIndex)->readData(_incomingPacketStream); @@ -664,7 +664,7 @@ void ReliableChannel::readData(QDataStream& in) { quint32 segments; in >> segments; bool readSome = false; - for (int i = 0; i < segments; i++) { + for (quint32 i = 0; i < segments; i++) { quint32 offset, size; in >> offset >> size; diff --git a/libraries/metavoxels/src/MetavoxelUtil.cpp b/libraries/metavoxels/src/MetavoxelUtil.cpp index c6f8f5451e..d6b42b11fb 100644 --- a/libraries/metavoxels/src/MetavoxelUtil.cpp +++ b/libraries/metavoxels/src/MetavoxelUtil.cpp @@ -141,6 +141,12 @@ QUuid readSessionID(const QByteArray& data, const SharedNodePointer& sendingNode return QUuid::fromRfc4122(QByteArray::fromRawData(data.constData() + headerSize, UUID_BYTES)); } +QByteArray signal(const char* signature) { + static QByteArray prototype = SIGNAL(dummyMethod()); + QByteArray signal = prototype; + return signal.replace("dummyMethod()", signature); +} + bool Box::contains(const Box& other) const { return other.minimum.x >= minimum.x && other.maximum.x <= maximum.x && other.minimum.y >= minimum.y && other.maximum.y <= maximum.y && @@ -322,8 +328,7 @@ void ParameterizedURLEditor::continueUpdatingParameters() { QMetaProperty widgetProperty = widgetMetaObject->property(widgetMetaObject->indexOfProperty(valuePropertyName)); widgetProperty.write(widget, _url.getParameters().value(parameter.name)); if (widgetProperty.hasNotifySignal()) { - connect(widget, QByteArray(SIGNAL()).append(widgetProperty.notifySignal().methodSignature()), - SLOT(updateURL())); + connect(widget, signal(widgetProperty.notifySignal().methodSignature()), SLOT(updateURL())); } } } diff --git a/libraries/metavoxels/src/MetavoxelUtil.h b/libraries/metavoxels/src/MetavoxelUtil.h index 9fa721d21c..7cfedd7a62 100644 --- a/libraries/metavoxels/src/MetavoxelUtil.h +++ b/libraries/metavoxels/src/MetavoxelUtil.h @@ -33,6 +33,9 @@ class NetworkProgram; /// \return the session ID, or a null ID if invalid (in which case a warning will be logged) QUuid readSessionID(const QByteArray& data, const SharedNodePointer& sendingNode, int& headerPlusIDSize); +/// Performs the runtime equivalent of Qt's SIGNAL macro, which is to attach a prefix to the signature. +QByteArray signal(const char* signature); + /// A streamable axis-aligned bounding box. class Box { STREAMABLE diff --git a/libraries/metavoxels/src/SharedObject.cpp b/libraries/metavoxels/src/SharedObject.cpp index a3af307297..f97e285bcf 100644 --- a/libraries/metavoxels/src/SharedObject.cpp +++ b/libraries/metavoxels/src/SharedObject.cpp @@ -13,6 +13,7 @@ #include #include "Bitstream.h" +#include "MetavoxelUtil.h" #include "SharedObject.h" SharedObject::SharedObject() : _referenceCount(0) { @@ -204,8 +205,7 @@ void SharedObjectEditor::updateType() { QMetaProperty widgetProperty = widgetMetaObject->property(widgetMetaObject->indexOfProperty(valuePropertyName)); widgetProperty.write(widget, property.read(newObject)); if (widgetProperty.hasNotifySignal()) { - connect(widget, QByteArray(SIGNAL()).append(widgetProperty.notifySignal().methodSignature()), - SLOT(propertyChanged())); + connect(widget, signal(widgetProperty.notifySignal().methodSignature()), SLOT(propertyChanged())); } } } From 712692f8bf327801e217f3c462a79804e279dd5f Mon Sep 17 00:00:00 2001 From: Stephen Birarda Date: Mon, 10 Feb 2014 11:15:35 -0800 Subject: [PATCH 09/17] add fallback default URLs for head and body models --- interface/src/avatar/Avatar.cpp | 4 ++-- interface/src/avatar/MyAvatar.cpp | 2 +- libraries/avatars/src/AvatarData.cpp | 14 ++++++++++---- 3 files changed, 13 insertions(+), 7 deletions(-) diff --git a/interface/src/avatar/Avatar.cpp b/interface/src/avatar/Avatar.cpp index 131eb4d37f..b97a50ce46 100644 --- a/interface/src/avatar/Avatar.cpp +++ b/interface/src/avatar/Avatar.cpp @@ -338,12 +338,12 @@ bool Avatar::findSphereCollision(const glm::vec3& sphereCenter, float sphereRadi void Avatar::setFaceModelURL(const QUrl &faceModelURL) { AvatarData::setFaceModelURL(faceModelURL); - _head.getFaceModel().setURL(faceModelURL); + _head.getFaceModel().setURL(_faceModelURL); } void Avatar::setSkeletonModelURL(const QUrl &skeletonModelURL) { AvatarData::setSkeletonModelURL(skeletonModelURL); - _skeletonModel.setURL(skeletonModelURL); + _skeletonModel.setURL(_skeletonModelURL); } int Avatar::parseData(const QByteArray& packet) { diff --git a/interface/src/avatar/MyAvatar.cpp b/interface/src/avatar/MyAvatar.cpp index 98eb9a4431..87d825e6e5 100644 --- a/interface/src/avatar/MyAvatar.cpp +++ b/interface/src/avatar/MyAvatar.cpp @@ -637,7 +637,7 @@ void MyAvatar::loadData(QSettings* settings) { Application::getInstance()->getCamera()->setScale(_scale); setFaceModelURL(settings->value("faceModelURL").toUrl()); - setSkeletonModelURL(settings->value("skeletonModelURL").toUrl()); + setSkeletonModelURL(settings->value("faceModelURL").toUrl()); settings->endGroup(); } diff --git a/libraries/avatars/src/AvatarData.cpp b/libraries/avatars/src/AvatarData.cpp index b2cd1cf076..a3f1897a04 100644 --- a/libraries/avatars/src/AvatarData.cpp +++ b/libraries/avatars/src/AvatarData.cpp @@ -300,14 +300,20 @@ QByteArray AvatarData::identityByteArray() { return identityData; } +const QUrl DEFAULT_HEAD_MODEL_URL = QUrl("https://s3-us-west-1.amazonaws.com/highfidelity-public/meshes/defaultAvatar_head.fbx"); + void AvatarData::setFaceModelURL(const QUrl& faceModelURL) { - qDebug() << "Changing face model for avatar to" << faceModelURL.toString(); - _faceModelURL = faceModelURL; + _faceModelURL = faceModelURL.isEmpty() ? DEFAULT_HEAD_MODEL_URL : faceModelURL; + + qDebug() << "Changing face model for avatar to" << _faceModelURL.toString(); } +const QUrl DEFAULT_BODY_MODEL_URL = QUrl("https://s3-us-west-1.amazonaws.com/highfidelity-public/meshes/defaultAvatar_body.fbx"); + void AvatarData::setSkeletonModelURL(const QUrl& skeletonModelURL) { - qDebug() << "Changing skeleton model for avatar to" << skeletonModelURL.toString(); - _skeletonModelURL = skeletonModelURL; + _skeletonModelURL = skeletonModelURL.isEmpty() ? DEFAULT_BODY_MODEL_URL : skeletonModelURL; + + qDebug() << "Changing skeleton model for avatar to" << _skeletonModelURL.toString(); } void AvatarData::setClampedTargetScale(float targetScale) { From a0789d3a29bd2615be8c45973aac08a540598e3d Mon Sep 17 00:00:00 2001 From: ZappoMan Date: Mon, 10 Feb 2014 11:21:27 -0800 Subject: [PATCH 10/17] clean up bad edit packet warnings to help reduce log clutter --- libraries/shared/src/Logging.cpp | 3 +++ libraries/shared/src/PacketHeaders.cpp | 3 +++ libraries/voxels/src/VoxelTree.cpp | 30 ++++++++++++++++++++------ 3 files changed, 30 insertions(+), 6 deletions(-) diff --git a/libraries/shared/src/Logging.cpp b/libraries/shared/src/Logging.cpp index bc0d4af084..a9f71e346e 100644 --- a/libraries/shared/src/Logging.cpp +++ b/libraries/shared/src/Logging.cpp @@ -98,6 +98,9 @@ const char* stringForLogType(QtMsgType msgType) { const char DATE_STRING_FORMAT[] = "%F %H:%M:%S %z"; void Logging::verboseMessageHandler(QtMsgType type, const QMessageLogContext& context, const QString& message) { + if (message.isEmpty()) { + return; + } // log prefix is in the following format // [DEBUG] [TIMESTAMP] [PID:PARENT_PID] [TARGET] logged string diff --git a/libraries/shared/src/PacketHeaders.cpp b/libraries/shared/src/PacketHeaders.cpp index b46a57d4aa..b1f6ef1730 100644 --- a/libraries/shared/src/PacketHeaders.cpp +++ b/libraries/shared/src/PacketHeaders.cpp @@ -57,6 +57,9 @@ PacketVersion versionForPacketType(PacketType type) { case PacketTypeDataServerConfirm: case PacketTypeDataServerSend: return 1; + case PacketTypeVoxelSet: + case PacketTypeVoxelSetDestructive: + return 1; default: return 0; } diff --git a/libraries/voxels/src/VoxelTree.cpp b/libraries/voxels/src/VoxelTree.cpp index e29cfda41d..08ec2cfc3c 100644 --- a/libraries/voxels/src/VoxelTree.cpp +++ b/libraries/voxels/src/VoxelTree.cpp @@ -521,6 +521,8 @@ bool VoxelTree::handlesEditPacketType(PacketType packetType) const { } } +const unsigned int REPORT_OVERFLOW_WARNING_INTERVAL = 100; +unsigned int overflowWarnings = 0; int VoxelTree::processEditPacketData(PacketType packetType, const unsigned char* packetData, int packetLength, const unsigned char* editData, int maxLength, const SharedNodePointer& node) { @@ -532,9 +534,17 @@ int VoxelTree::processEditPacketData(PacketType packetType, const unsigned char* int octets = numberOfThreeBitSectionsInCode(editData, maxLength); if (octets == OVERFLOWED_OCTCODE_BUFFER) { - printf("WARNING! Got voxel edit record that would overflow buffer in numberOfThreeBitSectionsInCode(), "); - printf("bailing processing of packet!\n"); - return 0; + overflowWarnings++; + if (overflowWarnings % REPORT_OVERFLOW_WARNING_INTERVAL == 1) { + qDebug() << "WARNING! Got voxel edit record that would overflow buffer in numberOfThreeBitSectionsInCode()" + " [NOTE: this is warning number" << overflowWarnings << ", the next" << + (REPORT_OVERFLOW_WARNING_INTERVAL-1) << "will be suppressed.]"; + + QDebug debug = qDebug(); + debug << "edit data contents:"; + outputBufferBits(editData, maxLength, &debug); + } + return maxLength; } const int COLOR_SIZE_IN_BYTES = 3; @@ -542,9 +552,17 @@ int VoxelTree::processEditPacketData(PacketType packetType, const unsigned char* int voxelDataSize = voxelCodeSize + COLOR_SIZE_IN_BYTES; if (voxelDataSize > maxLength) { - printf("WARNING! Got voxel edit record that would overflow buffer, bailing processing of packet!\n"); - printf("bailing processing of packet!\n"); - return 0; + overflowWarnings++; + if (overflowWarnings % REPORT_OVERFLOW_WARNING_INTERVAL == 1) { + qDebug() << "WARNING! Got voxel edit record that would overflow buffer." + " [NOTE: this is warning number" << overflowWarnings << ", the next" << + (REPORT_OVERFLOW_WARNING_INTERVAL-1) << "will be suppressed.]"; + + QDebug debug = qDebug(); + debug << "edit data contents:"; + outputBufferBits(editData, maxLength, &debug); + } + return maxLength; } readCodeColorBufferToTree(editData, destructive); From 3e90bcd523f0519adb9436c38ed4bb4d914ee3c6 Mon Sep 17 00:00:00 2001 From: Stephen Birarda Date: Mon, 10 Feb 2014 11:28:22 -0800 Subject: [PATCH 11/17] use shorter default URLs, set as placeholder text in prefs --- interface/src/Menu.cpp | 2 ++ interface/src/avatar/MyAvatar.cpp | 2 +- libraries/avatars/src/AvatarData.cpp | 4 ---- libraries/avatars/src/AvatarData.h | 3 +++ 4 files changed, 6 insertions(+), 5 deletions(-) diff --git a/interface/src/Menu.cpp b/interface/src/Menu.cpp index af41e583b9..e7f6120d52 100644 --- a/interface/src/Menu.cpp +++ b/interface/src/Menu.cpp @@ -774,11 +774,13 @@ void Menu::editPreferences() { QString faceURLString = applicationInstance->getAvatar()->getHead().getFaceModel().getURL().toString(); QLineEdit* faceURLEdit = new QLineEdit(faceURLString); faceURLEdit->setMinimumWidth(QLINE_MINIMUM_WIDTH); + faceURLEdit->setPlaceholderText(DEFAULT_HEAD_MODEL_URL.toString()); form->addRow("Face URL:", faceURLEdit); QString skeletonURLString = applicationInstance->getAvatar()->getSkeletonModel().getURL().toString(); QLineEdit* skeletonURLEdit = new QLineEdit(skeletonURLString); skeletonURLEdit->setMinimumWidth(QLINE_MINIMUM_WIDTH); + skeletonURLEdit->setPlaceholderText(DEFAULT_BODY_MODEL_URL.toString()); form->addRow("Skeleton URL:", skeletonURLEdit); QSlider* pupilDilation = new QSlider(Qt::Horizontal); diff --git a/interface/src/avatar/MyAvatar.cpp b/interface/src/avatar/MyAvatar.cpp index 87d825e6e5..98eb9a4431 100644 --- a/interface/src/avatar/MyAvatar.cpp +++ b/interface/src/avatar/MyAvatar.cpp @@ -637,7 +637,7 @@ void MyAvatar::loadData(QSettings* settings) { Application::getInstance()->getCamera()->setScale(_scale); setFaceModelURL(settings->value("faceModelURL").toUrl()); - setSkeletonModelURL(settings->value("faceModelURL").toUrl()); + setSkeletonModelURL(settings->value("skeletonModelURL").toUrl()); settings->endGroup(); } diff --git a/libraries/avatars/src/AvatarData.cpp b/libraries/avatars/src/AvatarData.cpp index a3f1897a04..8981f78023 100644 --- a/libraries/avatars/src/AvatarData.cpp +++ b/libraries/avatars/src/AvatarData.cpp @@ -300,16 +300,12 @@ QByteArray AvatarData::identityByteArray() { return identityData; } -const QUrl DEFAULT_HEAD_MODEL_URL = QUrl("https://s3-us-west-1.amazonaws.com/highfidelity-public/meshes/defaultAvatar_head.fbx"); - void AvatarData::setFaceModelURL(const QUrl& faceModelURL) { _faceModelURL = faceModelURL.isEmpty() ? DEFAULT_HEAD_MODEL_URL : faceModelURL; qDebug() << "Changing face model for avatar to" << _faceModelURL.toString(); } -const QUrl DEFAULT_BODY_MODEL_URL = QUrl("https://s3-us-west-1.amazonaws.com/highfidelity-public/meshes/defaultAvatar_body.fbx"); - void AvatarData::setSkeletonModelURL(const QUrl& skeletonModelURL) { _skeletonModelURL = skeletonModelURL.isEmpty() ? DEFAULT_BODY_MODEL_URL : skeletonModelURL; diff --git a/libraries/avatars/src/AvatarData.h b/libraries/avatars/src/AvatarData.h index 5c3c9ad9af..43db344899 100755 --- a/libraries/avatars/src/AvatarData.h +++ b/libraries/avatars/src/AvatarData.h @@ -52,6 +52,9 @@ static const float MIN_AVATAR_SCALE = .005f; const float MAX_AUDIO_LOUDNESS = 1000.0; // close enough for mouth animation +const QUrl DEFAULT_HEAD_MODEL_URL = QUrl("http://public.highfidelity.io/meshes/defaultAvatar_head.fbx"); +const QUrl DEFAULT_BODY_MODEL_URL = QUrl("http://public.highfidelity.io/meshes/defaultAvatar_body.fbx"); + enum KeyState { NO_KEY_DOWN = 0, INSERT_KEY_DOWN, From 988c98d4d7e7fe95cf048949aa56c1b65a01b39d Mon Sep 17 00:00:00 2001 From: Stephen Birarda Date: Mon, 10 Feb 2014 12:04:52 -0800 Subject: [PATCH 12/17] remove an antequated comment --- libraries/shared/src/NodeList.cpp | 5 +---- 1 file changed, 1 insertion(+), 4 deletions(-) diff --git a/libraries/shared/src/NodeList.cpp b/libraries/shared/src/NodeList.cpp index 92f6f961d5..d079d8a07d 100644 --- a/libraries/shared/src/NodeList.cpp +++ b/libraries/shared/src/NodeList.cpp @@ -80,10 +80,7 @@ NodeList::~NodeList() { clear(); } -bool NodeList::packetVersionAndHashMatch(const QByteArray& packet) { - // currently this just checks if the version in the packet matches our return from versionForPacketType - // may need to be expanded in the future for types and versions that take > than 1 byte - +bool NodeList::packetVersionAndHashMatch(const QByteArray& packet) { if (packet[1] != versionForPacketType(packetTypeForPacket(packet)) && packetTypeForPacket(packet) != PacketTypeStunResponse) { PacketType mismatchType = packetTypeForPacket(packet); From 37d7872677ff8df917db69f160bf1470ca813e92 Mon Sep 17 00:00:00 2001 From: Stephen Birarda Date: Mon, 10 Feb 2014 12:06:08 -0800 Subject: [PATCH 13/17] fix a bug in version mismatched packets --- libraries/shared/src/NodeList.cpp | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/libraries/shared/src/NodeList.cpp b/libraries/shared/src/NodeList.cpp index d079d8a07d..c0bc7f0010 100644 --- a/libraries/shared/src/NodeList.cpp +++ b/libraries/shared/src/NodeList.cpp @@ -80,11 +80,11 @@ NodeList::~NodeList() { clear(); } -bool NodeList::packetVersionAndHashMatch(const QByteArray& packet) { +bool NodeList::packetVersionAndHashMatch(const QByteArray& packet) { if (packet[1] != versionForPacketType(packetTypeForPacket(packet)) && packetTypeForPacket(packet) != PacketTypeStunResponse) { PacketType mismatchType = packetTypeForPacket(packet); - int numPacketTypeBytes = arithmeticCodingValueFromBuffer(packet.data()); + int numPacketTypeBytes = numBytesArithmeticCodingFromBuffer(packet.data()); qDebug() << "Packet version mismatch on" << packetTypeForPacket(packet) << "- Sender" << uuidFromPacketHeader(packet) << "sent" << qPrintable(QString::number(packet[numPacketTypeBytes])) << "but" From f8a24a2e995c5b6f51c72f7d888cb5eaab96fd8d Mon Sep 17 00:00:00 2001 From: ZappoMan Date: Mon, 10 Feb 2014 12:19:46 -0800 Subject: [PATCH 14/17] fix crash in multiple voxel edit scripts running simultaneously --- examples/gameoflife.js | 1 - libraries/shared/src/NodeList.cpp | 2 +- .../voxels/src/VoxelEditPacketSender.cpp | 23 +++++-------------- 3 files changed, 7 insertions(+), 19 deletions(-) diff --git a/examples/gameoflife.js b/examples/gameoflife.js index 6779941dc7..d72a72c2de 100644 --- a/examples/gameoflife.js +++ b/examples/gameoflife.js @@ -115,7 +115,6 @@ function sendNextCells() { var sentFirstBoard = false; function step() { -print("step()..."); if (sentFirstBoard) { // we've already sent the first full board, perform a step in time updateCells(); diff --git a/libraries/shared/src/NodeList.cpp b/libraries/shared/src/NodeList.cpp index 2672f3b27b..18b6e9883e 100644 --- a/libraries/shared/src/NodeList.cpp +++ b/libraries/shared/src/NodeList.cpp @@ -87,7 +87,7 @@ bool NodeList::packetVersionAndHashMatch(const QByteArray& packet) { if (packet[1] != versionForPacketType(packetTypeForPacket(packet)) && packetTypeForPacket(packet) != PacketTypeStunResponse) { PacketType mismatchType = packetTypeForPacket(packet); - int numPacketTypeBytes = arithmeticCodingValueFromBuffer(packet.data()); + int numPacketTypeBytes = numBytesArithmeticCodingFromBuffer(packet.data()); qDebug() << "Packet version mismatch on" << packetTypeForPacket(packet) << "- Sender" << uuidFromPacketHeader(packet) << "sent" << qPrintable(QString::number(packet[numPacketTypeBytes])) << "but" diff --git a/libraries/voxels/src/VoxelEditPacketSender.cpp b/libraries/voxels/src/VoxelEditPacketSender.cpp index fea75abf23..a6d3668207 100644 --- a/libraries/voxels/src/VoxelEditPacketSender.cpp +++ b/libraries/voxels/src/VoxelEditPacketSender.cpp @@ -14,24 +14,13 @@ #include #include "VoxelEditPacketSender.h" -////////////////////////////////////////////////////////////////////////////////////////// -// Function: createVoxelEditMessage() -// Description: creates an "insert" or "remove" voxel message for a voxel code -// corresponding to the closest voxel which encloses a cube with -// lower corners at x,y,z, having side of length S. -// The input values x,y,z range 0.0 <= v < 1.0 -// message should be either 'S' for SET or 'E' for ERASE -// -// IMPORTANT: The buffer is returned to you a buffer which you MUST delete when you are -// done with it. -// -// HACK ATTACK: Well, what if this is larger than the MTU? That's the caller's problem, we -// just truncate the message -// -// Complaints: Brad :) #define GUESS_OF_VOXELCODE_SIZE 10 #define MAXIMUM_EDIT_VOXEL_MESSAGE_SIZE 1500 #define SIZE_OF_COLOR_DATA sizeof(rgbColor) +/// creates an "insert" or "remove" voxel message for a voxel code corresponding to the closest voxel which encloses a cube +/// with lower corners at x,y,z, having side of length S. The input values x,y,z range 0.0 <= v < 1.0 message should be either +/// PacketTypeVoxelSet, PacketTypeVoxelSetDestructive, or PacketTypeVoxelErase. The buffer is returned to caller becomes +/// responsibility of caller and MUST be deleted by caller. bool createVoxelEditMessage(PacketType command, short int sequence, int voxelCount, VoxelDetail* voxelDetails, unsigned char*& bufferOut, int& sizeOut) { @@ -144,9 +133,9 @@ void VoxelEditPacketSender::queueVoxelEditMessages(PacketType type, int numberOf for (int i = 0; i < numberOfDetails; i++) { // use MAX_PACKET_SIZE since it's static and guarenteed to be larger than _maxPacketSize - static unsigned char bufferOut[MAX_PACKET_SIZE]; + unsigned char bufferOut[MAX_PACKET_SIZE]; int sizeOut = 0; - + if (encodeVoxelEditMessageDetails(type, 1, &details[i], &bufferOut[0], _maxPacketSize, sizeOut)) { queueOctreeEditMessage(type, bufferOut, sizeOut); } From 4bb7bb2b77ec1a08bb62df128da79f610cd9d077 Mon Sep 17 00:00:00 2001 From: ZappoMan Date: Mon, 10 Feb 2014 12:48:45 -0800 Subject: [PATCH 15/17] make pending packets thread safe --- .../octree/src/OctreeEditPacketSender.cpp | 22 +++++++++---------- libraries/octree/src/OctreeEditPacketSender.h | 5 +++-- 2 files changed, 13 insertions(+), 14 deletions(-) diff --git a/libraries/octree/src/OctreeEditPacketSender.cpp b/libraries/octree/src/OctreeEditPacketSender.cpp index 1c1ed13f8d..38abd15dc8 100644 --- a/libraries/octree/src/OctreeEditPacketSender.cpp +++ b/libraries/octree/src/OctreeEditPacketSender.cpp @@ -39,6 +39,7 @@ OctreeEditPacketSender::OctreeEditPacketSender() : } OctreeEditPacketSender::~OctreeEditPacketSender() { + _pendingPacketsLock.lock(); while (!_preServerSingleMessagePackets.empty()) { EditPacketBuffer* packet = _preServerSingleMessagePackets.front(); delete packet; @@ -49,6 +50,7 @@ OctreeEditPacketSender::~OctreeEditPacketSender() { delete packet; _preServerPackets.erase(_preServerPackets.begin()); } + _pendingPacketsLock.unlock(); //printf("OctreeEditPacketSender::~OctreeEditPacketSender() [%p] destroyed... \n", this); } @@ -115,6 +117,7 @@ void OctreeEditPacketSender::processPreServerExistsPackets() { assert(serversExist()); // we should only be here if we have jurisdictions // First send out all the single message packets... + _pendingPacketsLock.lock(); while (!_preServerSingleMessagePackets.empty()) { EditPacketBuffer* packet = _preServerSingleMessagePackets.front(); queuePacketToNodes(&packet->_currentBuffer[0], packet->_currentSize); @@ -129,6 +132,7 @@ void OctreeEditPacketSender::processPreServerExistsPackets() { delete packet; _preServerPackets.erase(_preServerPackets.begin()); } + _pendingPacketsLock.unlock(); // if while waiting for the jurisdictions the caller called releaseQueuedMessages() // then we want to honor that request now. @@ -140,8 +144,10 @@ void OctreeEditPacketSender::processPreServerExistsPackets() { void OctreeEditPacketSender::queuePendingPacketToNodes(PacketType type, unsigned char* buffer, ssize_t length) { // If we're asked to save messages while waiting for voxel servers to arrive, then do so... + if (_maxPendingMessages > 0) { EditPacketBuffer* packet = new EditPacketBuffer(type, buffer, length); + _pendingPacketsLock.lock(); _preServerSingleMessagePackets.push_back(packet); // if we've saved MORE than our max, then clear out the oldest packet... int allPendingMessages = _preServerSingleMessagePackets.size() + _preServerPackets.size(); @@ -150,6 +156,7 @@ void OctreeEditPacketSender::queuePendingPacketToNodes(PacketType type, unsigned delete packet; _preServerSingleMessagePackets.erase(_preServerSingleMessagePackets.begin()); } + _pendingPacketsLock.unlock(); } } @@ -197,6 +204,7 @@ void OctreeEditPacketSender::queueOctreeEditMessage(PacketType type, unsigned ch if (!serversExist()) { if (_maxPendingMessages > 0) { EditPacketBuffer* packet = new EditPacketBuffer(type, codeColorBuffer, length); + _pendingPacketsLock.lock(); _preServerPackets.push_back(packet); // if we've saved MORE than out max, then clear out the oldest packet... @@ -206,12 +214,11 @@ void OctreeEditPacketSender::queueOctreeEditMessage(PacketType type, unsigned ch delete packet; _preServerPackets.erase(_preServerPackets.begin()); } - } + _pendingPacketsLock.unlock(); + } return; // bail early } - //qDebug() << "queueOctreeEditMessage() line:" << __LINE__; - // We want to filter out edit messages for servers based on the server's Jurisdiction // But we can't really do that with a packed message, since each edit message could be destined // for a different server... So we need to actually manage multiple queued packets... one @@ -229,18 +236,14 @@ void OctreeEditPacketSender::queueOctreeEditMessage(PacketType type, unsigned ch if ((*_serverJurisdictions).find(nodeUUID) != (*_serverJurisdictions).end()) { const JurisdictionMap& map = (*_serverJurisdictions)[nodeUUID]; isMyJurisdiction = (map.isMyJurisdiction(codeColorBuffer, CHECK_NODE_ONLY) == JurisdictionMap::WITHIN); - //qDebug() << "queueOctreeEditMessage() line:" << __LINE__ << " isMyJurisdiction=" << isMyJurisdiction; } else { isMyJurisdiction = false; - //qDebug() << "queueOctreeEditMessage() line:" << __LINE__; } } if (isMyJurisdiction) { EditPacketBuffer& packetBuffer = _pendingEditPackets[nodeUUID]; packetBuffer._nodeUUID = nodeUUID; - //qDebug() << "queueOctreeEditMessage() line:" << __LINE__; - // If we're switching type, then we send the last one and start over if ((type != packetBuffer._currentType && packetBuffer._currentSize > 0) || (packetBuffer._currentSize + length >= _maxPacketSize)) { @@ -259,11 +262,8 @@ void OctreeEditPacketSender::queueOctreeEditMessage(PacketType type, unsigned ch // fixup the buffer for any clock skew if (node->getClockSkewUsec() != 0) { adjustEditPacketForClockSkew(codeColorBuffer, length, node->getClockSkewUsec()); - //qDebug() << "queueOctreeEditMessage() line:" << __LINE__; } - //qDebug() << "queueOctreeEditMessage() line:" << __LINE__; - memcpy(&packetBuffer._currentBuffer[packetBuffer._currentSize], codeColorBuffer, length); packetBuffer._currentSize += length; } @@ -280,14 +280,12 @@ void OctreeEditPacketSender::releaseQueuedMessages() { } else { for (std::map::iterator i = _pendingEditPackets.begin(); i != _pendingEditPackets.end(); i++) { releaseQueuedPacket(i->second); - //qDebug() << "releaseQueuedMessages() line:" << __LINE__; } } } void OctreeEditPacketSender::releaseQueuedPacket(EditPacketBuffer& packetBuffer) { if (packetBuffer._currentSize > 0 && packetBuffer._currentType != PacketTypeUnknown) { - //qDebug() << "OctreeEditPacketSender::releaseQueuedPacket() line:" << __LINE__; queuePacketToNode(packetBuffer._nodeUUID, &packetBuffer._currentBuffer[0], packetBuffer._currentSize); } packetBuffer._currentSize = 0; diff --git a/libraries/octree/src/OctreeEditPacketSender.h b/libraries/octree/src/OctreeEditPacketSender.h index 3f20ed45a3..75ad02a1c6 100644 --- a/libraries/octree/src/OctreeEditPacketSender.h +++ b/libraries/octree/src/OctreeEditPacketSender.h @@ -105,8 +105,9 @@ protected: // These are packets that are waiting to be processed because we don't yet know if there are servers int _maxPendingMessages; bool _releaseQueuedMessagesPending; - std::vector _preServerPackets; // these will get packed into other larger packets - std::vector _preServerSingleMessagePackets; // these will go out as is + QMutex _pendingPacketsLock; + QVector _preServerPackets; // these will get packed into other larger packets + QVector _preServerSingleMessagePackets; // these will go out as is NodeToJurisdictionMap* _serverJurisdictions; From c8bb539a0c2d08b0cd9078a06210a9bb7e4dbaa0 Mon Sep 17 00:00:00 2001 From: Andrew Meadows Date: Mon, 10 Feb 2014 16:24:10 -0800 Subject: [PATCH 16/17] Theoretical fix for windows build of tests/metavoxels --- tests/metavoxels/CMakeLists.txt | 3 +++ 1 file changed, 3 insertions(+) diff --git a/tests/metavoxels/CMakeLists.txt b/tests/metavoxels/CMakeLists.txt index a21dc2f316..416f398470 100644 --- a/tests/metavoxels/CMakeLists.txt +++ b/tests/metavoxels/CMakeLists.txt @@ -29,4 +29,7 @@ include(${MACRO_DIR}/LinkHifiLibrary.cmake) link_hifi_library(metavoxels ${TARGET_NAME} ${ROOT_DIR}) link_hifi_library(shared ${TARGET_NAME} ${ROOT_DIR}) +IF (WIN32) + target_link_libraries(${TARGET_NAME} Winmm Ws2_32) +ENDIF(WIN32) From b9a5b13f0a95624fa8fa1eff4ab0eb2637444b2c Mon Sep 17 00:00:00 2001 From: Andrzej Kapolka Date: Mon, 10 Feb 2014 16:33:35 -0800 Subject: [PATCH 17/17] This should fix the remaining signed/unsigned warning on Windows. --- libraries/metavoxels/src/DatagramSequencer.cpp | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/libraries/metavoxels/src/DatagramSequencer.cpp b/libraries/metavoxels/src/DatagramSequencer.cpp index 3cbf4a3ac9..fcbe6b5e87 100644 --- a/libraries/metavoxels/src/DatagramSequencer.cpp +++ b/libraries/metavoxels/src/DatagramSequencer.cpp @@ -181,7 +181,7 @@ void DatagramSequencer::receivedDatagram(const QByteArray& datagram) { for (quint32 i = 0; i < highPriorityMessageCount; i++) { QVariant data; _inputStream >> data; - if (i >= _receivedHighPriorityMessages) { + if ((int)i >= _receivedHighPriorityMessages) { handleHighPriorityMessage(data); } }