From 77ca4cf515601a7e8e249dc992f1e3ed690c06de Mon Sep 17 00:00:00 2001 From: David Kelly Date: Thu, 26 Jan 2017 17:38:32 -0700 Subject: [PATCH 01/41] Avoid holding 2 locks at same time If they are always called in same order, you are fine. But just in case, lets instead grab one lock, update the data, release it and then grab the other. No possible deadlock in that, even if the order is changed in one place or another. --- libraries/networking/src/NodeList.cpp | 40 +++++++++++++++++---------- 1 file changed, 26 insertions(+), 14 deletions(-) diff --git a/libraries/networking/src/NodeList.cpp b/libraries/networking/src/NodeList.cpp index 7bfe1d1845..f4a02ad805 100644 --- a/libraries/networking/src/NodeList.cpp +++ b/libraries/networking/src/NodeList.cpp @@ -827,18 +827,26 @@ void NodeList::ignoreNodeBySessionID(const QUuid& nodeID, bool ignoreEnabled) { }); if (ignoreEnabled) { - QReadLocker ignoredSetLocker{ &_ignoredSetLock }; // read lock for insert - QReadLocker personalMutedSetLocker{ &_personalMutedSetLock }; // read lock for insert - // add this nodeID to our set of ignored IDs - _ignoredNodeIDs.insert(nodeID); - // add this nodeID to our set of personal muted IDs - _personalMutedNodeIDs.insert(nodeID); + { + QReadLocker ignoredSetLocker{ &_ignoredSetLock }; // read lock for insert + // add this nodeID to our set of ignored IDs + _ignoredNodeIDs.insert(nodeID); + } + { + QReadLocker personalMutedSetLocker{ &_personalMutedSetLock }; // read lock for insert + // add this nodeID to our set of personal muted IDs + _personalMutedNodeIDs.insert(nodeID); + } emit ignoredNode(nodeID, true); } else { - QWriteLocker ignoredSetLocker{ &_ignoredSetLock }; // write lock for unsafe_erase - QWriteLocker personalMutedSetLocker{ &_personalMutedSetLock }; // write lock for unsafe_erase - _ignoredNodeIDs.unsafe_erase(nodeID); - _personalMutedNodeIDs.unsafe_erase(nodeID); + { + QWriteLocker ignoredSetLocker{ &_ignoredSetLock }; // write lock for unsafe_erase + _ignoredNodeIDs.unsafe_erase(nodeID); + } + { + QWriteLocker personalMutedSetLocker{ &_personalMutedSetLock }; // write lock for unsafe_erase + _personalMutedNodeIDs.unsafe_erase(nodeID); + } emit ignoredNode(nodeID, false); } @@ -850,10 +858,14 @@ void NodeList::ignoreNodeBySessionID(const QUuid& nodeID, bool ignoreEnabled) { void NodeList::removeFromIgnoreMuteSets(const QUuid& nodeID) { // don't remove yourself, or nobody if (!nodeID.isNull() && _sessionUUID != nodeID) { - QWriteLocker ignoredSetLocker{ &_ignoredSetLock }; - QWriteLocker personalMutedSetLocker{ &_personalMutedSetLock }; - _ignoredNodeIDs.unsafe_erase(nodeID); - _personalMutedNodeIDs.unsafe_erase(nodeID); + { + QWriteLocker ignoredSetLocker{ &_ignoredSetLock }; + _ignoredNodeIDs.unsafe_erase(nodeID); + } + { + QWriteLocker personalMutedSetLocker{ &_personalMutedSetLock }; + _personalMutedNodeIDs.unsafe_erase(nodeID); + } } } From 24ca765a393caf9250f03e7288b26de25da110d9 Mon Sep 17 00:00:00 2001 From: Zach Pomerantz Date: Wed, 25 Jan 2017 18:32:42 -0500 Subject: [PATCH 02/41] add hrtf gain getter --- libraries/audio/src/AudioHRTF.h | 1 + 1 file changed, 1 insertion(+) diff --git a/libraries/audio/src/AudioHRTF.h b/libraries/audio/src/AudioHRTF.h index 6a17a2d3cc..a197264994 100644 --- a/libraries/audio/src/AudioHRTF.h +++ b/libraries/audio/src/AudioHRTF.h @@ -48,6 +48,7 @@ public: // HRTF local gain adjustment in amplitude (1.0 == unity) // void setGainAdjustment(float gain) { _gainAdjust = HRTF_GAIN * gain; }; + float getGainAdjustment() { return _gainAdjust; } private: AudioHRTF(const AudioHRTF&) = delete; From 7bcf136dc9cce8a79796bf2c3f359b8601f44b66 Mon Sep 17 00:00:00 2001 From: Zach Pomerantz Date: Fri, 27 Jan 2017 15:26:32 -0500 Subject: [PATCH 03/41] modularize audio throttle gain appx --- .../src/audio/AudioMixerSlave.cpp | 33 ++++++++++++++----- 1 file changed, 24 insertions(+), 9 deletions(-) diff --git a/assignment-client/src/audio/AudioMixerSlave.cpp b/assignment-client/src/audio/AudioMixerSlave.cpp index 4b02ca1567..52dc5c343a 100644 --- a/assignment-client/src/audio/AudioMixerSlave.cpp +++ b/assignment-client/src/audio/AudioMixerSlave.cpp @@ -47,9 +47,11 @@ void sendEnvironmentPacket(const SharedNodePointer& node, AudioMixerClientData& // mix helpers bool shouldIgnoreNode(const SharedNodePointer& listener, const SharedNodePointer& node); -float gainForSource(const AvatarAudioStream& listeningNodeStream, const PositionalAudioStream& streamToAdd, +float approximateGain(const AvatarAudioStream& listeningNodeStream, const PositionalAudioStream& streamToAdd, + const glm::vec3& relativePosition); +float computeGain(const AvatarAudioStream& listeningNodeStream, const PositionalAudioStream& streamToAdd, const glm::vec3& relativePosition, bool isEcho); -float azimuthForSource(const AvatarAudioStream& listeningNodeStream, const PositionalAudioStream& streamToAdd, +float computeAzimuth(const AvatarAudioStream& listeningNodeStream, const PositionalAudioStream& streamToAdd, const glm::vec3& relativePosition); void AudioMixerSlave::configure(ConstIter begin, ConstIter end, unsigned int frame, float throttlingRatio) { @@ -126,9 +128,10 @@ bool AudioMixerSlave::prepareMix(const SharedNodePointer& listener) { AudioMixerClientData&, const QUuid&, const AvatarAudioStream&, const PositionalAudioStream&); auto allStreams = [&](const SharedNodePointer& node, MixFunctor mixFunctor) { AudioMixerClientData* nodeData = static_cast(node->getLinkedData()); + auto nodeID = node->getUUID(); for (auto& streamPair : nodeData->getAudioStreams()) { auto nodeStream = streamPair.second; - (this->*mixFunctor)(*listenerData, node->getUUID(), *listenerAudioStream, *nodeStream); + (this->*mixFunctor)(*listenerData, nodeID, *listenerAudioStream, *nodeStream); } }; @@ -148,13 +151,19 @@ bool AudioMixerSlave::prepareMix(const SharedNodePointer& listener) { allStreams(node, &AudioMixerSlave::mixStream); } else { AudioMixerClientData* nodeData = static_cast(node->getLinkedData()); + auto nodeID = node->getUUID(); // compute the node's max relative volume float nodeVolume; for (auto& streamPair : nodeData->getAudioStreams()) { auto nodeStream = streamPair.second; - float distance = glm::length(nodeStream->getPosition() - listenerAudioStream->getPosition()); - nodeVolume = std::max(nodeStream->getLastPopOutputTrailingLoudness() / distance, nodeVolume); + + // approximate the gain + glm::vec3 relativePosition = nodeStream->getPosition() - listenerAudioStream->getPosition(); + float gain = approximateGain(*listenerAudioStream, *nodeStream, relativePosition); + + auto streamVolume = nodeStream->getLastPopOutputTrailingLoudness() * gain; + nodeVolume = std::max(streamVolume, nodeVolume); } // max-heapify the nodes by relative volume @@ -227,8 +236,8 @@ void AudioMixerSlave::addStream(AudioMixerClientData& listenerNodeData, const QU glm::vec3 relativePosition = streamToAdd.getPosition() - listeningNodeStream.getPosition(); float distance = glm::max(glm::length(relativePosition), EPSILON); - float gain = gainForSource(listeningNodeStream, streamToAdd, relativePosition, isEcho); - float azimuth = isEcho ? 0.0f : azimuthForSource(listeningNodeStream, listeningNodeStream, relativePosition); + float gain = computeGain(listeningNodeStream, streamToAdd, relativePosition, isEcho); + float azimuth = isEcho ? 0.0f : computeAzimuth(listeningNodeStream, listeningNodeStream, relativePosition); static const int HRTF_DATASET_INDEX = 1; if (!streamToAdd.lastPopSucceeded()) { @@ -475,7 +484,13 @@ bool shouldIgnoreNode(const SharedNodePointer& listener, const SharedNodePointer return ignore; } -float gainForSource(const AvatarAudioStream& listeningNodeStream, const PositionalAudioStream& streamToAdd, +float approximateGain(const AvatarAudioStream& listeningNodeStream, const PositionalAudioStream& streamToAdd, + const glm::vec3& relativePosition) { + float distance = glm::length(relativePosition); + return 1.0f / distance; +} + +float computeGain(const AvatarAudioStream& listeningNodeStream, const PositionalAudioStream& streamToAdd, const glm::vec3& relativePosition, bool isEcho) { float gain = 1.0f; @@ -535,7 +550,7 @@ float gainForSource(const AvatarAudioStream& listeningNodeStream, const Position return gain; } -float azimuthForSource(const AvatarAudioStream& listeningNodeStream, const PositionalAudioStream& streamToAdd, +float computeAzimuth(const AvatarAudioStream& listeningNodeStream, const PositionalAudioStream& streamToAdd, const glm::vec3& relativePosition) { glm::quat inverseOrientation = glm::inverse(listeningNodeStream.getOrientation()); From c4190ad4c478183a87a10fd3052c6b67e9329e7d Mon Sep 17 00:00:00 2001 From: Zach Pomerantz Date: Fri, 27 Jan 2017 15:26:49 -0500 Subject: [PATCH 04/41] add hrtf adjustment to throttle gain appx --- assignment-client/src/audio/AudioMixerSlave.cpp | 4 ++++ 1 file changed, 4 insertions(+) diff --git a/assignment-client/src/audio/AudioMixerSlave.cpp b/assignment-client/src/audio/AudioMixerSlave.cpp index 52dc5c343a..abee2aa054 100644 --- a/assignment-client/src/audio/AudioMixerSlave.cpp +++ b/assignment-client/src/audio/AudioMixerSlave.cpp @@ -162,6 +162,10 @@ bool AudioMixerSlave::prepareMix(const SharedNodePointer& listener) { glm::vec3 relativePosition = nodeStream->getPosition() - listenerAudioStream->getPosition(); float gain = approximateGain(*listenerAudioStream, *nodeStream, relativePosition); + // modify by hrtf gain adjustment + auto& hrtf = listenerData->hrtfForStream(nodeID, nodeStream->getStreamIdentifier()); + gain *= hrtf.getGainAdjustment(); + auto streamVolume = nodeStream->getLastPopOutputTrailingLoudness() * gain; nodeVolume = std::max(streamVolume, nodeVolume); } From ca727b792022eb94a955de388a27f7f1557abd71 Mon Sep 17 00:00:00 2001 From: Zach Pomerantz Date: Fri, 27 Jan 2017 15:28:43 -0500 Subject: [PATCH 05/41] inline audio mix helpers --- assignment-client/src/audio/AudioMixerSlave.cpp | 8 ++++---- 1 file changed, 4 insertions(+), 4 deletions(-) diff --git a/assignment-client/src/audio/AudioMixerSlave.cpp b/assignment-client/src/audio/AudioMixerSlave.cpp index abee2aa054..fc17e2f573 100644 --- a/assignment-client/src/audio/AudioMixerSlave.cpp +++ b/assignment-client/src/audio/AudioMixerSlave.cpp @@ -46,12 +46,12 @@ void sendMutePacket(const SharedNodePointer& node, AudioMixerClientData&); void sendEnvironmentPacket(const SharedNodePointer& node, AudioMixerClientData& data); // mix helpers -bool shouldIgnoreNode(const SharedNodePointer& listener, const SharedNodePointer& node); -float approximateGain(const AvatarAudioStream& listeningNodeStream, const PositionalAudioStream& streamToAdd, +inline bool shouldIgnoreNode(const SharedNodePointer& listener, const SharedNodePointer& node); +inline float approximateGain(const AvatarAudioStream& listeningNodeStream, const PositionalAudioStream& streamToAdd, const glm::vec3& relativePosition); -float computeGain(const AvatarAudioStream& listeningNodeStream, const PositionalAudioStream& streamToAdd, +inline float computeGain(const AvatarAudioStream& listeningNodeStream, const PositionalAudioStream& streamToAdd, const glm::vec3& relativePosition, bool isEcho); -float computeAzimuth(const AvatarAudioStream& listeningNodeStream, const PositionalAudioStream& streamToAdd, +inline float computeAzimuth(const AvatarAudioStream& listeningNodeStream, const PositionalAudioStream& streamToAdd, const glm::vec3& relativePosition); void AudioMixerSlave::configure(ConstIter begin, ConstIter end, unsigned int frame, float throttlingRatio) { From fe1847f96521b8eb4f6db6d8fa5e19fc748ba15d Mon Sep 17 00:00:00 2001 From: Zach Pomerantz Date: Fri, 27 Jan 2017 15:46:12 -0500 Subject: [PATCH 06/41] clean up audio-mixer computeGain helper --- .../src/audio/AudioMixerSlave.cpp | 33 ++++++++----------- 1 file changed, 14 insertions(+), 19 deletions(-) diff --git a/assignment-client/src/audio/AudioMixerSlave.cpp b/assignment-client/src/audio/AudioMixerSlave.cpp index fc17e2f573..6dced2d8cb 100644 --- a/assignment-client/src/audio/AudioMixerSlave.cpp +++ b/assignment-client/src/audio/AudioMixerSlave.cpp @@ -498,36 +498,30 @@ float computeGain(const AvatarAudioStream& listeningNodeStream, const Positional const glm::vec3& relativePosition, bool isEcho) { float gain = 1.0f; - float distanceBetween = glm::length(relativePosition); - - if (distanceBetween < EPSILON) { - distanceBetween = EPSILON; - } - + // injector: apply attenuation if (streamToAdd.getType() == PositionalAudioStream::Injector) { gain *= reinterpret_cast(&streamToAdd)->getAttenuationRatio(); } + // avatar: apply fixed off-axis attentuation to make them quieter as they turn away if (!isEcho && (streamToAdd.getType() == PositionalAudioStream::Microphone)) { - // source is another avatar, apply fixed off-axis attenuation to make them quieter as they turn away from listener glm::vec3 rotatedListenerPosition = glm::inverse(streamToAdd.getOrientation()) * relativePosition; - float angleOfDelivery = glm::angle(glm::vec3(0.0f, 0.0f, -1.0f), glm::normalize(rotatedListenerPosition)); const float MAX_OFF_AXIS_ATTENUATION = 0.2f; - const float OFF_AXIS_ATTENUATION_FORMULA_STEP = (1 - MAX_OFF_AXIS_ATTENUATION) / 2.0f; - + const float OFF_AXIS_ATTENUATION_STEP = (1 - MAX_OFF_AXIS_ATTENUATION) / 2.0f; float offAxisCoefficient = MAX_OFF_AXIS_ATTENUATION + - (OFF_AXIS_ATTENUATION_FORMULA_STEP * (angleOfDelivery / PI_OVER_TWO)); + (OFF_AXIS_ATTENUATION_STEP * (angleOfDelivery / PI_OVER_TWO)); - // multiply the current attenuation coefficient by the calculated off axis coefficient gain *= offAxisCoefficient; } - float attenuationPerDoublingInDistance = AudioMixer::getAttenuationPerDoublingInDistance(); - auto& zoneSettings = AudioMixer::getZoneSettings(); auto& audioZones = AudioMixer::getAudioZones(); + auto& zoneSettings = AudioMixer::getZoneSettings(); + + // find distance attenuation coefficient + float attenuationPerDoublingInDistance = AudioMixer::getAttenuationPerDoublingInDistance(); for (int i = 0; i < zoneSettings.length(); ++i) { if (audioZones[zoneSettings[i].source].contains(streamToAdd.getPosition()) && audioZones[zoneSettings[i].listener].contains(listeningNodeStream.getPosition())) { @@ -536,16 +530,17 @@ float computeGain(const AvatarAudioStream& listeningNodeStream, const Positional } } - const float ATTENUATION_BEGINS_AT_DISTANCE = 1.0f; - if (distanceBetween >= ATTENUATION_BEGINS_AT_DISTANCE) { + // compute distance attenuation + const float ATTENUATION_START_DISTANCE = 1.0f; + float distance = std::max(glm::length(relativePosition), EPSILON); + if (distance >= ATTENUATION_START_DISTANCE) { // translate the zone setting to gain per log2(distance) float g = 1.0f - attenuationPerDoublingInDistance; - g = (g < EPSILON) ? EPSILON : g; - g = (g > 1.0f) ? 1.0f : g; + g = glm::clamp(g, EPSILON, 1.0f); // calculate the distance coefficient using the distance to this node - float distanceCoefficient = fastExp2f(fastLog2f(g) * fastLog2f(distanceBetween/ATTENUATION_BEGINS_AT_DISTANCE)); + float distanceCoefficient = fastExp2f(fastLog2f(g) * fastLog2f(distance/ATTENUATION_START_DISTANCE)); // multiply the current attenuation coefficient by the distance coefficient gain *= distanceCoefficient; From 2da4dce8beff82c61979b3d6cc193b6e35115a76 Mon Sep 17 00:00:00 2001 From: Zach Pomerantz Date: Fri, 27 Jan 2017 15:46:25 -0500 Subject: [PATCH 07/41] rm statics from audio-mixer mixing --- assignment-client/src/audio/AudioMixerSlave.cpp | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/assignment-client/src/audio/AudioMixerSlave.cpp b/assignment-client/src/audio/AudioMixerSlave.cpp index 6dced2d8cb..14ab033263 100644 --- a/assignment-client/src/audio/AudioMixerSlave.cpp +++ b/assignment-client/src/audio/AudioMixerSlave.cpp @@ -242,7 +242,7 @@ void AudioMixerSlave::addStream(AudioMixerClientData& listenerNodeData, const QU float distance = glm::max(glm::length(relativePosition), EPSILON); float gain = computeGain(listeningNodeStream, streamToAdd, relativePosition, isEcho); float azimuth = isEcho ? 0.0f : computeAzimuth(listeningNodeStream, listeningNodeStream, relativePosition); - static const int HRTF_DATASET_INDEX = 1; + const int HRTF_DATASET_INDEX = 1; if (!streamToAdd.lastPopSucceeded()) { bool forceSilentBlock = true; @@ -343,7 +343,7 @@ std::unique_ptr createAudioPacket(PacketType type, int size, quint16 s } void sendMixPacket(const SharedNodePointer& node, AudioMixerClientData& data, QByteArray& buffer) { - static const int MIX_PACKET_SIZE = + const int MIX_PACKET_SIZE = sizeof(quint16) + AudioConstants::MAX_CODEC_NAME_LENGTH_ON_WIRE + AudioConstants::NETWORK_FRAME_BYTES_STEREO; quint16 sequence = data.getOutgoingSequenceNumber(); QString codec = data.getCodecName(); @@ -358,7 +358,7 @@ void sendMixPacket(const SharedNodePointer& node, AudioMixerClientData& data, QB } void sendSilentPacket(const SharedNodePointer& node, AudioMixerClientData& data) { - static const int SILENT_PACKET_SIZE = + const int SILENT_PACKET_SIZE = sizeof(quint16) + AudioConstants::MAX_CODEC_NAME_LENGTH_ON_WIRE + sizeof(quint16); quint16 sequence = data.getOutgoingSequenceNumber(); QString codec = data.getCodecName(); From e7e550d78c92b7324e046eb6d0fd79ed60f36c5c Mon Sep 17 00:00:00 2001 From: Zach Pomerantz Date: Fri, 27 Jan 2017 16:18:10 -0500 Subject: [PATCH 08/41] add injector attenuation to audio gain appx --- .../src/audio/AudioMixerSlave.cpp | 28 ++++++++++++++----- 1 file changed, 21 insertions(+), 7 deletions(-) diff --git a/assignment-client/src/audio/AudioMixerSlave.cpp b/assignment-client/src/audio/AudioMixerSlave.cpp index 14ab033263..565652ac2f 100644 --- a/assignment-client/src/audio/AudioMixerSlave.cpp +++ b/assignment-client/src/audio/AudioMixerSlave.cpp @@ -488,10 +488,25 @@ bool shouldIgnoreNode(const SharedNodePointer& listener, const SharedNodePointer return ignore; } +static const float ATTENUATION_START_DISTANCE = 1.0f; + float approximateGain(const AvatarAudioStream& listeningNodeStream, const PositionalAudioStream& streamToAdd, const glm::vec3& relativePosition) { + float gain = 1.0f; + + // injector: apply attenuation + if (streamToAdd.getType() == PositionalAudioStream::Injector) { + gain *= reinterpret_cast(&streamToAdd)->getAttenuationRatio(); + } + + // avatar: skip attenuation - it is too costly to approximate + + // distance attenuation: approximate, ignore zone-specific attenuations + // this is a good approximation for streams further than ATTENUATION_START_DISTANCE + // those streams closer will be amplified; amplifying close streams is acceptable + // when throttling, as close streams are expected to be heard by a user float distance = glm::length(relativePosition); - return 1.0f / distance; + return gain / distance; } float computeGain(const AvatarAudioStream& listeningNodeStream, const PositionalAudioStream& streamToAdd, @@ -501,10 +516,9 @@ float computeGain(const AvatarAudioStream& listeningNodeStream, const Positional // injector: apply attenuation if (streamToAdd.getType() == PositionalAudioStream::Injector) { gain *= reinterpret_cast(&streamToAdd)->getAttenuationRatio(); - } - // avatar: apply fixed off-axis attentuation to make them quieter as they turn away - if (!isEcho && (streamToAdd.getType() == PositionalAudioStream::Microphone)) { + // avatar: apply fixed off-axis attenuation to make them quieter as they turn away + } else if (!isEcho && (streamToAdd.getType() == PositionalAudioStream::Microphone)) { glm::vec3 rotatedListenerPosition = glm::inverse(streamToAdd.getOrientation()) * relativePosition; float angleOfDelivery = glm::angle(glm::vec3(0.0f, 0.0f, -1.0f), glm::normalize(rotatedListenerPosition)); @@ -530,9 +544,9 @@ float computeGain(const AvatarAudioStream& listeningNodeStream, const Positional } } - // compute distance attenuation - const float ATTENUATION_START_DISTANCE = 1.0f; - float distance = std::max(glm::length(relativePosition), EPSILON); + // distance attenuation + float distance = glm::length(relativePosition); + assert(ATTENUATION_START_DISTANCE > EPSILON); if (distance >= ATTENUATION_START_DISTANCE) { // translate the zone setting to gain per log2(distance) From eb00c7ba1115a39d65b48778c59240539f76ca02 Mon Sep 17 00:00:00 2001 From: Zach Pomerantz Date: Mon, 30 Jan 2017 17:37:32 +0000 Subject: [PATCH 09/41] group constants in audio mixer gain computation --- assignment-client/src/audio/AudioMixerSlave.cpp | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/assignment-client/src/audio/AudioMixerSlave.cpp b/assignment-client/src/audio/AudioMixerSlave.cpp index 565652ac2f..1e78507e28 100644 --- a/assignment-client/src/audio/AudioMixerSlave.cpp +++ b/assignment-client/src/audio/AudioMixerSlave.cpp @@ -526,7 +526,7 @@ float computeGain(const AvatarAudioStream& listeningNodeStream, const Positional const float MAX_OFF_AXIS_ATTENUATION = 0.2f; const float OFF_AXIS_ATTENUATION_STEP = (1 - MAX_OFF_AXIS_ATTENUATION) / 2.0f; float offAxisCoefficient = MAX_OFF_AXIS_ATTENUATION + - (OFF_AXIS_ATTENUATION_STEP * (angleOfDelivery / PI_OVER_TWO)); + (angleOfDelivery * (OFF_AXIS_ATTENUATION_STEP / PI_OVER_TWO)); gain *= offAxisCoefficient; } From 069c85c7c250773f1302870d3679e1d1fd187c62 Mon Sep 17 00:00:00 2001 From: David Kelly Date: Fri, 3 Feb 2017 08:52:33 -0700 Subject: [PATCH 10/41] I like this better --- libraries/networking/src/NodeList.cpp | 53 ++++++++++++++------------- 1 file changed, 27 insertions(+), 26 deletions(-) diff --git a/libraries/networking/src/NodeList.cpp b/libraries/networking/src/NodeList.cpp index f4a02ad805..ee8c2e1ecf 100644 --- a/libraries/networking/src/NodeList.cpp +++ b/libraries/networking/src/NodeList.cpp @@ -827,26 +827,28 @@ void NodeList::ignoreNodeBySessionID(const QUuid& nodeID, bool ignoreEnabled) { }); if (ignoreEnabled) { - { - QReadLocker ignoredSetLocker{ &_ignoredSetLock }; // read lock for insert - // add this nodeID to our set of ignored IDs - _ignoredNodeIDs.insert(nodeID); - } - { - QReadLocker personalMutedSetLocker{ &_personalMutedSetLock }; // read lock for insert - // add this nodeID to our set of personal muted IDs - _personalMutedNodeIDs.insert(nodeID); - } + // add this nodeID to our set of ignored IDs + _ignoredSetLock.lockForRead(); + _ignoredNodeIDs.insert(nodeID); + _ignoredSetLock.unlock(); + + // add this nodeID to our set of personal muted IDs + _personalMutedSetLock.lockForRead(); + _personalMutedNodeIDs.insert(nodeID); + _personalMutedSetLock.unlock(); + emit ignoredNode(nodeID, true); } else { - { - QWriteLocker ignoredSetLocker{ &_ignoredSetLock }; // write lock for unsafe_erase - _ignoredNodeIDs.unsafe_erase(nodeID); - } - { - QWriteLocker personalMutedSetLocker{ &_personalMutedSetLock }; // write lock for unsafe_erase - _personalMutedNodeIDs.unsafe_erase(nodeID); - } + // write lock for unsafe_erase + _ignoredSetLock.lockForWrite(); + _ignoredNodeIDs.unsafe_erase(nodeID); + _ignoredSetLock.unlock(); + + // write lock for unsafe_erase + _personalMutedSetLock.lockForWrite(); + _personalMutedNodeIDs.unsafe_erase(nodeID); + _personalMutedSetLock.unlock(); + emit ignoredNode(nodeID, false); } @@ -858,14 +860,13 @@ void NodeList::ignoreNodeBySessionID(const QUuid& nodeID, bool ignoreEnabled) { void NodeList::removeFromIgnoreMuteSets(const QUuid& nodeID) { // don't remove yourself, or nobody if (!nodeID.isNull() && _sessionUUID != nodeID) { - { - QWriteLocker ignoredSetLocker{ &_ignoredSetLock }; - _ignoredNodeIDs.unsafe_erase(nodeID); - } - { - QWriteLocker personalMutedSetLocker{ &_personalMutedSetLock }; - _personalMutedNodeIDs.unsafe_erase(nodeID); - } + _ignoredSetLock.lockForWrite(); + _ignoredNodeIDs.unsafe_erase(nodeID); + _ignoredSetLock.unlock(); + + _personalMutedSetLock.lockForWrite(); + _personalMutedNodeIDs.unsafe_erase(nodeID); + _personalMutedSetLock.unlock(); } } From 8612302f07798cc5cc25cdad738b652d61043635 Mon Sep 17 00:00:00 2001 From: David Kelly Date: Fri, 3 Feb 2017 10:08:46 -0700 Subject: [PATCH 11/41] Revert "I like this better" This reverts commit 069c85c7c250773f1302870d3679e1d1fd187c62. --- libraries/networking/src/NodeList.cpp | 53 +++++++++++++-------------- 1 file changed, 26 insertions(+), 27 deletions(-) diff --git a/libraries/networking/src/NodeList.cpp b/libraries/networking/src/NodeList.cpp index ee8c2e1ecf..f4a02ad805 100644 --- a/libraries/networking/src/NodeList.cpp +++ b/libraries/networking/src/NodeList.cpp @@ -827,28 +827,26 @@ void NodeList::ignoreNodeBySessionID(const QUuid& nodeID, bool ignoreEnabled) { }); if (ignoreEnabled) { - // add this nodeID to our set of ignored IDs - _ignoredSetLock.lockForRead(); - _ignoredNodeIDs.insert(nodeID); - _ignoredSetLock.unlock(); - - // add this nodeID to our set of personal muted IDs - _personalMutedSetLock.lockForRead(); - _personalMutedNodeIDs.insert(nodeID); - _personalMutedSetLock.unlock(); - + { + QReadLocker ignoredSetLocker{ &_ignoredSetLock }; // read lock for insert + // add this nodeID to our set of ignored IDs + _ignoredNodeIDs.insert(nodeID); + } + { + QReadLocker personalMutedSetLocker{ &_personalMutedSetLock }; // read lock for insert + // add this nodeID to our set of personal muted IDs + _personalMutedNodeIDs.insert(nodeID); + } emit ignoredNode(nodeID, true); } else { - // write lock for unsafe_erase - _ignoredSetLock.lockForWrite(); - _ignoredNodeIDs.unsafe_erase(nodeID); - _ignoredSetLock.unlock(); - - // write lock for unsafe_erase - _personalMutedSetLock.lockForWrite(); - _personalMutedNodeIDs.unsafe_erase(nodeID); - _personalMutedSetLock.unlock(); - + { + QWriteLocker ignoredSetLocker{ &_ignoredSetLock }; // write lock for unsafe_erase + _ignoredNodeIDs.unsafe_erase(nodeID); + } + { + QWriteLocker personalMutedSetLocker{ &_personalMutedSetLock }; // write lock for unsafe_erase + _personalMutedNodeIDs.unsafe_erase(nodeID); + } emit ignoredNode(nodeID, false); } @@ -860,13 +858,14 @@ void NodeList::ignoreNodeBySessionID(const QUuid& nodeID, bool ignoreEnabled) { void NodeList::removeFromIgnoreMuteSets(const QUuid& nodeID) { // don't remove yourself, or nobody if (!nodeID.isNull() && _sessionUUID != nodeID) { - _ignoredSetLock.lockForWrite(); - _ignoredNodeIDs.unsafe_erase(nodeID); - _ignoredSetLock.unlock(); - - _personalMutedSetLock.lockForWrite(); - _personalMutedNodeIDs.unsafe_erase(nodeID); - _personalMutedSetLock.unlock(); + { + QWriteLocker ignoredSetLocker{ &_ignoredSetLock }; + _ignoredNodeIDs.unsafe_erase(nodeID); + } + { + QWriteLocker personalMutedSetLocker{ &_personalMutedSetLock }; + _personalMutedNodeIDs.unsafe_erase(nodeID); + } } } From 2a3d0b6b1770a29dcfc8ede5407c7849e4b5ed2d Mon Sep 17 00:00:00 2001 From: Dante Ruiz Date: Fri, 3 Feb 2017 19:13:57 +0000 Subject: [PATCH 12/41] enforce more button clicking logic --- scripts/system/controllers/handControllerGrab.js | 1 - 1 file changed, 1 deletion(-) diff --git a/scripts/system/controllers/handControllerGrab.js b/scripts/system/controllers/handControllerGrab.js index d2c9fdc05a..109d5171c4 100644 --- a/scripts/system/controllers/handControllerGrab.js +++ b/scripts/system/controllers/handControllerGrab.js @@ -2882,7 +2882,6 @@ function MyController(hand) { this.touchingEnterTimer += dt; if (this.state == STATE_OVERLAY_STYLUS_TOUCHING && this.triggerSmoothedSqueezed()) { - this.setState(STATE_OFF, "trigger squeezed"); return; } From 37de4aefa6a31e892ccc5e461aa815de76d6f4f0 Mon Sep 17 00:00:00 2001 From: Seth Alves Date: Fri, 3 Feb 2017 14:05:23 -0800 Subject: [PATCH 13/41] don't send AVATAR_SELF_ID over wire --- libraries/entities/src/EntityEditPacketSender.cpp | 13 ++++++++++++- 1 file changed, 12 insertions(+), 1 deletion(-) diff --git a/libraries/entities/src/EntityEditPacketSender.cpp b/libraries/entities/src/EntityEditPacketSender.cpp index e05db07d0d..97648c7079 100644 --- a/libraries/entities/src/EntityEditPacketSender.cpp +++ b/libraries/entities/src/EntityEditPacketSender.cpp @@ -100,7 +100,18 @@ void EntityEditPacketSender::queueEditEntityMessage(PacketType type, QByteArray bufferOut(NLPacket::maxPayloadSize(type), 0); - if (EntityItemProperties::encodeEntityEditPacket(type, entityItemID, properties, bufferOut)) { + bool success; + if (properties.parentIDChanged() && properties.getParentID() == AVATAR_SELF_ID) { + EntityItemProperties propertiesCopy = properties; + auto nodeList = DependencyManager::get(); + const QUuid myNodeID = nodeList->getSessionUUID(); + propertiesCopy.setParentID(myNodeID); + success = EntityItemProperties::encodeEntityEditPacket(type, entityItemID, propertiesCopy, bufferOut); + } else { + success = EntityItemProperties::encodeEntityEditPacket(type, entityItemID, properties, bufferOut); + } + + if (success) { #ifdef WANT_DEBUG qCDebug(entities) << "calling queueOctreeEditMessage()..."; qCDebug(entities) << " id:" << entityItemID; From 174c674fd31195777fda03a631e11c4729c6e9e8 Mon Sep 17 00:00:00 2001 From: Seth Alves Date: Fri, 3 Feb 2017 15:50:59 -0800 Subject: [PATCH 14/41] don't call stepKinematicMotion on kinematic entities --- libraries/physics/src/EntityMotionState.cpp | 9 +++------ 1 file changed, 3 insertions(+), 6 deletions(-) diff --git a/libraries/physics/src/EntityMotionState.cpp b/libraries/physics/src/EntityMotionState.cpp index e9891020b3..c90ca5a1a9 100644 --- a/libraries/physics/src/EntityMotionState.cpp +++ b/libraries/physics/src/EntityMotionState.cpp @@ -199,15 +199,12 @@ void EntityMotionState::getWorldTransform(btTransform& worldTrans) const { return; } assert(entityTreeIsLocked()); - if (_motionType == MOTION_TYPE_KINEMATIC) { + if (_motionType == MOTION_TYPE_KINEMATIC && !_entity->hasAncestorOfType(NestableType::Avatar)) { BT_PROFILE("kinematicIntegration"); // This is physical kinematic motion which steps strictly by the subframe count // of the physics simulation and uses full gravity for acceleration. - if (_entity->hasAncestorOfType(NestableType::Avatar)) { - _entity->setAcceleration(glm::vec3(0.0f)); - } else { - _entity->setAcceleration(_entity->getGravity()); - } + _entity->setAcceleration(_entity->getGravity()); + uint32_t thisStep = ObjectMotionState::getWorldSimulationStep(); float dt = (thisStep - _lastKinematicStep) * PHYSICS_ENGINE_FIXED_SUBSTEP; _entity->stepKinematicMotion(dt); From c7200d2d3a2571d5fe6894e72bf377d5554c1111 Mon Sep 17 00:00:00 2001 From: Seth Alves Date: Fri, 3 Feb 2017 16:14:15 -0800 Subject: [PATCH 15/41] make hasAncestorOfType understand AVATAR_SELF_ID --- libraries/physics/src/EntityMotionState.cpp | 9 ++++++--- libraries/shared/src/SpatiallyNestable.cpp | 7 +++++++ 2 files changed, 13 insertions(+), 3 deletions(-) diff --git a/libraries/physics/src/EntityMotionState.cpp b/libraries/physics/src/EntityMotionState.cpp index c90ca5a1a9..e9891020b3 100644 --- a/libraries/physics/src/EntityMotionState.cpp +++ b/libraries/physics/src/EntityMotionState.cpp @@ -199,12 +199,15 @@ void EntityMotionState::getWorldTransform(btTransform& worldTrans) const { return; } assert(entityTreeIsLocked()); - if (_motionType == MOTION_TYPE_KINEMATIC && !_entity->hasAncestorOfType(NestableType::Avatar)) { + if (_motionType == MOTION_TYPE_KINEMATIC) { BT_PROFILE("kinematicIntegration"); // This is physical kinematic motion which steps strictly by the subframe count // of the physics simulation and uses full gravity for acceleration. - _entity->setAcceleration(_entity->getGravity()); - + if (_entity->hasAncestorOfType(NestableType::Avatar)) { + _entity->setAcceleration(glm::vec3(0.0f)); + } else { + _entity->setAcceleration(_entity->getGravity()); + } uint32_t thisStep = ObjectMotionState::getWorldSimulationStep(); float dt = (thisStep - _lastKinematicStep) * PHYSICS_ENGINE_FIXED_SUBSTEP; _entity->stepKinematicMotion(dt); diff --git a/libraries/shared/src/SpatiallyNestable.cpp b/libraries/shared/src/SpatiallyNestable.cpp index 35e574bf06..f071bda71f 100644 --- a/libraries/shared/src/SpatiallyNestable.cpp +++ b/libraries/shared/src/SpatiallyNestable.cpp @@ -1034,6 +1034,13 @@ AACube SpatiallyNestable::getQueryAACube() const { bool SpatiallyNestable::hasAncestorOfType(NestableType nestableType) const { bool success; + if (nestableType == NestableType::Avatar) { + QUuid parentID = getParentID(); + if (parentID == AVATAR_SELF_ID) { + return true; + } + } + SpatiallyNestablePointer parent = getParentPointer(success); if (!success || !parent) { return false; From 22f9fc70617280714bf9ed26fad3b0ad1ea90262 Mon Sep 17 00:00:00 2001 From: Seth Alves Date: Fri, 3 Feb 2017 16:36:06 -0800 Subject: [PATCH 16/41] don't call stepKinematicMotion on kinematic entities when they are children of avatars --- libraries/physics/src/EntityMotionState.cpp | 9 +++------ 1 file changed, 3 insertions(+), 6 deletions(-) diff --git a/libraries/physics/src/EntityMotionState.cpp b/libraries/physics/src/EntityMotionState.cpp index e9891020b3..c90ca5a1a9 100644 --- a/libraries/physics/src/EntityMotionState.cpp +++ b/libraries/physics/src/EntityMotionState.cpp @@ -199,15 +199,12 @@ void EntityMotionState::getWorldTransform(btTransform& worldTrans) const { return; } assert(entityTreeIsLocked()); - if (_motionType == MOTION_TYPE_KINEMATIC) { + if (_motionType == MOTION_TYPE_KINEMATIC && !_entity->hasAncestorOfType(NestableType::Avatar)) { BT_PROFILE("kinematicIntegration"); // This is physical kinematic motion which steps strictly by the subframe count // of the physics simulation and uses full gravity for acceleration. - if (_entity->hasAncestorOfType(NestableType::Avatar)) { - _entity->setAcceleration(glm::vec3(0.0f)); - } else { - _entity->setAcceleration(_entity->getGravity()); - } + _entity->setAcceleration(_entity->getGravity()); + uint32_t thisStep = ObjectMotionState::getWorldSimulationStep(); float dt = (thisStep - _lastKinematicStep) * PHYSICS_ENGINE_FIXED_SUBSTEP; _entity->stepKinematicMotion(dt); From ad83f20199b2a75177fdc2916468c241e7a15043 Mon Sep 17 00:00:00 2001 From: Seth Alves Date: Fri, 3 Feb 2017 16:43:36 -0800 Subject: [PATCH 17/41] make findAncestorOfType understand AVATAR_SELF_ID --- libraries/shared/src/SpatiallyNestable.cpp | 8 ++++++++ 1 file changed, 8 insertions(+) diff --git a/libraries/shared/src/SpatiallyNestable.cpp b/libraries/shared/src/SpatiallyNestable.cpp index f071bda71f..ddc3f416e0 100644 --- a/libraries/shared/src/SpatiallyNestable.cpp +++ b/libraries/shared/src/SpatiallyNestable.cpp @@ -1055,6 +1055,14 @@ bool SpatiallyNestable::hasAncestorOfType(NestableType nestableType) const { const QUuid SpatiallyNestable::findAncestorOfType(NestableType nestableType) const { bool success; + + if (nestableType == NestableType::Avatar) { + QUuid parentID = getParentID(); + if (parentID == AVATAR_SELF_ID) { + return AVATAR_SELF_ID; // TODO -- can we put nodeID here? + } + } + SpatiallyNestablePointer parent = getParentPointer(success); if (!success || !parent) { return QUuid(); From 81bf5ee6554317c657c50df57dce1df4e73da77e Mon Sep 17 00:00:00 2001 From: Seth Alves Date: Fri, 3 Feb 2017 16:47:03 -0800 Subject: [PATCH 18/41] another AVATAR_SELF_ID adjustment --- libraries/entities/src/EntityItem.cpp | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/libraries/entities/src/EntityItem.cpp b/libraries/entities/src/EntityItem.cpp index 233ce7d88e..2503687dce 100644 --- a/libraries/entities/src/EntityItem.cpp +++ b/libraries/entities/src/EntityItem.cpp @@ -1828,7 +1828,8 @@ void EntityItem::computeCollisionGroupAndFinalMask(int16_t& group, int16_t& mask // "bootstrapping" problem where you can shoot yourself across the room by grabbing something // and holding it against your own avatar. QUuid ancestorID = findAncestorOfType(NestableType::Avatar); - if (!ancestorID.isNull() && ancestorID == Physics::getSessionUUID()) { + if (!ancestorID.isNull() && + (ancestorID == Physics::getSessionUUID() || ancestorID == AVATAR_SELF_ID)) { userMask &= ~USER_COLLISION_GROUP_MY_AVATAR; } } From 4acb99cd4fdb4310dd539bfaf7cc6aa89017807b Mon Sep 17 00:00:00 2001 From: Ken Cooke Date: Mon, 6 Feb 2017 09:29:39 -0800 Subject: [PATCH 19/41] Use interpolated audio (codec packet-loss concealment) or zero samples (if no codec) when audio packets are lost. This audio is still processed by the audio pipeline to avoid clicks/pops. --- libraries/audio/src/InboundAudioStream.cpp | 21 ++++++++++++++---- libraries/audio/src/InboundAudioStream.h | 3 +++ .../audio/src/MixedProcessedAudioStream.cpp | 22 +++++++++++++++++++ .../audio/src/MixedProcessedAudioStream.h | 1 + libraries/plugins/src/plugins/CodecPlugin.h | 3 +-- plugins/hifiCodec/src/HiFiCodec.cpp | 8 +++---- plugins/pcmCodec/src/PCMCodecManager.h | 9 ++++++-- 7 files changed, 54 insertions(+), 13 deletions(-) diff --git a/libraries/audio/src/InboundAudioStream.cpp b/libraries/audio/src/InboundAudioStream.cpp index 36a6079546..6f61c59cbd 100644 --- a/libraries/audio/src/InboundAudioStream.cpp +++ b/libraries/audio/src/InboundAudioStream.cpp @@ -136,7 +136,8 @@ int InboundAudioStream::parseData(ReceivedMessage& message) { // NOTE: we assume that each dropped packet contains the same number of samples // as the packet we just received. int packetsDropped = arrivalInfo._seqDiffFromExpected; - writeFramesForDroppedPackets(packetsDropped * networkFrames); + //writeFramesForDroppedPackets(packetsDropped * networkFrames); + lostAudioData(packetsDropped); // fall through to OnTime case } @@ -208,6 +209,21 @@ int InboundAudioStream::parseStreamProperties(PacketType type, const QByteArray& } } +int InboundAudioStream::lostAudioData(int numPackets) { + QByteArray decodedBuffer; + + while (numPackets--) { + if (_decoder) { + _decoder->lostFrame(decodedBuffer); + } else { + decodedBuffer.resize(AudioConstants::NETWORK_FRAME_BYTES_STEREO); + memset(decodedBuffer.data(), 0, decodedBuffer.size()); + } + _ringBuffer.writeData(decodedBuffer.data(), decodedBuffer.size()); + } + return 0; +} + int InboundAudioStream::parseAudioData(PacketType type, const QByteArray& packetAfterStreamProperties) { QByteArray decodedBuffer; if (_decoder) { @@ -220,9 +236,6 @@ int InboundAudioStream::parseAudioData(PacketType type, const QByteArray& packet } int InboundAudioStream::writeDroppableSilentFrames(int silentFrames) { - if (_decoder) { - _decoder->trackLostFrames(silentFrames); - } // calculate how many silent frames we should drop. int silentSamples = silentFrames * _numChannels; diff --git a/libraries/audio/src/InboundAudioStream.h b/libraries/audio/src/InboundAudioStream.h index f7b79ab136..b349016452 100644 --- a/libraries/audio/src/InboundAudioStream.h +++ b/libraries/audio/src/InboundAudioStream.h @@ -134,6 +134,9 @@ protected: /// default implementation assumes packet contains raw audio samples after stream properties virtual int parseAudioData(PacketType type, const QByteArray& packetAfterStreamProperties); + /// produces audio data for lost network packets. + virtual int lostAudioData(int numPackets); + /// writes silent frames to the buffer that may be dropped to reduce latency caused by the buffer virtual int writeDroppableSilentFrames(int silentFrames); diff --git a/libraries/audio/src/MixedProcessedAudioStream.cpp b/libraries/audio/src/MixedProcessedAudioStream.cpp index 671d3a9d60..c3170c3259 100644 --- a/libraries/audio/src/MixedProcessedAudioStream.cpp +++ b/libraries/audio/src/MixedProcessedAudioStream.cpp @@ -38,6 +38,28 @@ int MixedProcessedAudioStream::writeLastFrameRepeatedWithFade(int frames) { return deviceFramesWritten; } +int MixedProcessedAudioStream::lostAudioData(int numPackets) { + QByteArray decodedBuffer; + QByteArray outputBuffer; + + while (numPackets--) { + if (_decoder) { + _decoder->lostFrame(decodedBuffer); + } else { + decodedBuffer.resize(AudioConstants::NETWORK_FRAME_BYTES_STEREO); + memset(decodedBuffer.data(), 0, decodedBuffer.size()); + } + + emit addedStereoSamples(decodedBuffer); + + emit processSamples(decodedBuffer, outputBuffer); + + _ringBuffer.writeData(outputBuffer.data(), outputBuffer.size()); + qCDebug(audiostream, "Wrote %d samples to buffer (%d available)", outputBuffer.size() / (int)sizeof(int16_t), getSamplesAvailable()); + } + return 0; +} + int MixedProcessedAudioStream::parseAudioData(PacketType type, const QByteArray& packetAfterStreamProperties) { QByteArray decodedBuffer; if (_decoder) { diff --git a/libraries/audio/src/MixedProcessedAudioStream.h b/libraries/audio/src/MixedProcessedAudioStream.h index d536163d2d..83c4cea635 100644 --- a/libraries/audio/src/MixedProcessedAudioStream.h +++ b/libraries/audio/src/MixedProcessedAudioStream.h @@ -36,6 +36,7 @@ protected: int writeDroppableSilentFrames(int silentFrames) override; int writeLastFrameRepeatedWithFade(int frames) override; int parseAudioData(PacketType type, const QByteArray& packetAfterStreamProperties) override; + int lostAudioData(int numPackets) override; private: int networkToDeviceFrames(int networkFrames); diff --git a/libraries/plugins/src/plugins/CodecPlugin.h b/libraries/plugins/src/plugins/CodecPlugin.h index 404f05e860..cb5b857be8 100644 --- a/libraries/plugins/src/plugins/CodecPlugin.h +++ b/libraries/plugins/src/plugins/CodecPlugin.h @@ -23,8 +23,7 @@ public: virtual ~Decoder() { } virtual void decode(const QByteArray& encodedBuffer, QByteArray& decodedBuffer) = 0; - // numFrames - number of samples (mono) or sample-pairs (stereo) - virtual void trackLostFrames(int numFrames) = 0; + virtual void lostFrame(QByteArray& decodedBuffer) = 0; }; class CodecPlugin : public Plugin { diff --git a/plugins/hifiCodec/src/HiFiCodec.cpp b/plugins/hifiCodec/src/HiFiCodec.cpp index 77c369dcae..2c7151fe59 100644 --- a/plugins/hifiCodec/src/HiFiCodec.cpp +++ b/plugins/hifiCodec/src/HiFiCodec.cpp @@ -65,12 +65,10 @@ public: AudioDecoder::process((const int16_t*)encodedBuffer.constData(), (int16_t*)decodedBuffer.data(), AudioConstants::NETWORK_FRAME_SAMPLES_PER_CHANNEL, true); } - virtual void trackLostFrames(int numFrames) override { - QByteArray encodedBuffer; - QByteArray decodedBuffer; + virtual void lostFrame(QByteArray& decodedBuffer) override { decodedBuffer.resize(_decodedSize); - // NOTE: we don't actually use the results of this decode, we just do it to keep the state of the codec clean - AudioDecoder::process((const int16_t*)encodedBuffer.constData(), (int16_t*)decodedBuffer.data(), AudioConstants::NETWORK_FRAME_SAMPLES_PER_CHANNEL, false); + // this performs packet loss interpolation + AudioDecoder::process(nullptr, (int16_t*)decodedBuffer.data(), AudioConstants::NETWORK_FRAME_SAMPLES_PER_CHANNEL, false); } private: int _decodedSize; diff --git a/plugins/pcmCodec/src/PCMCodecManager.h b/plugins/pcmCodec/src/PCMCodecManager.h index d58a219fef..608e9a1556 100644 --- a/plugins/pcmCodec/src/PCMCodecManager.h +++ b/plugins/pcmCodec/src/PCMCodecManager.h @@ -38,11 +38,14 @@ public: virtual void encode(const QByteArray& decodedBuffer, QByteArray& encodedBuffer) override { encodedBuffer = decodedBuffer; } + virtual void decode(const QByteArray& encodedBuffer, QByteArray& decodedBuffer) override { decodedBuffer = encodedBuffer; } - virtual void trackLostFrames(int numFrames) override { } + virtual void lostFrame(QByteArray& decodedBuffer) override { + memset(decodedBuffer.data(), 0, decodedBuffer.size()); + } private: static const char* NAME; @@ -77,7 +80,9 @@ public: decodedBuffer = qUncompress(encodedBuffer); } - virtual void trackLostFrames(int numFrames) override { } + virtual void lostFrame(QByteArray& decodedBuffer) override { + memset(decodedBuffer.data(), 0, decodedBuffer.size()); + } private: static const char* NAME; From ab954412c13b88420f8c31259e627dc909e022ff Mon Sep 17 00:00:00 2001 From: Seth Alves Date: Mon, 6 Feb 2017 09:50:48 -0800 Subject: [PATCH 20/41] if an entity has an avatar parent but ends up in EntityMotionState::setWorldTransform, ignore the update from bullet --- libraries/physics/src/EntityMotionState.cpp | 3 +++ 1 file changed, 3 insertions(+) diff --git a/libraries/physics/src/EntityMotionState.cpp b/libraries/physics/src/EntityMotionState.cpp index c90ca5a1a9..d652e78f6a 100644 --- a/libraries/physics/src/EntityMotionState.cpp +++ b/libraries/physics/src/EntityMotionState.cpp @@ -230,6 +230,9 @@ void EntityMotionState::setWorldTransform(const btTransform& worldTrans) { assert(entityTreeIsLocked()); measureBodyAcceleration(); bool positionSuccess; + if (_entity->hasAncestorOfType(NestableType::Avatar)) { + return; + } _entity->setPosition(bulletToGLM(worldTrans.getOrigin()) + ObjectMotionState::getWorldOffset(), positionSuccess, false); if (!positionSuccess) { static QString repeatedMessage = From 8033f932a65734dc76a5cadbf0cd7f6c00ce7030 Mon Sep 17 00:00:00 2001 From: Ken Cooke Date: Mon, 6 Feb 2017 10:16:25 -0800 Subject: [PATCH 21/41] Simulate 1% random packet-loss, for debug/test --- libraries/audio/src/InboundAudioStream.cpp | 11 ++++++++++- 1 file changed, 10 insertions(+), 1 deletion(-) diff --git a/libraries/audio/src/InboundAudioStream.cpp b/libraries/audio/src/InboundAudioStream.cpp index 6f61c59cbd..57e3f4eb36 100644 --- a/libraries/audio/src/InboundAudioStream.cpp +++ b/libraries/audio/src/InboundAudioStream.cpp @@ -129,14 +129,23 @@ int InboundAudioStream::parseData(ReceivedMessage& message) { int propertyBytes = parseStreamProperties(message.getType(), message.readWithoutCopy(message.getBytesLeftToRead()), networkFrames); message.seek(prePropertyPosition + propertyBytes); + // simulate 1% packetloss + if (rand() < (RAND_MAX/100)) { + arrivalInfo._status = SequenceNumberStats::Unreasonable; + qDebug(audio) << "Simulated a lost packet..."; + } + // handle this packet based on its arrival status. switch (arrivalInfo._status) { + case SequenceNumberStats::Unreasonable: { + lostAudioData(1); + break; + } case SequenceNumberStats::Early: { // Packet is early; write droppable silent samples for each of the skipped packets. // NOTE: we assume that each dropped packet contains the same number of samples // as the packet we just received. int packetsDropped = arrivalInfo._seqDiffFromExpected; - //writeFramesForDroppedPackets(packetsDropped * networkFrames); lostAudioData(packetsDropped); // fall through to OnTime case From 49605d52e058249e7ccd2483ec70683045f044a8 Mon Sep 17 00:00:00 2001 From: Ken Cooke Date: Mon, 6 Feb 2017 10:42:32 -0800 Subject: [PATCH 22/41] Remove debug code --- libraries/audio/src/InboundAudioStream.cpp | 6 ------ 1 file changed, 6 deletions(-) diff --git a/libraries/audio/src/InboundAudioStream.cpp b/libraries/audio/src/InboundAudioStream.cpp index 57e3f4eb36..d23579430e 100644 --- a/libraries/audio/src/InboundAudioStream.cpp +++ b/libraries/audio/src/InboundAudioStream.cpp @@ -129,12 +129,6 @@ int InboundAudioStream::parseData(ReceivedMessage& message) { int propertyBytes = parseStreamProperties(message.getType(), message.readWithoutCopy(message.getBytesLeftToRead()), networkFrames); message.seek(prePropertyPosition + propertyBytes); - // simulate 1% packetloss - if (rand() < (RAND_MAX/100)) { - arrivalInfo._status = SequenceNumberStats::Unreasonable; - qDebug(audio) << "Simulated a lost packet..."; - } - // handle this packet based on its arrival status. switch (arrivalInfo._status) { case SequenceNumberStats::Unreasonable: { From 4bbd823a5a2831ea01246e3c646cb4b22b534b0c Mon Sep 17 00:00:00 2001 From: Stephen Birarda Date: Mon, 6 Feb 2017 10:50:59 -0800 Subject: [PATCH 23/41] remove open hours from domain settings --- .../resources/describe-settings.json | 212 ------------------ domain-server/src/DomainMetadata.cpp | 67 +----- domain-server/src/DomainMetadata.h | 9 - .../src/DomainServerSettingsManager.cpp | 42 ---- .../src/DomainServerSettingsManager.h | 2 - 5 files changed, 2 insertions(+), 330 deletions(-) diff --git a/domain-server/resources/describe-settings.json b/domain-server/resources/describe-settings.json index 20d2711743..d27d068f84 100644 --- a/domain-server/resources/describe-settings.json +++ b/domain-server/resources/describe-settings.json @@ -141,218 +141,6 @@ "can_set": true } ] - }, - { - "label": "Operating Hours", - "help": "\"Open\" domains can be searched using their operating hours. Hours are entered in the local timezone, selected below.", - - "name": "weekday_hours", - "caption": "Weekday Hours (Monday-Friday)", - "type": "table", - "can_add_new_rows": false, - "columns": [ - { - "name": "open", - "label": "Opening Time", - "type": "time", - "default": "00:00", - "editable": true - }, - { - "name": "close", - "label": "Closing Time", - "type": "time", - "default": "23:59", - "editable": true - } - ] - }, - { - "name": "weekend_hours", - "label": "Weekend Hours (Saturday/Sunday)", - "type": "table", - "can_add_new_rows": false, - "columns": [ - { - "name": "open", - "label": "Opening Time", - "type": "time", - "default": "00:00", - "editable": true - }, - { - "name": "close", - "label": "Closing Time", - "type": "time", - "default": "23:59", - "editable": true - } - ] - }, - { - "label": "Time Zone", - "name": "utc_offset", - "caption": "Time Zone", - "help": "This server's time zone. Used to define your server's operating hours.", - "type": "select", - "options": [ - { - "value": "-12", - "label": "UTC-12:00" - }, - { - "value": "-11", - "label": "UTC-11:00" - }, - { - "value": "-10", - "label": "UTC-10:00" - }, - { - "value": "-9.5", - "label": "UTC-09:30" - }, - { - "value": "-9", - "label": "UTC-09:00" - }, - { - "value": "-8", - "label": "UTC-08:00" - }, - { - "value": "-7", - "label": "UTC-07:00" - }, - { - "value": "-6", - "label": "UTC-06:00" - }, - { - "value": "-5", - "label": "UTC-05:00" - }, - { - "value": "-4", - "label": "UTC-04:00" - }, - { - "value": "-3.5", - "label": "UTC-03:30" - }, - { - "value": "-3", - "label": "UTC-03:00" - }, - { - "value": "-2", - "label": "UTC-02:00" - }, - { - "value": "-1", - "label": "UTC-01:00" - }, - { - "value": "", - "label": "UTC±00:00" - }, - { - "value": "1", - "label": "UTC+01:00" - }, - { - "value": "2", - "label": "UTC+02:00" - }, - { - "value": "3", - "label": "UTC+03:00" - }, - { - "value": "3.5", - "label": "UTC+03:30" - }, - { - "value": "4", - "label": "UTC+04:00" - }, - { - "value": "4.5", - "label": "UTC+04:30" - }, - { - "value": "5", - "label": "UTC+05:00" - }, - { - "value": "5.5", - "label": "UTC+05:30" - }, - { - "value": "5.75", - "label": "UTC+05:45" - }, - { - "value": "6", - "label": "UTC+06:00" - }, - { - "value": "6.5", - "label": "UTC+06:30" - }, - { - "value": "7", - "label": "UTC+07:00" - }, - { - "value": "8", - "label": "UTC+08:00" - }, - { - "value": "8.5", - "label": "UTC+08:30" - }, - { - "value": "8.75", - "label": "UTC+08:45" - }, - { - "value": "9", - "label": "UTC+09:00" - }, - { - "value": "9.5", - "label": "UTC+09:30" - }, - { - "value": "10", - "label": "UTC+10:00" - }, - { - "value": "10.5", - "label": "UTC+10:30" - }, - { - "value": "11", - "label": "UTC+11:00" - }, - { - "value": "12", - "label": "UTC+12:00" - }, - { - "value": "12.75", - "label": "UTC+12:45" - }, - { - "value": "13", - "label": "UTC+13:00" - }, - { - "value": "14", - "label": "UTC+14:00" - } - ] } ] }, diff --git a/domain-server/src/DomainMetadata.cpp b/domain-server/src/DomainMetadata.cpp index d614b1bbd3..c19cefa397 100644 --- a/domain-server/src/DomainMetadata.cpp +++ b/domain-server/src/DomainMetadata.cpp @@ -35,12 +35,6 @@ const QString DomainMetadata::Descriptors::RESTRICTION = "restriction"; // parse const QString DomainMetadata::Descriptors::MATURITY = "maturity"; const QString DomainMetadata::Descriptors::HOSTS = "hosts"; const QString DomainMetadata::Descriptors::TAGS = "tags"; -const QString DomainMetadata::Descriptors::HOURS = "hours"; -const QString DomainMetadata::Descriptors::Hours::WEEKDAY = "weekday"; -const QString DomainMetadata::Descriptors::Hours::WEEKEND = "weekend"; -const QString DomainMetadata::Descriptors::Hours::UTC_OFFSET = "utc_offset"; -const QString DomainMetadata::Descriptors::Hours::OPEN = "open"; -const QString DomainMetadata::Descriptors::Hours::CLOSE = "close"; // descriptors metadata will appear as (JSON): // { "description": String, // capped description // "capacity": Number, @@ -48,11 +42,6 @@ const QString DomainMetadata::Descriptors::Hours::CLOSE = "close"; // "maturity": String, // enum corresponding to ESRB ratings // "hosts": [ String ], // capped list of usernames // "tags": [ String ], // capped list of tags -// "hours": { -// "utc_offset": Number, -// "weekday": [ [ Time, Time ] ], -// "weekend": [ [ Time, Time ] ], -// } // } // metadata will appear as (JSON): @@ -60,52 +49,10 @@ const QString DomainMetadata::Descriptors::Hours::CLOSE = "close"; // // it is meant to be sent to and consumed by an external API -// merge delta into target -// target should be of the form [ OpenTime, CloseTime ], -// delta should be of the form [ { open: Time, close: Time } ] -void parseHours(QVariant delta, QVariant& target) { - using Hours = DomainMetadata::Descriptors::Hours; - static const QVariantList DEFAULT_HOURS{ - { QVariantList{ "00:00", "23:59" } } - }; - target.setValue(DEFAULT_HOURS); - - if (!delta.canConvert()) { - return; - } - - auto& deltaList = *static_cast(delta.data()); - if (deltaList.isEmpty()) { - return; - } - - auto& deltaHours = *static_cast(deltaList.first().data()); - auto open = deltaHours.find(Hours::OPEN); - auto close = deltaHours.find(Hours::CLOSE); - if (open == deltaHours.end() || close == deltaHours.end()) { - return; - } - - // merge delta into new hours - static const int OPEN_INDEX = 0; - static const int CLOSE_INDEX = 1; - auto& hours = *static_cast(static_cast(target.data())->first().data()); - hours[OPEN_INDEX] = open.value(); - hours[CLOSE_INDEX] = close.value(); - - assert(hours[OPEN_INDEX].canConvert()); - assert(hours[CLOSE_INDEX].canConvert()); -} - DomainMetadata::DomainMetadata(QObject* domainServer) : QObject(domainServer) { - // set up the structure necessary for casting during parsing (see parseHours, esp.) + // set up the structure necessary for casting during parsing _metadata[USERS] = QVariantMap {}; - _metadata[DESCRIPTORS] = QVariantMap { { - Descriptors::HOURS, QVariantMap { - { Descriptors::Hours::WEEKDAY, QVariant{} }, - { Descriptors::Hours::WEEKEND, QVariant{} } - } - } }; + _metadata[DESCRIPTORS] = QVariantMap {}; assert(dynamic_cast(domainServer)); DomainServer* server = static_cast(domainServer); @@ -154,16 +101,6 @@ void DomainMetadata::descriptorsChanged() { unsigned int capacity = capacityVariant ? capacityVariant->toUInt() : 0; state[Descriptors::CAPACITY] = capacity; - // parse operating hours - static const QString WEEKDAY_HOURS = "weekday_hours"; - static const QString WEEKEND_HOURS = "weekend_hours"; - static const QString UTC_OFFSET = "utc_offset"; - assert(state[Descriptors::HOURS].canConvert()); - auto& hours = *static_cast(state[Descriptors::HOURS].data()); - hours[Descriptors::Hours::UTC_OFFSET] = descriptors.take(UTC_OFFSET); - parseHours(descriptors[WEEKDAY_HOURS], hours[Descriptors::Hours::WEEKDAY]); - parseHours(descriptors[WEEKEND_HOURS], hours[Descriptors::Hours::WEEKEND]); - #if DEV_BUILD || PR_BUILD qDebug() << "Domain metadata descriptors set:" << QJsonObject::fromVariantMap(_metadata[DESCRIPTORS].toMap()); #endif diff --git a/domain-server/src/DomainMetadata.h b/domain-server/src/DomainMetadata.h index 41f3a60832..ed4e324464 100644 --- a/domain-server/src/DomainMetadata.h +++ b/domain-server/src/DomainMetadata.h @@ -39,15 +39,6 @@ public: static const QString MATURITY; static const QString HOSTS; static const QString TAGS; - static const QString HOURS; - class Hours { - public: - static const QString WEEKDAY; - static const QString WEEKEND; - static const QString UTC_OFFSET; - static const QString OPEN; - static const QString CLOSE; - }; }; DomainMetadata(QObject* domainServer); diff --git a/domain-server/src/DomainServerSettingsManager.cpp b/domain-server/src/DomainServerSettingsManager.cpp index a0b80875b0..31d6845972 100644 --- a/domain-server/src/DomainServerSettingsManager.cpp +++ b/domain-server/src/DomainServerSettingsManager.cpp @@ -21,7 +21,6 @@ #include #include #include -#include #include #include @@ -270,11 +269,6 @@ void DomainServerSettingsManager::setupConfigMap(const QStringList& argumentList _agentPermissions.clear(); } - if (oldVersion < 1.5) { - // This was prior to operating hours, so add default hours - validateDescriptorsMap(); - } - if (oldVersion < 1.6) { unpackPermissions(); @@ -305,46 +299,10 @@ void DomainServerSettingsManager::setupConfigMap(const QStringList& argumentList } QVariantMap& DomainServerSettingsManager::getDescriptorsMap() { - validateDescriptorsMap(); - static const QString DESCRIPTORS{ "descriptors" }; return *static_cast(getSettingsMap()[DESCRIPTORS].data()); } -void DomainServerSettingsManager::validateDescriptorsMap() { - static const QString WEEKDAY_HOURS{ "descriptors.weekday_hours" }; - static const QString WEEKEND_HOURS{ "descriptors.weekend_hours" }; - static const QString UTC_OFFSET{ "descriptors.utc_offset" }; - - QVariant* weekdayHours = _configMap.valueForKeyPath(WEEKDAY_HOURS, true); - QVariant* weekendHours = _configMap.valueForKeyPath(WEEKEND_HOURS, true); - QVariant* utcOffset = _configMap.valueForKeyPath(UTC_OFFSET, true); - - static const QString OPEN{ "open" }; - static const QString CLOSE{ "close" }; - static const QString DEFAULT_OPEN{ "00:00" }; - static const QString DEFAULT_CLOSE{ "23:59" }; - bool wasMalformed = false; - if (weekdayHours->isNull()) { - *weekdayHours = QVariantList{ QVariantMap{ { OPEN, QVariant(DEFAULT_OPEN) }, { CLOSE, QVariant(DEFAULT_CLOSE) } } }; - wasMalformed = true; - } - if (weekendHours->isNull()) { - *weekendHours = QVariantList{ QVariantMap{ { OPEN, QVariant(DEFAULT_OPEN) }, { CLOSE, QVariant(DEFAULT_CLOSE) } } }; - wasMalformed = true; - } - if (utcOffset->isNull()) { - *utcOffset = QVariant(QTimeZone::systemTimeZone().offsetFromUtc(QDateTime::currentDateTime()) / (float)SECS_PER_HOUR); - wasMalformed = true; - } - - if (wasMalformed) { - // write the new settings to file - persistToFile(); - } -} - - void DomainServerSettingsManager::initializeGroupPermissions(NodePermissionsMap& permissionsRows, QString groupName, NodePermissionsPointer perms) { // this is called when someone has used the domain-settings webpage to add a group. They type the group's name diff --git a/domain-server/src/DomainServerSettingsManager.h b/domain-server/src/DomainServerSettingsManager.h index 2b5f9518a0..d56a786d4b 100644 --- a/domain-server/src/DomainServerSettingsManager.h +++ b/domain-server/src/DomainServerSettingsManager.h @@ -138,8 +138,6 @@ private: friend class DomainServer; - void validateDescriptorsMap(); - // these cause calls to metaverse's group api void apiGetGroupID(const QString& groupName); void apiGetGroupRanks(const QUuid& groupID); From 8b44e83d9e38d49f4b2b53c2bc32b973b8577ac0 Mon Sep 17 00:00:00 2001 From: howard-stearns Date: Mon, 6 Feb 2017 11:11:18 -0800 Subject: [PATCH 24/41] Add new EntityPhysics packet type. --- assignment-client/src/entities/EntityServer.cpp | 2 +- libraries/entities/src/EntityEditPacketSender.cpp | 2 +- libraries/networking/src/udt/PacketHeaders.cpp | 3 ++- libraries/networking/src/udt/PacketHeaders.h | 4 +++- 4 files changed, 7 insertions(+), 4 deletions(-) diff --git a/assignment-client/src/entities/EntityServer.cpp b/assignment-client/src/entities/EntityServer.cpp index dc1a693590..02dc552dae 100644 --- a/assignment-client/src/entities/EntityServer.cpp +++ b/assignment-client/src/entities/EntityServer.cpp @@ -34,7 +34,7 @@ EntityServer::EntityServer(ReceivedMessage& message) : DependencyManager::set(); auto& packetReceiver = DependencyManager::get()->getPacketReceiver(); - packetReceiver.registerListenerForTypes({ PacketType::EntityAdd, PacketType::EntityEdit, PacketType::EntityErase }, + packetReceiver.registerListenerForTypes({ PacketType::EntityAdd, PacketType::EntityEdit, PacketType::EntityErase, PacketType::EntityPhysics }, this, "handleEntityPacket"); } diff --git a/libraries/entities/src/EntityEditPacketSender.cpp b/libraries/entities/src/EntityEditPacketSender.cpp index e05db07d0d..43de66f4b2 100644 --- a/libraries/entities/src/EntityEditPacketSender.cpp +++ b/libraries/entities/src/EntityEditPacketSender.cpp @@ -29,7 +29,7 @@ void EntityEditPacketSender::processEntityEditNackPacket(QSharedPointer(EntityQueryPacketVersion::JsonFilter); case PacketType::AvatarIdentity: diff --git a/libraries/networking/src/udt/PacketHeaders.h b/libraries/networking/src/udt/PacketHeaders.h index d695bde62a..88b5ec19ad 100644 --- a/libraries/networking/src/udt/PacketHeaders.h +++ b/libraries/networking/src/udt/PacketHeaders.h @@ -110,7 +110,8 @@ public: EntityScriptGetStatus, EntityScriptGetStatusReply, ReloadEntityServerScript, - LAST_PACKET_TYPE = ReloadEntityServerScript + EntityPhysics, + LAST_PACKET_TYPE = EntityPhysics }; }; @@ -201,6 +202,7 @@ const PacketVersion VERSION_WEB_ENTITIES_SUPPORT_DPI = 63; const PacketVersion VERSION_ENTITIES_ARROW_ACTION = 64; const PacketVersion VERSION_ENTITIES_LAST_EDITED_BY = 65; const PacketVersion VERSION_ENTITIES_SERVER_SCRIPTS = 66; +const PacketVersion VERSION_ENTITIES_PHYSICS_PACKET = 67; enum class EntityQueryPacketVersion: PacketVersion { JsonFilter = 18 From dacaa3df1d98b0a687f611484b9157f2483e4d57 Mon Sep 17 00:00:00 2001 From: howard-stearns Date: Mon, 6 Feb 2017 11:12:18 -0800 Subject: [PATCH 25/41] Send new entity physics packet from physics simulation, accept in server, and make it go through filter (when supplied) regardless of lock rights. --- libraries/entities/src/EntityTree.cpp | 13 ++++++++----- libraries/physics/src/EntityMotionState.cpp | 4 ++-- 2 files changed, 10 insertions(+), 7 deletions(-) diff --git a/libraries/entities/src/EntityTree.cpp b/libraries/entities/src/EntityTree.cpp index b7a8841772..60e05c75e0 100644 --- a/libraries/entities/src/EntityTree.cpp +++ b/libraries/entities/src/EntityTree.cpp @@ -104,6 +104,7 @@ bool EntityTree::handlesEditPacketType(PacketType packetType) const { case PacketType::EntityAdd: case PacketType::EntityEdit: case PacketType::EntityErase: + case PacketType::EntityPhysics: return true; default: return false; @@ -1001,6 +1002,7 @@ int EntityTree::processEditPacketData(ReceivedMessage& message, const unsigned c case PacketType::EntityAdd: isAdd = true; // fall through to next case + case PacketType::EntityPhysics: case PacketType::EntityEdit: { quint64 startDecode = 0, endDecode = 0; quint64 startLookup = 0, endLookup = 0; @@ -1010,6 +1012,7 @@ int EntityTree::processEditPacketData(ReceivedMessage& message, const unsigned c quint64 startLogging = 0, endLogging = 0; bool suppressDisallowedScript = false; + bool isPhysics = message.getType() == PacketType::EntityPhysics; _totalEditMessages++; @@ -1021,6 +1024,7 @@ int EntityTree::processEditPacketData(ReceivedMessage& message, const unsigned c entityItemID, properties); endDecode = usecTimestampNow(); + if (validEditPacket && !_entityScriptSourceWhitelist.isEmpty() && !properties.getScript().isEmpty()) { bool passedWhiteList = false; @@ -1053,8 +1057,7 @@ int EntityTree::processEditPacketData(ReceivedMessage& message, const unsigned c } } - if ((isAdd || - (message.getType() == PacketType::EntityEdit && properties.lifetimeChanged())) && + if ((isAdd || properties.lifetimeChanged()) && !senderNode->getCanRez() && senderNode->getCanRezTmp()) { // this node is only allowed to rez temporary entities. if need be, cap the lifetime. if (properties.getLifetime() == ENTITY_ITEM_IMMORTAL_LIFETIME || @@ -1070,8 +1073,8 @@ int EntityTree::processEditPacketData(ReceivedMessage& message, const unsigned c startFilter = usecTimestampNow(); bool wasChanged = false; - // Having (un)lock rights bypasses the filter. - bool allowed = senderNode->isAllowedEditor() || filterProperties(properties, properties, wasChanged, isAdd); + // Having (un)lock rights bypasses the filter, unless it's a physics result. + bool allowed = (!isPhysics && senderNode->isAllowedEditor()) || filterProperties(properties, properties, wasChanged, isAdd); if (!allowed) { auto timestamp = properties.getLastEdited(); properties = EntityItemProperties(); @@ -1088,7 +1091,7 @@ int EntityTree::processEditPacketData(ReceivedMessage& message, const unsigned c startLookup = usecTimestampNow(); EntityItemPointer existingEntity = findEntityByEntityItemID(entityItemID); endLookup = usecTimestampNow(); - if (existingEntity && message.getType() == PacketType::EntityEdit) { + if (existingEntity && !isAdd) { if (suppressDisallowedScript) { bumpTimestamp(properties); diff --git a/libraries/physics/src/EntityMotionState.cpp b/libraries/physics/src/EntityMotionState.cpp index e9891020b3..eb75718d92 100644 --- a/libraries/physics/src/EntityMotionState.cpp +++ b/libraries/physics/src/EntityMotionState.cpp @@ -614,7 +614,7 @@ void EntityMotionState::sendUpdate(OctreeEditPacketSender* packetSender, uint32_ properties.setClientOnly(_entity->getClientOnly()); properties.setOwningAvatarID(_entity->getOwningAvatarID()); - entityPacketSender->queueEditEntityMessage(PacketType::EntityEdit, tree, id, properties); + entityPacketSender->queueEditEntityMessage(PacketType::EntityPhysics, tree, id, properties); _entity->setLastBroadcast(now); // if we've moved an entity with children, check/update the queryAACube of all descendents and tell the server @@ -630,7 +630,7 @@ void EntityMotionState::sendUpdate(OctreeEditPacketSender* packetSender, uint32_ newQueryCubeProperties.setClientOnly(entityDescendant->getClientOnly()); newQueryCubeProperties.setOwningAvatarID(entityDescendant->getOwningAvatarID()); - entityPacketSender->queueEditEntityMessage(PacketType::EntityEdit, tree, + entityPacketSender->queueEditEntityMessage(PacketType::EntityPhysics, tree, descendant->getID(), newQueryCubeProperties); entityDescendant->setLastBroadcast(now); } From 5ff91c018b56b8821ecd33e06c85baee214e6673 Mon Sep 17 00:00:00 2001 From: Dante Ruiz Date: Mon, 6 Feb 2017 19:39:57 +0000 Subject: [PATCH 26/41] snapshot fix --- .../resources/qml/hifi/dialogs/AvatarPreferencesDialog.qml | 2 +- interface/src/ui/PreferencesDialog.cpp | 2 +- 2 files changed, 2 insertions(+), 2 deletions(-) diff --git a/interface/resources/qml/hifi/dialogs/AvatarPreferencesDialog.qml b/interface/resources/qml/hifi/dialogs/AvatarPreferencesDialog.qml index 45414cfaf8..86f195612c 100644 --- a/interface/resources/qml/hifi/dialogs/AvatarPreferencesDialog.qml +++ b/interface/resources/qml/hifi/dialogs/AvatarPreferencesDialog.qml @@ -7,7 +7,7 @@ PreferencesDialog { id: root objectName: "AvatarPreferencesDialog" title: "Avatar Settings" - showCategories: [ "Avatar Basics", "Snapshots", "Avatar Tuning", "Avatar Camera" ] + showCategories: [ "Avatar Basics", "Avatar Tuning", "Avatar Camera" ] property var settings: Settings { category: root.objectName property alias x: root.x diff --git a/interface/src/ui/PreferencesDialog.cpp b/interface/src/ui/PreferencesDialog.cpp index d68d9b4531..2bdfe795e2 100644 --- a/interface/src/ui/PreferencesDialog.cpp +++ b/interface/src/ui/PreferencesDialog.cpp @@ -104,7 +104,7 @@ void setupPreferences() { { auto getter = []()->bool { return SnapshotAnimated::alsoTakeAnimatedSnapshot.get(); }; auto setter = [](bool value) { SnapshotAnimated::alsoTakeAnimatedSnapshot.set(value); }; - preferences->addPreference(new CheckPreference(SNAPSHOTS, "Take Animated GIF Snapshot with HUD Button", getter, setter)); + preferences->addPreference(new CheckPreference(SNAPSHOTS, "Take Animated GIF Snapshot with tablet button", getter, setter)); } { auto getter = []()->float { return SnapshotAnimated::snapshotAnimatedDuration.get(); }; From 27e1bd50d0e24d5328cc6d1430a7721d190466d9 Mon Sep 17 00:00:00 2001 From: Dante Ruiz Date: Mon, 6 Feb 2017 19:52:51 +0000 Subject: [PATCH 27/41] reworded text --- interface/src/ui/PreferencesDialog.cpp | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/interface/src/ui/PreferencesDialog.cpp b/interface/src/ui/PreferencesDialog.cpp index 2bdfe795e2..6377cda281 100644 --- a/interface/src/ui/PreferencesDialog.cpp +++ b/interface/src/ui/PreferencesDialog.cpp @@ -104,7 +104,7 @@ void setupPreferences() { { auto getter = []()->bool { return SnapshotAnimated::alsoTakeAnimatedSnapshot.get(); }; auto setter = [](bool value) { SnapshotAnimated::alsoTakeAnimatedSnapshot.set(value); }; - preferences->addPreference(new CheckPreference(SNAPSHOTS, "Take Animated GIF Snapshot with tablet button", getter, setter)); + preferences->addPreference(new CheckPreference(SNAPSHOTS, "Take Animated GIF Snapshot", getter, setter)); } { auto getter = []()->float { return SnapshotAnimated::snapshotAnimatedDuration.get(); }; From 45c73439eb0b64af64164098d2696a1cc7b54107 Mon Sep 17 00:00:00 2001 From: Andrew Meadows Date: Thu, 2 Feb 2017 13:34:22 -0800 Subject: [PATCH 28/41] use type definition --- libraries/model-networking/src/model-networking/ModelCache.h | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/libraries/model-networking/src/model-networking/ModelCache.h b/libraries/model-networking/src/model-networking/ModelCache.h index bcca846625..adef3ce2b5 100644 --- a/libraries/model-networking/src/model-networking/ModelCache.h +++ b/libraries/model-networking/src/model-networking/ModelCache.h @@ -159,7 +159,7 @@ protected: class Texture { public: QString name; - QSharedPointer texture; + NetworkTexturePointer texture; }; using Textures = std::vector; From 24b1f3ddbc90d903937663ebf7b8708068a5e9c4 Mon Sep 17 00:00:00 2001 From: Andrew Meadows Date: Thu, 2 Feb 2017 13:34:44 -0800 Subject: [PATCH 29/41] cleanup whitespace --- libraries/fbx/src/FBXReader_Material.cpp | 19 ++++++++----------- 1 file changed, 8 insertions(+), 11 deletions(-) diff --git a/libraries/fbx/src/FBXReader_Material.cpp b/libraries/fbx/src/FBXReader_Material.cpp index 8c0f4b34ac..f3d0c30a41 100644 --- a/libraries/fbx/src/FBXReader_Material.cpp +++ b/libraries/fbx/src/FBXReader_Material.cpp @@ -47,10 +47,10 @@ FBXTexture FBXReader::getTexture(const QString& textureID) { texture.texcoordSet = 0; if (_textureParams.contains(textureID)) { auto p = _textureParams.value(textureID); - + texture.transform.setTranslation(p.translation); texture.transform.setRotation(glm::quat(glm::radians(p.rotation))); - + auto scaling = p.scaling; // Protect from bad scaling which should never happen if (scaling.x == 0.0f) { @@ -63,7 +63,7 @@ FBXTexture FBXReader::getTexture(const QString& textureID) { scaling.z = 1.0f; } texture.transform.setScale(scaling); - + if ((p.UVSet != "map1") && (p.UVSet != "UVSet0")) { texture.texcoordSet = 1; } @@ -78,11 +78,10 @@ void FBXReader::consolidateFBXMaterials(const QVariantHash& mapping) { QJsonDocument materialMapDocument = QJsonDocument::fromJson(materialMapString.toUtf8()); QJsonObject materialMap = materialMapDocument.object(); - // foreach (const QString& materialID, materials) { for (QHash::iterator it = _fbxMaterials.begin(); it != _fbxMaterials.end(); it++) { FBXMaterial& material = (*it); - // Maya is the exporting the shading model and we aretrying to use it + // Maya is the exporting the shading model and we are trying to use it bool isMaterialLambert = (material.shadingModel.toLower() == "lambert"); // the pure material associated with this part @@ -127,21 +126,19 @@ void FBXReader::consolidateFBXMaterials(const QVariantHash& mapping) { detectDifferentUVs |= (transparentTexture.texcoordSet != 0) || (!transparentTexture.transform.isIdentity()); } - - FBXTexture normalTexture; QString bumpTextureID = bumpTextures.value(material.materialID); QString normalTextureID = normalTextures.value(material.materialID); if (!normalTextureID.isNull()) { normalTexture = getTexture(normalTextureID); normalTexture.isBumpmap = false; - + material.normalTexture = normalTexture; detectDifferentUVs |= (normalTexture.texcoordSet != 0) || (!normalTexture.transform.isIdentity()); } else if (!bumpTextureID.isNull()) { normalTexture = getTexture(bumpTextureID); normalTexture.isBumpmap = true; - + material.normalTexture = normalTexture; detectDifferentUVs |= (normalTexture.texcoordSet != 0) || (!normalTexture.transform.isIdentity()); } @@ -151,7 +148,7 @@ void FBXReader::consolidateFBXMaterials(const QVariantHash& mapping) { if (!specularTextureID.isNull()) { specularTexture = getTexture(specularTextureID); detectDifferentUVs |= (specularTexture.texcoordSet != 0) || (!specularTexture.transform.isIdentity()); - material.specularTexture = specularTexture; + material.specularTexture = specularTexture; } FBXTexture metallicTexture; @@ -222,7 +219,7 @@ void FBXReader::consolidateFBXMaterials(const QVariantHash& mapping) { ambientTextureID = ambientFactorTextures.value(material.materialID); } } - + if (_loadLightmaps && !ambientTextureID.isNull()) { ambientTexture = getTexture(ambientTextureID); detectDifferentUVs |= (ambientTexture.texcoordSet != 0) || (!ambientTexture.transform.isIdentity()); From b3ae6f9aec65917d18bcfc26145b807ca85a1792 Mon Sep 17 00:00:00 2001 From: Andrew Meadows Date: Thu, 2 Feb 2017 13:35:40 -0800 Subject: [PATCH 30/41] cap size of textures from FBX files --- libraries/fbx/src/FBXReader.h | 9 ++-- .../src/model-networking/ModelCache.cpp | 2 +- .../src/model-networking/TextureCache.cpp | 53 +++++++++++++------ .../src/model-networking/TextureCache.h | 13 +++-- libraries/networking/src/ResourceCache.h | 4 +- 5 files changed, 54 insertions(+), 27 deletions(-) diff --git a/libraries/fbx/src/FBXReader.h b/libraries/fbx/src/FBXReader.h index cd1dbc5c4f..8b58ad62d1 100644 --- a/libraries/fbx/src/FBXReader.h +++ b/libraries/fbx/src/FBXReader.h @@ -100,22 +100,25 @@ public: /// A single binding to a joint in an FBX document. class FBXCluster { public: - + int jointIndex; glm::mat4 inverseBindMatrix; }; +const int MAX_FBX_TEXTURE_SIZE = 1024 * 1024; + /// A texture map in an FBX document. class FBXTexture { public: QString name; QByteArray filename; QByteArray content; - + Transform transform; + int maxSize { MAX_FBX_TEXTURE_SIZE }; int texcoordSet; QString texcoordSetName; - + bool isBumpmap{ false }; bool isNull() const { return name.isEmpty() && filename.isEmpty() && content.isEmpty(); } diff --git a/libraries/model-networking/src/model-networking/ModelCache.cpp b/libraries/model-networking/src/model-networking/ModelCache.cpp index 2a02540821..946e507d98 100644 --- a/libraries/model-networking/src/model-networking/ModelCache.cpp +++ b/libraries/model-networking/src/model-networking/ModelCache.cpp @@ -472,7 +472,7 @@ QUrl NetworkMaterial::getTextureUrl(const QUrl& baseUrl, const FBXTexture& textu model::TextureMapPointer NetworkMaterial::fetchTextureMap(const QUrl& baseUrl, const FBXTexture& fbxTexture, TextureType type, MapChannel channel) { const auto url = getTextureUrl(baseUrl, fbxTexture); - const auto texture = DependencyManager::get()->getTexture(url, type, fbxTexture.content); + const auto texture = DependencyManager::get()->getTexture(url, type, fbxTexture.content, fbxTexture.maxSize); _textures[channel] = Texture { fbxTexture.name, texture }; auto map = std::make_shared(); diff --git a/libraries/model-networking/src/model-networking/TextureCache.cpp b/libraries/model-networking/src/model-networking/TextureCache.cpp index 6a4edd60ed..e108e69aa1 100644 --- a/libraries/model-networking/src/model-networking/TextureCache.cpp +++ b/libraries/model-networking/src/model-networking/TextureCache.cpp @@ -167,16 +167,17 @@ class TextureExtra { public: NetworkTexture::Type type; const QByteArray& content; + int maxSize; }; -ScriptableResource* TextureCache::prefetch(const QUrl& url, int type) { +ScriptableResource* TextureCache::prefetch(const QUrl& url, int type, int maxSize) { auto byteArray = QByteArray(); - TextureExtra extra = { (Type)type, byteArray }; + TextureExtra extra = { (Type)type, byteArray, maxSize }; return ResourceCache::prefetch(url, &extra); } -NetworkTexturePointer TextureCache::getTexture(const QUrl& url, Type type, const QByteArray& content) { - TextureExtra extra = { type, content }; +NetworkTexturePointer TextureCache::getTexture(const QUrl& url, Type type, const QByteArray& content, int maxSize) { + TextureExtra extra = { type, content, maxSize }; return ResourceCache::getResource(url, QUrl(), &extra).staticCast(); } @@ -251,13 +252,15 @@ QSharedPointer TextureCache::createResource(const QUrl& url, const QSh const TextureExtra* textureExtra = static_cast(extra); auto type = textureExtra ? textureExtra->type : Type::DEFAULT_TEXTURE; auto content = textureExtra ? textureExtra->content : QByteArray(); - return QSharedPointer(new NetworkTexture(url, type, content), + auto maxSize = textureExtra ? textureExtra->maxSize : ABSOLUTE_MAX_TEXTURE_SIZE; + return QSharedPointer(new NetworkTexture(url, type, content, maxSize), &Resource::deleter); } -NetworkTexture::NetworkTexture(const QUrl& url, Type type, const QByteArray& content) : +NetworkTexture::NetworkTexture(const QUrl& url, Type type, const QByteArray& content, int maxSize) : Resource(url), - _type(type) + _type(type), + _maxSize(maxSize) { _textureSource = std::make_shared(); @@ -274,7 +277,7 @@ NetworkTexture::NetworkTexture(const QUrl& url, Type type, const QByteArray& con } NetworkTexture::NetworkTexture(const QUrl& url, const TextureLoaderFunc& textureLoader, const QByteArray& content) : - NetworkTexture(url, CUSTOM_TEXTURE, content) + NetworkTexture(url, CUSTOM_TEXTURE, content, ABSOLUTE_MAX_TEXTURE_SIZE) { _textureLoader = textureLoader; } @@ -290,7 +293,8 @@ NetworkTexture::TextureLoaderFunc NetworkTexture::getTextureLoader() const { class ImageReader : public QRunnable { public: - ImageReader(const QWeakPointer& resource, const QByteArray& data, const QUrl& url = QUrl()); + ImageReader(const QWeakPointer& resource, const QByteArray& data, + const QUrl& url = QUrl(), int maxSize = ABSOLUTE_MAX_TEXTURE_SIZE); virtual void run() override; @@ -300,6 +304,7 @@ private: QWeakPointer _resource; QUrl _url; QByteArray _content; + int _maxSize; }; void NetworkTexture::downloadFinished(const QByteArray& data) { @@ -308,14 +313,15 @@ void NetworkTexture::downloadFinished(const QByteArray& data) { } void NetworkTexture::loadContent(const QByteArray& content) { - QThreadPool::globalInstance()->start(new ImageReader(_self, content, _url)); + QThreadPool::globalInstance()->start(new ImageReader(_self, content, _url, _maxSize)); } ImageReader::ImageReader(const QWeakPointer& resource, const QByteArray& data, - const QUrl& url) : + const QUrl& url, int maxSize) : _resource(resource), _url(url), - _content(data) + _content(data), + _maxSize(maxSize) { #if DEBUG_DUMP_TEXTURE_LOADS static auto start = usecTimestampNow() / USECS_PER_MSEC; @@ -375,10 +381,10 @@ void ImageReader::run() { // Note that QImage.format is the pixel format which is different from the "format" of the image file... auto imageFormat = image.format(); - int originalWidth = image.width(); - int originalHeight = image.height(); + int imageWidth = image.width(); + int imageHeight = image.height(); - if (originalWidth == 0 || originalHeight == 0 || imageFormat == QImage::Format_Invalid) { + if (imageWidth == 0 || imageHeight == 0 || imageFormat == QImage::Format_Invalid) { if (filenameExtension.empty()) { qCDebug(modelnetworking) << "QImage failed to create from content, no file extension:" << _url; } else { @@ -386,6 +392,21 @@ void ImageReader::run() { } return; } + + if (imageWidth * imageHeight > _maxSize) { + float scaleFactor = sqrtf(_maxSize / (float)(imageWidth * imageHeight)); + int originalWidth = imageWidth; + int originalHeight = imageHeight; + imageWidth = (int)(scaleFactor * (float)imageWidth + 0.5f); + imageHeight = (int)(scaleFactor * (float)imageHeight + 0.5f); + //QSize newSize(imageWidth, imageHeight); + QImage newImage = image.scaled(QSize(imageWidth, imageHeight), Qt::IgnoreAspectRatio); + image.swap(newImage); + qCDebug(modelnetworking) << "Downscale image" << _url + << "from" << originalWidth << "x" << originalHeight + << "to" << imageWidth << "x" << imageHeight; + } + gpu::TexturePointer texture = nullptr; { // Double-check the resource still exists between long operations. @@ -408,7 +429,7 @@ void ImageReader::run() { } else { QMetaObject::invokeMethod(resource.data(), "setImage", Q_ARG(gpu::TexturePointer, texture), - Q_ARG(int, originalWidth), Q_ARG(int, originalHeight)); + Q_ARG(int, imageWidth), Q_ARG(int, imageHeight)); } } diff --git a/libraries/model-networking/src/model-networking/TextureCache.h b/libraries/model-networking/src/model-networking/TextureCache.h index 8ccfe5364c..9ad06eddfb 100644 --- a/libraries/model-networking/src/model-networking/TextureCache.h +++ b/libraries/model-networking/src/model-networking/TextureCache.h @@ -23,6 +23,8 @@ #include #include +const int ABSOLUTE_MAX_TEXTURE_SIZE = 8192 * 8192; + namespace gpu { class Batch; } @@ -60,7 +62,7 @@ public: typedef gpu::Texture* TextureLoader(const QImage& image, const std::string& srcImageName); using TextureLoaderFunc = std::function; - NetworkTexture(const QUrl& url, Type type, const QByteArray& content); + NetworkTexture(const QUrl& url, Type type, const QByteArray& content, int maxSize); NetworkTexture(const QUrl& url, const TextureLoaderFunc& textureLoader, const QByteArray& content); QString getType() const override { return "NetworkTexture"; } @@ -70,7 +72,7 @@ public: int getWidth() const { return _width; } int getHeight() const { return _height; } Type getTextureType() const { return _type; } - + TextureLoaderFunc getTextureLoader() const; signals: @@ -81,7 +83,7 @@ protected: virtual bool isCacheable() const override { return _loaded; } virtual void downloadFinished(const QByteArray& data) override; - + Q_INVOKABLE void loadContent(const QByteArray& content); Q_INVOKABLE void setImage(gpu::TexturePointer texture, int originalWidth, int originalHeight); @@ -92,6 +94,7 @@ private: int _originalHeight { 0 }; int _width { 0 }; int _height { 0 }; + int _maxSize { ABSOLUTE_MAX_TEXTURE_SIZE }; }; using NetworkTexturePointer = QSharedPointer; @@ -129,11 +132,11 @@ public: /// Loads a texture from the specified URL. NetworkTexturePointer getTexture(const QUrl& url, Type type = Type::DEFAULT_TEXTURE, - const QByteArray& content = QByteArray()); + const QByteArray& content = QByteArray(), int maxSize = ABSOLUTE_MAX_TEXTURE_SIZE); protected: // Overload ResourceCache::prefetch to allow specifying texture type for loads - Q_INVOKABLE ScriptableResource* prefetch(const QUrl& url, int type); + Q_INVOKABLE ScriptableResource* prefetch(const QUrl& url, int type, int maxSize = ABSOLUTE_MAX_TEXTURE_SIZE); virtual QSharedPointer createResource(const QUrl& url, const QSharedPointer& fallback, const void* extra) override; diff --git a/libraries/networking/src/ResourceCache.h b/libraries/networking/src/ResourceCache.h index 03d37bc9ac..a369416ebe 100644 --- a/libraries/networking/src/ResourceCache.h +++ b/libraries/networking/src/ResourceCache.h @@ -86,7 +86,7 @@ private: /// Wrapper to expose resources to JS/QML class ScriptableResource : public QObject { Q_OBJECT - Q_PROPERTY(QUrl url READ getUrl) + Q_PROPERTY(QUrl url READ getURL) Q_PROPERTY(int state READ getState NOTIFY stateChanged) /**jsdoc @@ -125,7 +125,7 @@ public: */ Q_INVOKABLE void release(); - const QUrl& getUrl() const { return _url; } + const QUrl& getURL() const { return _url; } int getState() const { return (int)_state; } const QSharedPointer& getResource() const { return _resource; } From d65cbbb455cab8a1994be4db65c6a6e1e3ba9ab1 Mon Sep 17 00:00:00 2001 From: Andrew Meadows Date: Thu, 2 Feb 2017 14:09:47 -0800 Subject: [PATCH 31/41] name change: maxSize to maxNumPixels --- libraries/fbx/src/FBXReader.h | 4 +-- .../src/model-networking/ModelCache.cpp | 2 +- .../src/model-networking/TextureCache.cpp | 35 +++++++++---------- .../src/model-networking/TextureCache.h | 10 +++--- 4 files changed, 25 insertions(+), 26 deletions(-) diff --git a/libraries/fbx/src/FBXReader.h b/libraries/fbx/src/FBXReader.h index 8b58ad62d1..5b4d3ec3bf 100644 --- a/libraries/fbx/src/FBXReader.h +++ b/libraries/fbx/src/FBXReader.h @@ -105,7 +105,7 @@ public: glm::mat4 inverseBindMatrix; }; -const int MAX_FBX_TEXTURE_SIZE = 1024 * 1024; +const int MAX_FBX_TEXTURE_NUM_PIXELS = 1024 * 1024; /// A texture map in an FBX document. class FBXTexture { @@ -115,7 +115,7 @@ public: QByteArray content; Transform transform; - int maxSize { MAX_FBX_TEXTURE_SIZE }; + int maxNumPixels { MAX_FBX_TEXTURE_NUM_PIXELS }; int texcoordSet; QString texcoordSetName; diff --git a/libraries/model-networking/src/model-networking/ModelCache.cpp b/libraries/model-networking/src/model-networking/ModelCache.cpp index 946e507d98..dd3193073d 100644 --- a/libraries/model-networking/src/model-networking/ModelCache.cpp +++ b/libraries/model-networking/src/model-networking/ModelCache.cpp @@ -472,7 +472,7 @@ QUrl NetworkMaterial::getTextureUrl(const QUrl& baseUrl, const FBXTexture& textu model::TextureMapPointer NetworkMaterial::fetchTextureMap(const QUrl& baseUrl, const FBXTexture& fbxTexture, TextureType type, MapChannel channel) { const auto url = getTextureUrl(baseUrl, fbxTexture); - const auto texture = DependencyManager::get()->getTexture(url, type, fbxTexture.content, fbxTexture.maxSize); + const auto texture = DependencyManager::get()->getTexture(url, type, fbxTexture.content, fbxTexture.maxNumPixels); _textures[channel] = Texture { fbxTexture.name, texture }; auto map = std::make_shared(); diff --git a/libraries/model-networking/src/model-networking/TextureCache.cpp b/libraries/model-networking/src/model-networking/TextureCache.cpp index e108e69aa1..447a1b93c8 100644 --- a/libraries/model-networking/src/model-networking/TextureCache.cpp +++ b/libraries/model-networking/src/model-networking/TextureCache.cpp @@ -167,17 +167,17 @@ class TextureExtra { public: NetworkTexture::Type type; const QByteArray& content; - int maxSize; + int maxNumPixels; }; -ScriptableResource* TextureCache::prefetch(const QUrl& url, int type, int maxSize) { +ScriptableResource* TextureCache::prefetch(const QUrl& url, int type, int maxNumPixels) { auto byteArray = QByteArray(); - TextureExtra extra = { (Type)type, byteArray, maxSize }; + TextureExtra extra = { (Type)type, byteArray, maxNumPixels }; return ResourceCache::prefetch(url, &extra); } -NetworkTexturePointer TextureCache::getTexture(const QUrl& url, Type type, const QByteArray& content, int maxSize) { - TextureExtra extra = { type, content, maxSize }; +NetworkTexturePointer TextureCache::getTexture(const QUrl& url, Type type, const QByteArray& content, int maxNumPixels) { + TextureExtra extra = { type, content, maxNumPixels }; return ResourceCache::getResource(url, QUrl(), &extra).staticCast(); } @@ -252,15 +252,15 @@ QSharedPointer TextureCache::createResource(const QUrl& url, const QSh const TextureExtra* textureExtra = static_cast(extra); auto type = textureExtra ? textureExtra->type : Type::DEFAULT_TEXTURE; auto content = textureExtra ? textureExtra->content : QByteArray(); - auto maxSize = textureExtra ? textureExtra->maxSize : ABSOLUTE_MAX_TEXTURE_SIZE; - return QSharedPointer(new NetworkTexture(url, type, content, maxSize), + auto maxNumPixels = textureExtra ? textureExtra->maxNumPixels : ABSOLUTE_MAX_TEXTURE_NUM_PIXELS; + return QSharedPointer(new NetworkTexture(url, type, content, maxNumPixels), &Resource::deleter); } -NetworkTexture::NetworkTexture(const QUrl& url, Type type, const QByteArray& content, int maxSize) : +NetworkTexture::NetworkTexture(const QUrl& url, Type type, const QByteArray& content, int maxNumPixels) : Resource(url), _type(type), - _maxSize(maxSize) + _maxNumPixels(maxNumPixels) { _textureSource = std::make_shared(); @@ -277,7 +277,7 @@ NetworkTexture::NetworkTexture(const QUrl& url, Type type, const QByteArray& con } NetworkTexture::NetworkTexture(const QUrl& url, const TextureLoaderFunc& textureLoader, const QByteArray& content) : - NetworkTexture(url, CUSTOM_TEXTURE, content, ABSOLUTE_MAX_TEXTURE_SIZE) + NetworkTexture(url, CUSTOM_TEXTURE, content, ABSOLUTE_MAX_TEXTURE_NUM_PIXELS) { _textureLoader = textureLoader; } @@ -294,7 +294,7 @@ class ImageReader : public QRunnable { public: ImageReader(const QWeakPointer& resource, const QByteArray& data, - const QUrl& url = QUrl(), int maxSize = ABSOLUTE_MAX_TEXTURE_SIZE); + const QUrl& url = QUrl(), int maxNumPixels = ABSOLUTE_MAX_TEXTURE_NUM_PIXELS); virtual void run() override; @@ -304,7 +304,7 @@ private: QWeakPointer _resource; QUrl _url; QByteArray _content; - int _maxSize; + int _maxNumPixels; }; void NetworkTexture::downloadFinished(const QByteArray& data) { @@ -313,15 +313,15 @@ void NetworkTexture::downloadFinished(const QByteArray& data) { } void NetworkTexture::loadContent(const QByteArray& content) { - QThreadPool::globalInstance()->start(new ImageReader(_self, content, _url, _maxSize)); + QThreadPool::globalInstance()->start(new ImageReader(_self, content, _url, _maxNumPixels)); } ImageReader::ImageReader(const QWeakPointer& resource, const QByteArray& data, - const QUrl& url, int maxSize) : + const QUrl& url, int maxNumPixels) : _resource(resource), _url(url), _content(data), - _maxSize(maxSize) + _maxNumPixels(maxNumPixels) { #if DEBUG_DUMP_TEXTURE_LOADS static auto start = usecTimestampNow() / USECS_PER_MSEC; @@ -393,13 +393,12 @@ void ImageReader::run() { return; } - if (imageWidth * imageHeight > _maxSize) { - float scaleFactor = sqrtf(_maxSize / (float)(imageWidth * imageHeight)); + if (imageWidth * imageHeight > _maxNumPixels) { + float scaleFactor = sqrtf(_maxNumPixels / (float)(imageWidth * imageHeight)); int originalWidth = imageWidth; int originalHeight = imageHeight; imageWidth = (int)(scaleFactor * (float)imageWidth + 0.5f); imageHeight = (int)(scaleFactor * (float)imageHeight + 0.5f); - //QSize newSize(imageWidth, imageHeight); QImage newImage = image.scaled(QSize(imageWidth, imageHeight), Qt::IgnoreAspectRatio); image.swap(newImage); qCDebug(modelnetworking) << "Downscale image" << _url diff --git a/libraries/model-networking/src/model-networking/TextureCache.h b/libraries/model-networking/src/model-networking/TextureCache.h index 9ad06eddfb..cb509490c6 100644 --- a/libraries/model-networking/src/model-networking/TextureCache.h +++ b/libraries/model-networking/src/model-networking/TextureCache.h @@ -23,7 +23,7 @@ #include #include -const int ABSOLUTE_MAX_TEXTURE_SIZE = 8192 * 8192; +const int ABSOLUTE_MAX_TEXTURE_NUM_PIXELS = 8192 * 8192; namespace gpu { class Batch; @@ -62,7 +62,7 @@ public: typedef gpu::Texture* TextureLoader(const QImage& image, const std::string& srcImageName); using TextureLoaderFunc = std::function; - NetworkTexture(const QUrl& url, Type type, const QByteArray& content, int maxSize); + NetworkTexture(const QUrl& url, Type type, const QByteArray& content, int maxNumPixels); NetworkTexture(const QUrl& url, const TextureLoaderFunc& textureLoader, const QByteArray& content); QString getType() const override { return "NetworkTexture"; } @@ -94,7 +94,7 @@ private: int _originalHeight { 0 }; int _width { 0 }; int _height { 0 }; - int _maxSize { ABSOLUTE_MAX_TEXTURE_SIZE }; + int _maxNumPixels { ABSOLUTE_MAX_TEXTURE_NUM_PIXELS }; }; using NetworkTexturePointer = QSharedPointer; @@ -132,11 +132,11 @@ public: /// Loads a texture from the specified URL. NetworkTexturePointer getTexture(const QUrl& url, Type type = Type::DEFAULT_TEXTURE, - const QByteArray& content = QByteArray(), int maxSize = ABSOLUTE_MAX_TEXTURE_SIZE); + const QByteArray& content = QByteArray(), int maxNumPixels = ABSOLUTE_MAX_TEXTURE_NUM_PIXELS); protected: // Overload ResourceCache::prefetch to allow specifying texture type for loads - Q_INVOKABLE ScriptableResource* prefetch(const QUrl& url, int type, int maxSize = ABSOLUTE_MAX_TEXTURE_SIZE); + Q_INVOKABLE ScriptableResource* prefetch(const QUrl& url, int type, int maxNumPixels = ABSOLUTE_MAX_TEXTURE_NUM_PIXELS); virtual QSharedPointer createResource(const QUrl& url, const QSharedPointer& fallback, const void* extra) override; From 8b2c720e2994c89c6dfe2f3202047e0f8b971f21 Mon Sep 17 00:00:00 2001 From: Andrew Meadows Date: Fri, 3 Feb 2017 11:14:40 -0800 Subject: [PATCH 32/41] sliding texture caps for models --- libraries/fbx/src/FBXReader.cpp | 27 +++++++++++++ libraries/fbx/src/FBXReader.h | 8 +++- libraries/fbx/src/FBXReader_Material.cpp | 50 ++++++++++++++++++++++++ 3 files changed, 83 insertions(+), 2 deletions(-) diff --git a/libraries/fbx/src/FBXReader.cpp b/libraries/fbx/src/FBXReader.cpp index 42922ce226..917009956e 100644 --- a/libraries/fbx/src/FBXReader.cpp +++ b/libraries/fbx/src/FBXReader.cpp @@ -1465,6 +1465,33 @@ FBXGeometry* FBXReader::extractFBXGeometry(const QVariantHash& mapping, const QS // Create the Material Library consolidateFBXMaterials(mapping); + + // HACK: until we get proper LOD management we're going to cap model textures + // according to how many unique textures the model uses: + // 1 - 7 textures --> 2048 + // 8 - 31 textures --> 1024 + // 32 - 127 textures --> 512 + // etc... + QSet uniqueTextures; + for (auto& material : _fbxMaterials) { + material.getTextureNames(uniqueTextures); + } + int numTextures = uniqueTextures.size(); + const int MAX_NUM_TEXTURES_AT_MAX_RESOLUTION = 7; + if (numTextures > MAX_NUM_TEXTURES_AT_MAX_RESOLUTION) { + int maxWidth = sqrt(MAX_NUM_PIXELS_FOR_FBX_TEXTURE + 1); + int t = numTextures; + t /= MAX_NUM_TEXTURES_AT_MAX_RESOLUTION; + while (t > 0) { + maxWidth /= 2; + t /= 4; + } + qCDebug(modelformat) << "max square texture width =" << maxWidth << " for model" << url; + for (auto& material : _fbxMaterials) { + material.setMaxNumPixelsPerTexture(maxWidth * maxWidth); + } + } + geometry.materials = _fbxMaterials; // see if any materials have texture children diff --git a/libraries/fbx/src/FBXReader.h b/libraries/fbx/src/FBXReader.h index 5b4d3ec3bf..b0e34dd674 100644 --- a/libraries/fbx/src/FBXReader.h +++ b/libraries/fbx/src/FBXReader.h @@ -13,6 +13,7 @@ #define hifi_FBXReader_h #include +#include #include #include #include @@ -105,7 +106,7 @@ public: glm::mat4 inverseBindMatrix; }; -const int MAX_FBX_TEXTURE_NUM_PIXELS = 1024 * 1024; +const int MAX_NUM_PIXELS_FOR_FBX_TEXTURE = 2048 * 2048; /// A texture map in an FBX document. class FBXTexture { @@ -115,7 +116,7 @@ public: QByteArray content; Transform transform; - int maxNumPixels { MAX_FBX_TEXTURE_NUM_PIXELS }; + int maxNumPixels { MAX_NUM_PIXELS_FOR_FBX_TEXTURE }; int texcoordSet; QString texcoordSetName; @@ -146,6 +147,9 @@ public: shininess(shininess), opacity(opacity) {} + void getTextureNames(QSet& textureList) const; + void setMaxNumPixelsPerTexture(int maxNumPixels); + glm::vec3 diffuseColor{ 1.0f }; float diffuseFactor{ 1.0f }; glm::vec3 specularColor{ 0.02f }; diff --git a/libraries/fbx/src/FBXReader_Material.cpp b/libraries/fbx/src/FBXReader_Material.cpp index f3d0c30a41..ca2ec557b4 100644 --- a/libraries/fbx/src/FBXReader_Material.cpp +++ b/libraries/fbx/src/FBXReader_Material.cpp @@ -27,6 +27,56 @@ #include "ModelFormatLogging.h" +void FBXMaterial::getTextureNames(QSet& textureList) const { + if (!normalTexture.isNull()) { + textureList.insert(normalTexture.name); + } + if (!albedoTexture.isNull()) { + textureList.insert(albedoTexture.name); + } + if (!opacityTexture.isNull()) { + textureList.insert(opacityTexture.name); + } + if (!glossTexture.isNull()) { + textureList.insert(glossTexture.name); + } + if (!roughnessTexture.isNull()) { + textureList.insert(roughnessTexture.name); + } + if (!specularTexture.isNull()) { + textureList.insert(specularTexture.name); + } + if (!metallicTexture.isNull()) { + textureList.insert(metallicTexture.name); + } + if (!emissiveTexture.isNull()) { + textureList.insert(emissiveTexture.name); + } + if (!occlusionTexture.isNull()) { + textureList.insert(occlusionTexture.name); + } + if (!scatteringTexture.isNull()) { + textureList.insert(scatteringTexture.name); + } + if (!lightmapTexture.isNull()) { + textureList.insert(lightmapTexture.name); + } +} + +void FBXMaterial::setMaxNumPixelsPerTexture(int maxNumPixels) { + normalTexture.maxNumPixels = maxNumPixels; + albedoTexture.maxNumPixels = maxNumPixels; + opacityTexture.maxNumPixels = maxNumPixels; + glossTexture.maxNumPixels = maxNumPixels; + roughnessTexture.maxNumPixels = maxNumPixels; + specularTexture.maxNumPixels = maxNumPixels; + metallicTexture.maxNumPixels = maxNumPixels; + emissiveTexture.maxNumPixels = maxNumPixels; + occlusionTexture.maxNumPixels = maxNumPixels; + scatteringTexture.maxNumPixels = maxNumPixels; + lightmapTexture.maxNumPixels = maxNumPixels; +} + bool FBXMaterial::needTangentSpace() const { return !normalTexture.isNull(); } From 2ea07390c8f0572634ad12f362d9305f1d0a546e Mon Sep 17 00:00:00 2001 From: Seth Alves Date: Mon, 6 Feb 2017 12:10:26 -0800 Subject: [PATCH 33/41] when the parentID is changed by a network update, set physics flags --- libraries/entities/src/EntityItem.cpp | 2 +- libraries/physics/src/EntityMotionState.cpp | 3 --- 2 files changed, 1 insertion(+), 4 deletions(-) diff --git a/libraries/entities/src/EntityItem.cpp b/libraries/entities/src/EntityItem.cpp index 2503687dce..a656827d5e 100644 --- a/libraries/entities/src/EntityItem.cpp +++ b/libraries/entities/src/EntityItem.cpp @@ -828,7 +828,7 @@ int EntityItem::readEntityDataFromBuffer(const unsigned char* data, int bytesLef { // parentID and parentJointIndex are also protected by simulation ownership bool oldOverwrite = overwriteLocalData; overwriteLocalData = overwriteLocalData && !weOwnSimulation; - READ_ENTITY_PROPERTY(PROP_PARENT_ID, QUuid, setParentID); + READ_ENTITY_PROPERTY(PROP_PARENT_ID, QUuid, updateParentID); READ_ENTITY_PROPERTY(PROP_PARENT_JOINT_INDEX, quint16, setParentJointIndex); overwriteLocalData = oldOverwrite; } diff --git a/libraries/physics/src/EntityMotionState.cpp b/libraries/physics/src/EntityMotionState.cpp index d652e78f6a..c90ca5a1a9 100644 --- a/libraries/physics/src/EntityMotionState.cpp +++ b/libraries/physics/src/EntityMotionState.cpp @@ -230,9 +230,6 @@ void EntityMotionState::setWorldTransform(const btTransform& worldTrans) { assert(entityTreeIsLocked()); measureBodyAcceleration(); bool positionSuccess; - if (_entity->hasAncestorOfType(NestableType::Avatar)) { - return; - } _entity->setPosition(bulletToGLM(worldTrans.getOrigin()) + ObjectMotionState::getWorldOffset(), positionSuccess, false); if (!positionSuccess) { static QString repeatedMessage = From 8b1f569132852cb61ffae0fe70ab1bea68460d78 Mon Sep 17 00:00:00 2001 From: Andrew Meadows Date: Mon, 6 Feb 2017 12:13:31 -0800 Subject: [PATCH 34/41] tweak max texture resolution thresholds --- libraries/fbx/src/FBXReader.cpp | 23 ++++++++++++----------- 1 file changed, 12 insertions(+), 11 deletions(-) diff --git a/libraries/fbx/src/FBXReader.cpp b/libraries/fbx/src/FBXReader.cpp index 917009956e..2e6544fbdb 100644 --- a/libraries/fbx/src/FBXReader.cpp +++ b/libraries/fbx/src/FBXReader.cpp @@ -1468,25 +1468,26 @@ FBXGeometry* FBXReader::extractFBXGeometry(const QVariantHash& mapping, const QS // HACK: until we get proper LOD management we're going to cap model textures // according to how many unique textures the model uses: - // 1 - 7 textures --> 2048 - // 8 - 31 textures --> 1024 - // 32 - 127 textures --> 512 + // 1 - 8 textures --> 2048 + // 8 - 32 textures --> 1024 + // 33 - 128 textures --> 512 // etc... QSet uniqueTextures; for (auto& material : _fbxMaterials) { material.getTextureNames(uniqueTextures); } int numTextures = uniqueTextures.size(); - const int MAX_NUM_TEXTURES_AT_MAX_RESOLUTION = 7; + const int MAX_NUM_TEXTURES_AT_MAX_RESOLUTION = 8; + int maxWidth = sqrt(MAX_NUM_PIXELS_FOR_FBX_TEXTURE); if (numTextures > MAX_NUM_TEXTURES_AT_MAX_RESOLUTION) { - int maxWidth = sqrt(MAX_NUM_PIXELS_FOR_FBX_TEXTURE + 1); - int t = numTextures; - t /= MAX_NUM_TEXTURES_AT_MAX_RESOLUTION; - while (t > 0) { + int numTextureThreshold = MAX_NUM_TEXTURES_AT_MAX_RESOLUTION; + const int MIN_MIP_TEXTURE_WIDTH = 64; + do { maxWidth /= 2; - t /= 4; - } - qCDebug(modelformat) << "max square texture width =" << maxWidth << " for model" << url; + numTextureThreshold *= 4; + } while (numTextureThreshold < numTextures && maxWidth > MIN_MIP_TEXTURE_WIDTH); + + qCDebug(modelformat) << "Capped square texture width =" << maxWidth << "for model" << url << "with" << numTextures << "textures"; for (auto& material : _fbxMaterials) { material.setMaxNumPixelsPerTexture(maxWidth * maxWidth); } From 9b0190f1cf829ad54f4216f8acd0b7bb24acbb89 Mon Sep 17 00:00:00 2001 From: Zach Pomerantz Date: Mon, 6 Feb 2017 20:31:19 +0000 Subject: [PATCH 35/41] add ns/throttle to audio mixer debug stats --- assignment-client/src/audio/AudioMixer.cpp | 4 ++++ assignment-client/src/audio/AudioMixerSlave.cpp | 11 +++++++++++ assignment-client/src/audio/AudioMixerStats.cpp | 6 ++++++ assignment-client/src/audio/AudioMixerStats.h | 8 ++++++++ 4 files changed, 29 insertions(+) diff --git a/assignment-client/src/audio/AudioMixer.cpp b/assignment-client/src/audio/AudioMixer.cpp index 04acae6f05..4f123a6a8f 100644 --- a/assignment-client/src/audio/AudioMixer.cpp +++ b/assignment-client/src/audio/AudioMixer.cpp @@ -316,6 +316,10 @@ void AudioMixer::sendStatsPacket() { addTiming(_mixTiming, "mix"); addTiming(_eventsTiming, "events"); +#ifdef HIFI_AUDIO_THROTTLE_DEBUG + timingStats["ns_per_throttle"] = (_stats.totalMixes > 0) ? (float)(_stats.throttleTime / _stats.totalMixes) : 0; +#endif + // call it "avg_..." to keep it higher in the display, sorted alphabetically statsObject["avg_timing_stats"] = timingStats; diff --git a/assignment-client/src/audio/AudioMixerSlave.cpp b/assignment-client/src/audio/AudioMixerSlave.cpp index 1e78507e28..adc6413316 100644 --- a/assignment-client/src/audio/AudioMixerSlave.cpp +++ b/assignment-client/src/audio/AudioMixerSlave.cpp @@ -150,6 +150,10 @@ bool AudioMixerSlave::prepareMix(const SharedNodePointer& listener) { if (!isThrottling) { allStreams(node, &AudioMixerSlave::mixStream); } else { +#ifdef HIFI_AUDIO_THROTTLE_DEBUG + auto throttleStart = p_high_resolution_clock::now(); +#endif + AudioMixerClientData* nodeData = static_cast(node->getLinkedData()); auto nodeID = node->getUUID(); @@ -175,6 +179,13 @@ bool AudioMixerSlave::prepareMix(const SharedNodePointer& listener) { if (!throttledNodes.empty()) { std::push_heap(throttledNodes.begin(), throttledNodes.end()); } + +#ifdef HIFI_AUDIO_THROTTLE_DEBUG + auto throttleEnd = p_high_resolution_clock::now(); + uint64_t throttleTime = + std::chrono::duration_cast(throttleEnd - throttleStart).count(); + stats.throttleTime += throttleTime; +#endif } } }); diff --git a/assignment-client/src/audio/AudioMixerStats.cpp b/assignment-client/src/audio/AudioMixerStats.cpp index a50c0d26c1..a3a3a215bc 100644 --- a/assignment-client/src/audio/AudioMixerStats.cpp +++ b/assignment-client/src/audio/AudioMixerStats.cpp @@ -20,6 +20,9 @@ void AudioMixerStats::reset() { hrtfThrottleRenders = 0; manualStereoMixes = 0; manualEchoMixes = 0; +#ifdef HIFI_AUDIO_THROTTLE_DEBUG + throttleTime = 0; +#endif } void AudioMixerStats::accumulate(const AudioMixerStats& otherStats) { @@ -31,4 +34,7 @@ void AudioMixerStats::accumulate(const AudioMixerStats& otherStats) { hrtfThrottleRenders += otherStats.hrtfThrottleRenders; manualStereoMixes += otherStats.manualStereoMixes; manualEchoMixes += otherStats.manualEchoMixes; +#ifdef HIFI_AUDIO_THROTTLE_DEBUG + throttleTime += otherStats.throttleTime; +#endif } diff --git a/assignment-client/src/audio/AudioMixerStats.h b/assignment-client/src/audio/AudioMixerStats.h index cb85006061..f7e3ed1525 100644 --- a/assignment-client/src/audio/AudioMixerStats.h +++ b/assignment-client/src/audio/AudioMixerStats.h @@ -12,6 +12,10 @@ #ifndef hifi_AudioMixerStats_h #define hifi_AudioMixerStats_h +#ifdef HIFI_AUDIO_THROTTLE_DEBUG +#include +#endif + struct AudioMixerStats { int sumStreams { 0 }; int sumListeners { 0 }; @@ -25,6 +29,10 @@ struct AudioMixerStats { int manualStereoMixes { 0 }; int manualEchoMixes { 0 }; +#ifdef HIFI_AUDIO_THROTTLE_DEBUG + uint64_t throttleTime { 0 }; +#endif + void reset(); void accumulate(const AudioMixerStats& otherStats); }; From a77e4262e3f1112ee77f715a50ce27b668a29d1a Mon Sep 17 00:00:00 2001 From: Ken Cooke Date: Mon, 6 Feb 2017 12:47:28 -0800 Subject: [PATCH 36/41] Remove obsoleted code --- libraries/audio/src/InboundAudioStream.cpp | 4 ---- libraries/audio/src/InboundAudioStream.h | 2 -- 2 files changed, 6 deletions(-) diff --git a/libraries/audio/src/InboundAudioStream.cpp b/libraries/audio/src/InboundAudioStream.cpp index d23579430e..c90032436b 100644 --- a/libraries/audio/src/InboundAudioStream.cpp +++ b/libraries/audio/src/InboundAudioStream.cpp @@ -432,10 +432,6 @@ void InboundAudioStream::packetReceivedUpdateTimingStats() { _lastPacketReceivedTime = now; } -int InboundAudioStream::writeFramesForDroppedPackets(int networkFrames) { - return writeLastFrameRepeatedWithFade(networkFrames); -} - int InboundAudioStream::writeLastFrameRepeatedWithFade(int frames) { AudioRingBuffer::ConstIterator frameToRepeat = _ringBuffer.lastFrameWritten(); int frameSize = _ringBuffer.getNumFrameSamples(); diff --git a/libraries/audio/src/InboundAudioStream.h b/libraries/audio/src/InboundAudioStream.h index b349016452..f96ef539c7 100644 --- a/libraries/audio/src/InboundAudioStream.h +++ b/libraries/audio/src/InboundAudioStream.h @@ -115,8 +115,6 @@ public slots: private: void packetReceivedUpdateTimingStats(); - int writeFramesForDroppedPackets(int networkFrames); - void popSamplesNoCheck(int samples); void framesAvailableChanged(); From 63c2aa1dfcd302576a2a541fbb0fede4a3f71f64 Mon Sep 17 00:00:00 2001 From: Ken Cooke Date: Mon, 6 Feb 2017 12:55:03 -0800 Subject: [PATCH 37/41] Remove obsoleted code --- libraries/audio/src/InboundAudioStream.cpp | 19 ------------------- libraries/audio/src/InboundAudioStream.h | 4 ---- .../audio/src/MixedProcessedAudioStream.cpp | 7 ------- .../audio/src/MixedProcessedAudioStream.h | 1 - 4 files changed, 31 deletions(-) diff --git a/libraries/audio/src/InboundAudioStream.cpp b/libraries/audio/src/InboundAudioStream.cpp index c90032436b..57c344adaf 100644 --- a/libraries/audio/src/InboundAudioStream.cpp +++ b/libraries/audio/src/InboundAudioStream.cpp @@ -432,25 +432,6 @@ void InboundAudioStream::packetReceivedUpdateTimingStats() { _lastPacketReceivedTime = now; } -int InboundAudioStream::writeLastFrameRepeatedWithFade(int frames) { - AudioRingBuffer::ConstIterator frameToRepeat = _ringBuffer.lastFrameWritten(); - int frameSize = _ringBuffer.getNumFrameSamples(); - int samplesToWrite = frames * _numChannels; - int indexOfRepeat = 0; - do { - int samplesToWriteThisIteration = std::min(samplesToWrite, frameSize); - float fade = calculateRepeatedFrameFadeFactor(indexOfRepeat); - if (fade == 1.0f) { - samplesToWrite -= _ringBuffer.writeSamples(frameToRepeat, samplesToWriteThisIteration); - } else { - samplesToWrite -= _ringBuffer.writeSamplesWithFade(frameToRepeat, samplesToWriteThisIteration, fade); - } - indexOfRepeat++; - } while (samplesToWrite > 0); - - return frames; -} - AudioStreamStats InboundAudioStream::getAudioStreamStats() const { AudioStreamStats streamStats; diff --git a/libraries/audio/src/InboundAudioStream.h b/libraries/audio/src/InboundAudioStream.h index f96ef539c7..9494b2f204 100644 --- a/libraries/audio/src/InboundAudioStream.h +++ b/libraries/audio/src/InboundAudioStream.h @@ -137,10 +137,6 @@ protected: /// writes silent frames to the buffer that may be dropped to reduce latency caused by the buffer virtual int writeDroppableSilentFrames(int silentFrames); - - /// writes the last written frame repeatedly, gradually fading to silence. - /// used for writing samples for dropped packets. - virtual int writeLastFrameRepeatedWithFade(int frames); protected: diff --git a/libraries/audio/src/MixedProcessedAudioStream.cpp b/libraries/audio/src/MixedProcessedAudioStream.cpp index c3170c3259..082977246b 100644 --- a/libraries/audio/src/MixedProcessedAudioStream.cpp +++ b/libraries/audio/src/MixedProcessedAudioStream.cpp @@ -31,13 +31,6 @@ int MixedProcessedAudioStream::writeDroppableSilentFrames(int silentFrames) { return deviceSilentFramesWritten; } -int MixedProcessedAudioStream::writeLastFrameRepeatedWithFade(int frames) { - int deviceFrames = networkToDeviceFrames(frames); - int deviceFramesWritten = InboundAudioStream::writeLastFrameRepeatedWithFade(deviceFrames); - emit addedLastFrameRepeatedWithFade(deviceToNetworkFrames(deviceFramesWritten)); - return deviceFramesWritten; -} - int MixedProcessedAudioStream::lostAudioData(int numPackets) { QByteArray decodedBuffer; QByteArray outputBuffer; diff --git a/libraries/audio/src/MixedProcessedAudioStream.h b/libraries/audio/src/MixedProcessedAudioStream.h index 83c4cea635..14da1d45af 100644 --- a/libraries/audio/src/MixedProcessedAudioStream.h +++ b/libraries/audio/src/MixedProcessedAudioStream.h @@ -34,7 +34,6 @@ public: protected: int writeDroppableSilentFrames(int silentFrames) override; - int writeLastFrameRepeatedWithFade(int frames) override; int parseAudioData(PacketType type, const QByteArray& packetAfterStreamProperties) override; int lostAudioData(int numPackets) override; From 12720cac327b709da43764ee5acd47563e45c56e Mon Sep 17 00:00:00 2001 From: howard-stearns Date: Mon, 6 Feb 2017 13:36:45 -0800 Subject: [PATCH 38/41] Pass filter type as an int, with constants to compare against. --- libraries/entities/src/EntityTree.cpp | 12 +++++++++--- libraries/entities/src/EntityTree.h | 7 ++++++- 2 files changed, 15 insertions(+), 4 deletions(-) diff --git a/libraries/entities/src/EntityTree.cpp b/libraries/entities/src/EntityTree.cpp index 60e05c75e0..4e92b2a572 100644 --- a/libraries/entities/src/EntityTree.cpp +++ b/libraries/entities/src/EntityTree.cpp @@ -932,10 +932,15 @@ void EntityTree::initEntityEditFilterEngine(QScriptEngine* engine, std::function qCDebug(entities) << "Filter function specified but not found. Will reject all edits."; _entityEditFilterEngine = nullptr; // So that we don't try to call it. See filterProperties. } + auto entitiesObject = _entityEditFilterEngine->newObject(); + entitiesObject.setProperty("ADD_FILTER_TYPE", FilterType::Add); + entitiesObject.setProperty("EDIT_FILTER_TYPE", FilterType::Edit); + entitiesObject.setProperty("PHYSICS_FILTER_TYPE", FilterType::Physics); + global.setProperty("Entities", entitiesObject); _hasEntityEditFilter = true; } -bool EntityTree::filterProperties(EntityItemProperties& propertiesIn, EntityItemProperties& propertiesOut, bool& wasChanged, bool isAdd) { +bool EntityTree::filterProperties(EntityItemProperties& propertiesIn, EntityItemProperties& propertiesOut, bool& wasChanged, FilterType filterType) { if (!_entityEditFilterEngine) { propertiesOut = propertiesIn; wasChanged = false; // not changed @@ -954,7 +959,7 @@ bool EntityTree::filterProperties(EntityItemProperties& propertiesIn, EntityItem auto in = QJsonValue::fromVariant(inputValues.toVariant()); // grab json copy now, because the inputValues might be side effected by the filter. QScriptValueList args; args << inputValues; - args << isAdd; + args << filterType; QScriptValue result = _entityEditFilterFunction.call(_nullObjectForFilter, args); if (_entityEditFilterHadUncaughtExceptions()) { @@ -1074,7 +1079,8 @@ int EntityTree::processEditPacketData(ReceivedMessage& message, const unsigned c startFilter = usecTimestampNow(); bool wasChanged = false; // Having (un)lock rights bypasses the filter, unless it's a physics result. - bool allowed = (!isPhysics && senderNode->isAllowedEditor()) || filterProperties(properties, properties, wasChanged, isAdd); + FilterType filterType = isPhysics ? FilterType::Physics : (isAdd ? FilterType::Add : FilterType::Edit); + bool allowed = (!isPhysics && senderNode->isAllowedEditor()) || filterProperties(properties, properties, wasChanged, filterType); if (!allowed) { auto timestamp = properties.getLastEdited(); properties = EntityItemProperties(); diff --git a/libraries/entities/src/EntityTree.h b/libraries/entities/src/EntityTree.h index 9b30096be5..5dad282d3b 100644 --- a/libraries/entities/src/EntityTree.h +++ b/libraries/entities/src/EntityTree.h @@ -60,6 +60,11 @@ public: class EntityTree : public Octree, public SpatialParentTree { Q_OBJECT public: + enum FilterType { + Add, + Edit, + Physics + }; EntityTree(bool shouldReaverage = false); virtual ~EntityTree(); @@ -357,7 +362,7 @@ protected: float _maxTmpEntityLifetime { DEFAULT_MAX_TMP_ENTITY_LIFETIME }; - bool filterProperties(EntityItemProperties& propertiesIn, EntityItemProperties& propertiesOut, bool& wasChanged, bool isAdd); + bool filterProperties(EntityItemProperties& propertiesIn, EntityItemProperties& propertiesOut, bool& wasChanged, FilterType filterType); bool _hasEntityEditFilter{ false }; QScriptEngine* _entityEditFilterEngine{}; QScriptValue _entityEditFilterFunction{}; From aaefc550f0e4ddcfda89242ef11528e7076017bf Mon Sep 17 00:00:00 2001 From: Seth Alves Date: Mon, 6 Feb 2017 14:16:47 -0800 Subject: [PATCH 39/41] avoid bootstrapping self with something that has someone else as simulation owner --- libraries/entities/src/EntityItem.cpp | 50 +++++++++++++++------------ 1 file changed, 27 insertions(+), 23 deletions(-) diff --git a/libraries/entities/src/EntityItem.cpp b/libraries/entities/src/EntityItem.cpp index a656827d5e..6543af5355 100644 --- a/libraries/entities/src/EntityItem.cpp +++ b/libraries/entities/src/EntityItem.cpp @@ -1823,29 +1823,6 @@ void EntityItem::computeCollisionGroupAndFinalMask(int16_t& group, int16_t& mask } uint8_t userMask = getCollisionMask(); - if (userMask & USER_COLLISION_GROUP_MY_AVATAR) { - // if this entity is a descendant of MyAvatar, don't collide with MyAvatar. This avoids the - // "bootstrapping" problem where you can shoot yourself across the room by grabbing something - // and holding it against your own avatar. - QUuid ancestorID = findAncestorOfType(NestableType::Avatar); - if (!ancestorID.isNull() && - (ancestorID == Physics::getSessionUUID() || ancestorID == AVATAR_SELF_ID)) { - userMask &= ~USER_COLLISION_GROUP_MY_AVATAR; - } - } - if (userMask & USER_COLLISION_GROUP_MY_AVATAR) { - // also, don't bootstrap our own avatar with a hold action - QList holdActions = getActionsOfType(ACTION_TYPE_HOLD); - QList::const_iterator i = holdActions.begin(); - while (i != holdActions.end()) { - EntityActionPointer action = *i; - if (action->isMine()) { - userMask &= ~USER_COLLISION_GROUP_MY_AVATAR; - break; - } - i++; - } - } if ((bool)(userMask & USER_COLLISION_GROUP_MY_AVATAR) != (bool)(userMask & USER_COLLISION_GROUP_OTHER_AVATAR)) { @@ -1855,6 +1832,33 @@ void EntityItem::computeCollisionGroupAndFinalMask(int16_t& group, int16_t& mask userMask ^= USER_COLLISION_MASK_AVATARS | ~userMask; } } + + if (userMask & USER_COLLISION_GROUP_MY_AVATAR) { + bool iAmHoldingThis = false; + // if this entity is a descendant of MyAvatar, don't collide with MyAvatar. This avoids the + // "bootstrapping" problem where you can shoot yourself across the room by grabbing something + // and holding it against your own avatar. + QUuid ancestorID = findAncestorOfType(NestableType::Avatar); + if (!ancestorID.isNull() && + (ancestorID == Physics::getSessionUUID() || ancestorID == AVATAR_SELF_ID)) { + iAmHoldingThis = true; + } + // also, don't bootstrap our own avatar with a hold action + QList holdActions = getActionsOfType(ACTION_TYPE_HOLD); + QList::const_iterator i = holdActions.begin(); + while (i != holdActions.end()) { + EntityActionPointer action = *i; + if (action->isMine()) { + iAmHoldingThis = true; + break; + } + i++; + } + + if (iAmHoldingThis) { + userMask &= ~USER_COLLISION_GROUP_MY_AVATAR; + } + } mask = Physics::getDefaultCollisionMask(group) & (int16_t)(userMask); } } From a14dcbd513c1df6b94851bc878c65b2940a5bba1 Mon Sep 17 00:00:00 2001 From: "Anthony J. Thibault" Date: Mon, 6 Feb 2017 15:13:25 -0800 Subject: [PATCH 40/41] Fix for findEntities not returning some entities that are within range. This bug has been around a long time... introduced by commit 39ed7f7b in 2014. This became apparent to me when testing the tablet-ui, there was a case where the hand styluses were not appearing when close to a web entity. Because findEntities is such a fundamental feature used by handControllerGrab, this fix should make all grabbing feel more consistent and predictable. I added a unit test that reproduces the issue. --- libraries/shared/src/AABox.cpp | 2 +- tests/shared/src/AABoxTests.cpp | 14 ++++++++++++++ tests/shared/src/AABoxTests.h | 1 + 3 files changed, 16 insertions(+), 1 deletion(-) diff --git a/libraries/shared/src/AABox.cpp b/libraries/shared/src/AABox.cpp index 4a74fb4033..89d5ce709d 100644 --- a/libraries/shared/src/AABox.cpp +++ b/libraries/shared/src/AABox.cpp @@ -360,7 +360,7 @@ glm::vec3 AABox::getClosestPointOnFace(const glm::vec3& point, BoxFace face) con case MIN_Z_FACE: return glm::clamp(point, glm::vec3(_corner.x, _corner.y, _corner.z), - glm::vec3(_corner.x + _scale.z, _corner.y + _scale.y, _corner.z)); + glm::vec3(_corner.x + _scale.x, _corner.y + _scale.y, _corner.z)); default: //quiet windows warnings case MAX_Z_FACE: diff --git a/tests/shared/src/AABoxTests.cpp b/tests/shared/src/AABoxTests.cpp index b9ab95bb09..2e9dfab497 100644 --- a/tests/shared/src/AABoxTests.cpp +++ b/tests/shared/src/AABoxTests.cpp @@ -169,3 +169,17 @@ void AABoxTests::testScale() { box3 += glm::vec3(-1.0f, -1.0f, -1.0f); QCOMPARE(box3.contains(glm::vec3(0.5f, 0.5f, 0.5f)), true); } + +void AABoxTests::testFindSpherePenetration() { + vec3 searchPosition(-0.0141186f, 0.0640736f, -0.116081f); + float searchRadius = 0.5f; + + vec3 boxMin(-0.800014f, -0.450025f, -0.00503815f); + vec3 boxDim(1.60003f, 0.900049f, 0.0100763f); + AABox testBox(boxMin, boxDim); + + vec3 penetration; + bool hit = testBox.findSpherePenetration(searchPosition, searchRadius, penetration); + QCOMPARE(hit, true); +} + diff --git a/tests/shared/src/AABoxTests.h b/tests/shared/src/AABoxTests.h index c777f8e94f..605db7d3ca 100644 --- a/tests/shared/src/AABoxTests.h +++ b/tests/shared/src/AABoxTests.h @@ -24,6 +24,7 @@ private slots: void testContainsPoint(); void testTouchesSphere(); void testScale(); + void testFindSpherePenetration(); }; #endif // hifi_AABoxTests_h From e3af5032c0281e46dc25020525cb49c4905800a2 Mon Sep 17 00:00:00 2001 From: Seth Alves Date: Mon, 6 Feb 2017 16:49:26 -0800 Subject: [PATCH 41/41] send UserActivityLogger events when tablet is opened or closed --- .../src/UserActivityLoggerScriptingInterface.cpp | 8 ++++++++ .../networking/src/UserActivityLoggerScriptingInterface.h | 2 ++ scripts/system/tablet-ui/tabletUI.js | 5 +++-- 3 files changed, 13 insertions(+), 2 deletions(-) diff --git a/libraries/networking/src/UserActivityLoggerScriptingInterface.cpp b/libraries/networking/src/UserActivityLoggerScriptingInterface.cpp index 02d1711230..f38d24c31f 100644 --- a/libraries/networking/src/UserActivityLoggerScriptingInterface.cpp +++ b/libraries/networking/src/UserActivityLoggerScriptingInterface.cpp @@ -16,6 +16,14 @@ void UserActivityLoggerScriptingInterface::enabledEdit() { logAction("enabled_edit"); } +void UserActivityLoggerScriptingInterface::openedTablet() { + logAction("opened_tablet"); +} + +void UserActivityLoggerScriptingInterface::closedTablet() { + logAction("closed_tablet"); +} + void UserActivityLoggerScriptingInterface::openedMarketplace() { logAction("opened_marketplace"); } diff --git a/libraries/networking/src/UserActivityLoggerScriptingInterface.h b/libraries/networking/src/UserActivityLoggerScriptingInterface.h index a202858a1c..b827b2262a 100644 --- a/libraries/networking/src/UserActivityLoggerScriptingInterface.h +++ b/libraries/networking/src/UserActivityLoggerScriptingInterface.h @@ -21,6 +21,8 @@ class UserActivityLoggerScriptingInterface : public QObject, public Dependency { Q_OBJECT public: Q_INVOKABLE void enabledEdit(); + Q_INVOKABLE void openedTablet(); + Q_INVOKABLE void closedTablet(); Q_INVOKABLE void openedMarketplace(); Q_INVOKABLE void toggledAway(bool isAway); Q_INVOKABLE void tutorialProgress(QString stepName, int stepNumber, float secondsToComplete, diff --git a/scripts/system/tablet-ui/tabletUI.js b/scripts/system/tablet-ui/tabletUI.js index eab3d85adc..dc1d71f402 100644 --- a/scripts/system/tablet-ui/tabletUI.js +++ b/scripts/system/tablet-ui/tabletUI.js @@ -12,7 +12,7 @@ // See the accompanying file LICENSE or http://www.apache.org/licenses/LICENSE-2.0.html // -/* global Script, HMD, WebTablet, UIWebTablet */ +/* global Script, HMD, WebTablet, UIWebTablet, UserActivityLogger, Settings, Entities, Messages, Tablet, Overlays, MyAvatar */ (function() { // BEGIN LOCAL_SCOPE var tabletShown = false; @@ -65,8 +65,10 @@ hideTabletUI(); HMD.closeTablet(); } else if (HMD.showTablet && !tabletShown) { + UserActivityLogger.openedTablet(); showTabletUI(); } else if (!HMD.showTablet && tabletShown) { + UserActivityLogger.closedTablet(); hideTabletUI(); } } @@ -86,7 +88,6 @@ var accumulatedLevel = 0.0; // Note: Might have to tweak the following two based on the rate we're getting the data var AVERAGING_RATIO = 0.05; - var MIC_LEVEL_UPDATE_INTERVAL_MS = 100; // Calculate microphone level with the same scaling equation (log scale, exponentially averaged) in AvatarInputs and pal.js function getMicLevel() {