From 6a1c76d14d7d5a189f70c7fc66ef313b8bca7054 Mon Sep 17 00:00:00 2001 From: Simon Walton Date: Fri, 14 Sep 2018 12:19:55 -0700 Subject: [PATCH 01/22] Only sort an estimated number of avatars --- assignment-client/src/avatars/AvatarMixerSlave.cpp | 10 +++++++++- libraries/shared/src/PrioritySortUtil.h | 10 ++++++++-- 2 files changed, 17 insertions(+), 3 deletions(-) diff --git a/assignment-client/src/avatars/AvatarMixerSlave.cpp b/assignment-client/src/avatars/AvatarMixerSlave.cpp index 7368db0c31..e17c108775 100644 --- a/assignment-client/src/avatars/AvatarMixerSlave.cpp +++ b/assignment-client/src/avatars/AvatarMixerSlave.cpp @@ -244,6 +244,9 @@ void AvatarMixerSlave::broadcastAvatarDataToAgent(const SharedNodePointer& node) // reset the internal state for correct random number distribution distribution.reset(); + // Base number to sort on number previously sent. + const int numToSendEst = std::max(nodeData->getNumAvatarsSentLastFrame() * 2, 20); + // reset the number of sent avatars nodeData->resetNumAvatarsSentLastFrame(); @@ -399,7 +402,7 @@ void AvatarMixerSlave::broadcastAvatarDataToAgent(const SharedNodePointer& node) int remainingAvatars = (int)sortedAvatars.size(); auto traitsPacketList = NLPacketList::create(PacketType::BulkAvatarTraits, QByteArray(), true, true); - const auto& sortedAvatarVector = sortedAvatars.getSortedVector(); + const auto& sortedAvatarVector = sortedAvatars.getSortedVector(numToSendEst); for (const auto& sortedAvatar : sortedAvatarVector) { const Node* otherNode = sortedAvatar.getNode(); auto lastEncodeForOther = sortedAvatar.getTimestamp(); @@ -524,6 +527,11 @@ void AvatarMixerSlave::broadcastAvatarDataToAgent(const SharedNodePointer& node) remainingAvatars--; } + if (nodeData->getNumAvatarsSentLastFrame() > numToSendEst) { + qCWarning(avatars) << "More avatars sent than upper estimate" << nodeData->getNumAvatarsSentLastFrame() + << " / " << numToSendEst; + } + quint64 startPacketSending = usecTimestampNow(); // close the current packet so that we're always sending something diff --git a/libraries/shared/src/PrioritySortUtil.h b/libraries/shared/src/PrioritySortUtil.h index 27f6b193ba..3a40fead71 100644 --- a/libraries/shared/src/PrioritySortUtil.h +++ b/libraries/shared/src/PrioritySortUtil.h @@ -66,8 +66,14 @@ namespace PrioritySortUtil { void reserve(size_t num) { _vector.reserve(num); } - const std::vector& getSortedVector() { - std::sort(_vector.begin(), _vector.end(), [](const T& left, const T& right) { return left.getPriority() > right.getPriority(); }); + const std::vector& getSortedVector(int numToSort = 0) { + if (numToSort == 0 || numToSort >= (int)_vector.size()) { + std::sort(_vector.begin(), _vector.end(), + [](const T& left, const T& right) { return left.getPriority() > right.getPriority(); }); + } else { + std::partial_sort(_vector.begin(), _vector.begin() + numToSort, _vector.end(), + [](const T& left, const T& right) { return left.getPriority() > right.getPriority(); }); + } return _vector; } From ca300db410eae5a8fef6ab6143ea415e0a099dae Mon Sep 17 00:00:00 2001 From: Simon Walton Date: Fri, 14 Sep 2018 18:24:08 -0700 Subject: [PATCH 02/22] Send only avatar data that will fit in packet (WIP) --- .../src/avatars/AvatarMixerSlave.cpp | 16 +- libraries/avatars/src/AvatarData.cpp | 222 ++++++++++-------- libraries/avatars/src/AvatarData.h | 2 +- 3 files changed, 137 insertions(+), 103 deletions(-) diff --git a/assignment-client/src/avatars/AvatarMixerSlave.cpp b/assignment-client/src/avatars/AvatarMixerSlave.cpp index e17c108775..20d85a3d4d 100644 --- a/assignment-client/src/avatars/AvatarMixerSlave.cpp +++ b/assignment-client/src/avatars/AvatarMixerSlave.cpp @@ -460,13 +460,13 @@ void AvatarMixerSlave::broadcastAvatarDataToAgent(const SharedNodePointer& node) bool distanceAdjust = true; glm::vec3 viewerPosition = myPosition; - AvatarDataPacket::HasFlags hasFlagsOut; // the result of the toByteArray + AvatarDataPacket::HasFlags hasFlagsOut = 0; // the result of the toByteArray bool dropFaceTracking = false; auto startSerialize = chrono::high_resolution_clock::now(); QByteArray bytes = otherAvatar->toByteArray(detail, lastEncodeForOther, lastSentJointsForOther, hasFlagsOut, dropFaceTracking, distanceAdjust, viewerPosition, - &lastSentJointsForOther); + &lastSentJointsForOther, maxAvatarDataBytes); auto endSerialize = chrono::high_resolution_clock::now(); _stats.toByteArrayElapsedTime += (quint64) chrono::duration_cast(endSerialize - startSerialize).count(); @@ -477,16 +477,16 @@ void AvatarMixerSlave::broadcastAvatarDataToAgent(const SharedNodePointer& node) dropFaceTracking = true; // first try dropping the facial data bytes = otherAvatar->toByteArray(detail, lastEncodeForOther, lastSentJointsForOther, - hasFlagsOut, dropFaceTracking, distanceAdjust, viewerPosition, &lastSentJointsForOther); + hasFlagsOut, dropFaceTracking, distanceAdjust, viewerPosition, &lastSentJointsForOther, maxAvatarDataBytes); if (bytes.size() > maxAvatarDataBytes) { qCWarning(avatars) << "otherAvatar.toByteArray() for" << otherNode->getUUID() << "without facial data resulted in very large buffer of" << bytes.size() << "bytes - reducing to MinimumData"; bytes = otherAvatar->toByteArray(AvatarData::MinimumData, lastEncodeForOther, lastSentJointsForOther, - hasFlagsOut, dropFaceTracking, distanceAdjust, viewerPosition, &lastSentJointsForOther); + hasFlagsOut, dropFaceTracking, distanceAdjust, viewerPosition, &lastSentJointsForOther, maxAvatarDataBytes); - if (bytes.size() > maxAvatarDataBytes) { + if (bytes.size() > maxAvatarDataBytes, maxAvatarDataBytes) { qCWarning(avatars) << "otherAvatar.toByteArray() for" << otherNode->getUUID() << "MinimumData resulted in very large buffer of" << bytes.size() << "bytes - refusing to send avatar"; @@ -604,7 +604,7 @@ void AvatarMixerSlave::broadcastAvatarDataToDownstreamMixer(const SharedNodePoin QVector emptyLastJointSendData { otherAvatar->getJointCount() }; QByteArray avatarByteArray = otherAvatar->toByteArray(AvatarData::SendAllData, 0, emptyLastJointSendData, - flagsOut, false, false, glm::vec3(0), nullptr); + flagsOut, false, false, glm::vec3(0), nullptr, 0); quint64 end = usecTimestampNow(); _stats.toByteArrayElapsedTime += (end - start); @@ -630,14 +630,14 @@ void AvatarMixerSlave::broadcastAvatarDataToDownstreamMixer(const SharedNodePoin << "-" << avatarByteArray.size() << "bytes"; avatarByteArray = otherAvatar->toByteArray(AvatarData::SendAllData, 0, emptyLastJointSendData, - flagsOut, true, false, glm::vec3(0), nullptr); + flagsOut, true, false, glm::vec3(0), nullptr, 0); if (avatarByteArray.size() > maxAvatarByteArraySize) { qCWarning(avatars) << "Replicated avatar data without facial data still too large for" << otherAvatar->getSessionUUID() << "-" << avatarByteArray.size() << "bytes"; avatarByteArray = otherAvatar->toByteArray(AvatarData::MinimumData, 0, emptyLastJointSendData, - flagsOut, true, false, glm::vec3(0), nullptr); + flagsOut, true, false, glm::vec3(0), nullptr, 0); } } diff --git a/libraries/avatars/src/AvatarData.cpp b/libraries/avatars/src/AvatarData.cpp index 834754e228..fdd5317bce 100644 --- a/libraries/avatars/src/AvatarData.cpp +++ b/libraries/avatars/src/AvatarData.cpp @@ -227,13 +227,13 @@ QByteArray AvatarData::toByteArrayStateful(AvatarDataDetail dataDetail, bool dro _lastToByteArray = usecTimestampNow(); return AvatarData::toByteArray(dataDetail, lastSentTime, getLastSentJointData(), hasFlagsOut, dropFaceTracking, false, glm::vec3(0), nullptr, - &_outboundDataRate); + 0, &_outboundDataRate); } QByteArray AvatarData::toByteArray(AvatarDataDetail dataDetail, quint64 lastSentTime, const QVector& lastSentJointData, - AvatarDataPacket::HasFlags& hasFlagsOut, bool dropFaceTracking, bool distanceAdjust, - glm::vec3 viewerPosition, QVector* sentJointDataOut, AvatarDataRate* outboundDataRateOut) const { + AvatarDataPacket::HasFlags& itemFlags, bool dropFaceTracking, bool distanceAdjust, + glm::vec3 viewerPosition, QVector* sentJointDataOut, int maxDataSize, AvatarDataRate* outboundDataRateOut) const { bool cullSmallChanges = (dataDetail == CullSmallData); bool sendAll = (dataDetail == SendAllData); @@ -270,22 +270,6 @@ QByteArray AvatarData::toByteArray(AvatarDataDetail dataDetail, quint64 lastSent auto parentID = getParentID(); - bool hasAvatarGlobalPosition = true; // always include global position - bool hasAvatarOrientation = false; - bool hasAvatarBoundingBox = false; - bool hasAvatarScale = false; - bool hasLookAtPosition = false; - bool hasAudioLoudness = false; - bool hasSensorToWorldMatrix = false; - bool hasAdditionalFlags = false; - - // local position, and parent info only apply to avatars that are parented. The local position - // and the parent info can change independently though, so we track their "changed since" - // separately - bool hasParentInfo = false; - bool hasAvatarLocalPosition = false; - - bool hasFaceTrackerInfo = false; bool hasJointData = false; bool hasJointDefaultPoseFlags = false; bool hasGrabJoints = false; @@ -294,80 +278,121 @@ QByteArray AvatarData::toByteArray(AvatarDataDetail dataDetail, quint64 lastSent glm::mat4 rightFarGrabMatrix; glm::mat4 mouseFarGrabMatrix; - if (sendPALMinimum) { - hasAudioLoudness = true; - } else { - hasAvatarOrientation = sendAll || rotationChangedSince(lastSentTime); - hasAvatarBoundingBox = sendAll || avatarBoundingBoxChangedSince(lastSentTime); - hasAvatarScale = sendAll || avatarScaleChangedSince(lastSentTime); - hasLookAtPosition = sendAll || lookAtPositionChangedSince(lastSentTime); - hasAudioLoudness = sendAll || audioLoudnessChangedSince(lastSentTime); - hasSensorToWorldMatrix = sendAll || sensorToWorldMatrixChangedSince(lastSentTime); - hasAdditionalFlags = sendAll || additionalFlagsChangedSince(lastSentTime); - hasParentInfo = sendAll || parentInfoChangedSince(lastSentTime); - hasAvatarLocalPosition = hasParent() && (sendAll || - tranlationChangedSince(lastSentTime) || - parentInfoChangedSince(lastSentTime)); + // Leading flags, to indicate how much data is actually included in the packet... + AvatarDataPacket::HasFlags packetStateFlags = 0; - hasFaceTrackerInfo = !dropFaceTracking && (hasFaceTracker() || getHasScriptedBlendshapes()) && - (sendAll || faceTrackerInfoChangedSince(lastSentTime)); - hasJointData = sendAll || !sendMinimum; - hasJointDefaultPoseFlags = hasJointData; - if (hasJointData) { - bool leftValid; - leftFarGrabMatrix = _farGrabLeftMatrixCache.get(leftValid); - if (!leftValid) { - leftFarGrabMatrix = glm::mat4(); + if (itemFlags == 0) { + bool hasAvatarGlobalPosition = true; // always include global position + bool hasAvatarOrientation = false; + bool hasAvatarBoundingBox = false; + bool hasAvatarScale = false; + bool hasLookAtPosition = false; + bool hasAudioLoudness = false; + bool hasSensorToWorldMatrix = false; + bool hasAdditionalFlags = false; + + // local position, and parent info only apply to avatars that are parented. The local position + // and the parent info can change independently though, so we track their "changed since" + // separately + bool hasParentInfo = false; + bool hasAvatarLocalPosition = false; + + bool hasFaceTrackerInfo = false; + + if (sendPALMinimum) { + hasAudioLoudness = true; + } else { + hasAvatarOrientation = sendAll || rotationChangedSince(lastSentTime); + hasAvatarBoundingBox = sendAll || avatarBoundingBoxChangedSince(lastSentTime); + hasAvatarScale = sendAll || avatarScaleChangedSince(lastSentTime); + hasLookAtPosition = sendAll || lookAtPositionChangedSince(lastSentTime); + hasAudioLoudness = sendAll || audioLoudnessChangedSince(lastSentTime); + hasSensorToWorldMatrix = sendAll || sensorToWorldMatrixChangedSince(lastSentTime); + hasAdditionalFlags = sendAll || additionalFlagsChangedSince(lastSentTime); + hasParentInfo = sendAll || parentInfoChangedSince(lastSentTime); + hasAvatarLocalPosition = hasParent() && (sendAll || + tranlationChangedSince(lastSentTime) || + parentInfoChangedSince(lastSentTime)); + + hasFaceTrackerInfo = !dropFaceTracking && (hasFaceTracker() || getHasScriptedBlendshapes()) && + (sendAll || faceTrackerInfoChangedSince(lastSentTime)); + hasJointData = sendAll || !sendMinimum; + hasJointDefaultPoseFlags = hasJointData; + if (hasJointData) { + bool leftValid; + leftFarGrabMatrix = _farGrabLeftMatrixCache.get(leftValid); + if (!leftValid) { + leftFarGrabMatrix = glm::mat4(); + } + bool rightValid; + rightFarGrabMatrix = _farGrabRightMatrixCache.get(rightValid); + if (!rightValid) { + rightFarGrabMatrix = glm::mat4(); + } + bool mouseValid; + mouseFarGrabMatrix = _farGrabMouseMatrixCache.get(mouseValid); + if (!mouseValid) { + mouseFarGrabMatrix = glm::mat4(); + } + hasGrabJoints = (leftValid || rightValid || mouseValid); } - bool rightValid; - rightFarGrabMatrix = _farGrabRightMatrixCache.get(rightValid); - if (!rightValid) { - rightFarGrabMatrix = glm::mat4(); - } - bool mouseValid; - mouseFarGrabMatrix = _farGrabMouseMatrixCache.get(mouseValid); - if (!mouseValid) { - mouseFarGrabMatrix = glm::mat4(); - } - hasGrabJoints = (leftValid || rightValid || mouseValid); } + + packetStateFlags = + (hasAvatarGlobalPosition ? AvatarDataPacket::PACKET_HAS_AVATAR_GLOBAL_POSITION : 0) + | (hasAvatarBoundingBox ? AvatarDataPacket::PACKET_HAS_AVATAR_BOUNDING_BOX : 0) + | (hasAvatarOrientation ? AvatarDataPacket::PACKET_HAS_AVATAR_ORIENTATION : 0) + | (hasAvatarScale ? AvatarDataPacket::PACKET_HAS_AVATAR_SCALE : 0) + | (hasLookAtPosition ? AvatarDataPacket::PACKET_HAS_LOOK_AT_POSITION : 0) + | (hasAudioLoudness ? AvatarDataPacket::PACKET_HAS_AUDIO_LOUDNESS : 0) + | (hasSensorToWorldMatrix ? AvatarDataPacket::PACKET_HAS_SENSOR_TO_WORLD_MATRIX : 0) + | (hasAdditionalFlags ? AvatarDataPacket::PACKET_HAS_ADDITIONAL_FLAGS : 0) + | (hasParentInfo ? AvatarDataPacket::PACKET_HAS_PARENT_INFO : 0) + | (hasAvatarLocalPosition ? AvatarDataPacket::PACKET_HAS_AVATAR_LOCAL_POSITION : 0) + | (hasFaceTrackerInfo ? AvatarDataPacket::PACKET_HAS_FACE_TRACKER_INFO : 0) + | (hasJointData ? AvatarDataPacket::PACKET_HAS_JOINT_DATA : 0) + | (hasJointDefaultPoseFlags ? AvatarDataPacket::PACKET_HAS_JOINT_DEFAULT_POSE_FLAGS : 0) + | (hasGrabJoints ? AvatarDataPacket::PACKET_HAS_GRAB_JOINTS : 0); + + } else { + packetStateFlags = itemFlags; } const size_t byteArraySize = AvatarDataPacket::MAX_CONSTANT_HEADER_SIZE + - (hasFaceTrackerInfo ? AvatarDataPacket::maxFaceTrackerInfoSize(_headData->getBlendshapeCoefficients().size()) : 0) + - (hasJointData ? AvatarDataPacket::maxJointDataSize(_jointData.size(), hasGrabJoints) : 0) + - (hasJointDefaultPoseFlags ? AvatarDataPacket::maxJointDefaultPoseFlagsSize(_jointData.size()) : 0); + (packetStateFlags & AvatarDataPacket::PACKET_HAS_FACE_TRACKER_INFO ? AvatarDataPacket::maxFaceTrackerInfoSize(_headData->getBlendshapeCoefficients().size()) : 0) + + (packetStateFlags & AvatarDataPacket::PACKET_HAS_JOINT_DATA ? AvatarDataPacket::maxJointDataSize(_jointData.size(), true) : 0) + + (packetStateFlags & AvatarDataPacket::PACKET_HAS_JOINT_DEFAULT_POSE_FLAGS ? AvatarDataPacket::maxJointDefaultPoseFlagsSize(_jointData.size()) : 0); + + if (maxDataSize == 0) { + maxDataSize = (int)byteArraySize; + } QByteArray avatarDataByteArray((int)byteArraySize, 0); unsigned char* destinationBuffer = reinterpret_cast(avatarDataByteArray.data()); unsigned char* startPosition = destinationBuffer; - // Leading flags, to indicate how much data is actually included in the packet... - AvatarDataPacket::HasFlags packetStateFlags = - (hasAvatarGlobalPosition ? AvatarDataPacket::PACKET_HAS_AVATAR_GLOBAL_POSITION : 0) - | (hasAvatarBoundingBox ? AvatarDataPacket::PACKET_HAS_AVATAR_BOUNDING_BOX : 0) - | (hasAvatarOrientation ? AvatarDataPacket::PACKET_HAS_AVATAR_ORIENTATION : 0) - | (hasAvatarScale ? AvatarDataPacket::PACKET_HAS_AVATAR_SCALE : 0) - | (hasLookAtPosition ? AvatarDataPacket::PACKET_HAS_LOOK_AT_POSITION : 0) - | (hasAudioLoudness ? AvatarDataPacket::PACKET_HAS_AUDIO_LOUDNESS : 0) - | (hasSensorToWorldMatrix ? AvatarDataPacket::PACKET_HAS_SENSOR_TO_WORLD_MATRIX : 0) - | (hasAdditionalFlags ? AvatarDataPacket::PACKET_HAS_ADDITIONAL_FLAGS : 0) - | (hasParentInfo ? AvatarDataPacket::PACKET_HAS_PARENT_INFO : 0) - | (hasAvatarLocalPosition ? AvatarDataPacket::PACKET_HAS_AVATAR_LOCAL_POSITION : 0) - | (hasFaceTrackerInfo ? AvatarDataPacket::PACKET_HAS_FACE_TRACKER_INFO : 0) - | (hasJointData ? AvatarDataPacket::PACKET_HAS_JOINT_DATA : 0) - | (hasJointDefaultPoseFlags ? AvatarDataPacket::PACKET_HAS_JOINT_DEFAULT_POSE_FLAGS : 0) - | (hasGrabJoints ? AvatarDataPacket::PACKET_HAS_GRAB_JOINTS : 0); - - memcpy(destinationBuffer, &packetStateFlags, sizeof(packetStateFlags)); + unsigned char * packetFlagsLocation = destinationBuffer; destinationBuffer += sizeof(packetStateFlags); + if (itemFlags == 0) { + itemFlags = packetStateFlags; + } + + AvatarDataPacket::HasFlags includedFlags = 0; + + const unsigned char * packetEnd = destinationBuffer + maxDataSize; + #define AVATAR_MEMCPY(src) \ memcpy(destinationBuffer, &(src), sizeof(src)); \ destinationBuffer += sizeof(src); - if (hasAvatarGlobalPosition) { +// If we want an item and there's sufficient space: +#define IF_AVATAR_SPACE(flag, space) \ + if ((itemFlags & AvatarDataPacket::flag) && (packetEnd - destinationBuffer) >= (space) \ + && (includedFlags |= AvatarDataPacket::flag)) + + IF_AVATAR_SPACE(PACKET_HAS_AVATAR_GLOBAL_POSITION, sizeof _globalPosition) { auto startSection = destinationBuffer; AVATAR_MEMCPY(_globalPosition); @@ -378,7 +403,7 @@ QByteArray AvatarData::toByteArray(AvatarDataDetail dataDetail, quint64 lastSent } } - if (hasAvatarBoundingBox) { + IF_AVATAR_SPACE(PACKET_HAS_AVATAR_BOUNDING_BOX, sizeof _globalBoundingBoxDimensions + sizeof _globalBoundingBoxOffset) { auto startSection = destinationBuffer; AVATAR_MEMCPY(_globalBoundingBoxDimensions); AVATAR_MEMCPY(_globalBoundingBoxOffset); @@ -389,7 +414,7 @@ QByteArray AvatarData::toByteArray(AvatarDataDetail dataDetail, quint64 lastSent } } - if (hasAvatarOrientation) { + IF_AVATAR_SPACE(PACKET_HAS_AVATAR_ORIENTATION, 6) { auto startSection = destinationBuffer; auto localOrientation = getOrientationOutbound(); destinationBuffer += packOrientationQuatToSixBytes(destinationBuffer, localOrientation); @@ -400,7 +425,7 @@ QByteArray AvatarData::toByteArray(AvatarDataDetail dataDetail, quint64 lastSent } } - if (hasAvatarScale) { + IF_AVATAR_SPACE(PACKET_HAS_AVATAR_SCALE, sizeof(AvatarDataPacket::AvatarScale)) { auto startSection = destinationBuffer; auto data = reinterpret_cast(destinationBuffer); auto scale = getDomainLimitedScale(); @@ -413,7 +438,7 @@ QByteArray AvatarData::toByteArray(AvatarDataDetail dataDetail, quint64 lastSent } } - if (hasLookAtPosition) { + IF_AVATAR_SPACE(PACKET_HAS_LOOK_AT_POSITION, sizeof(_headData->getLookAtPosition()) ) { auto startSection = destinationBuffer; AVATAR_MEMCPY(_headData->getLookAtPosition()); int numBytes = destinationBuffer - startSection; @@ -422,7 +447,7 @@ QByteArray AvatarData::toByteArray(AvatarDataDetail dataDetail, quint64 lastSent } } - if (hasAudioLoudness) { + IF_AVATAR_SPACE(PACKET_HAS_AUDIO_LOUDNESS, sizeof(AvatarDataPacket::AudioLoudness)) { auto startSection = destinationBuffer; auto data = reinterpret_cast(destinationBuffer); data->audioLoudness = packFloatGainToByte(getAudioLoudness() / AUDIO_LOUDNESS_SCALE); @@ -434,7 +459,7 @@ QByteArray AvatarData::toByteArray(AvatarDataDetail dataDetail, quint64 lastSent } } - if (hasSensorToWorldMatrix) { + IF_AVATAR_SPACE(PACKET_HAS_SENSOR_TO_WORLD_MATRIX, sizeof(AvatarDataPacket::SensorToWorldMatrix)) { auto startSection = destinationBuffer; auto data = reinterpret_cast(destinationBuffer); glm::mat4 sensorToWorldMatrix = getSensorToWorldMatrix(); @@ -452,7 +477,7 @@ QByteArray AvatarData::toByteArray(AvatarDataDetail dataDetail, quint64 lastSent } } - if (hasAdditionalFlags) { + IF_AVATAR_SPACE(PACKET_HAS_ADDITIONAL_FLAGS, sizeof (uint16_t)) { auto startSection = destinationBuffer; auto data = reinterpret_cast(destinationBuffer); @@ -500,7 +525,7 @@ QByteArray AvatarData::toByteArray(AvatarDataDetail dataDetail, quint64 lastSent } } - if (hasParentInfo) { + IF_AVATAR_SPACE(PACKET_HAS_PARENT_INFO, sizeof(AvatarDataPacket::ParentInfo)) { auto startSection = destinationBuffer; auto parentInfo = reinterpret_cast(destinationBuffer); QByteArray referentialAsBytes = parentID.toRfc4122(); @@ -514,7 +539,7 @@ QByteArray AvatarData::toByteArray(AvatarDataDetail dataDetail, quint64 lastSent } } - if (hasAvatarLocalPosition) { + IF_AVATAR_SPACE(PACKET_HAS_AVATAR_LOCAL_POSITION, sizeof(getLocalPosition()) ) { auto startSection = destinationBuffer; const auto localPosition = getLocalPosition(); AVATAR_MEMCPY(localPosition); @@ -525,11 +550,11 @@ QByteArray AvatarData::toByteArray(AvatarDataDetail dataDetail, quint64 lastSent } } + const auto& blendshapeCoefficients = _headData->getBlendshapeCoefficients(); // If it is connected, pack up the data - if (hasFaceTrackerInfo) { + IF_AVATAR_SPACE(PACKET_HAS_FACE_TRACKER_INFO, blendshapeCoefficients.size() * (int)sizeof(float)) { auto startSection = destinationBuffer; auto faceTrackerInfo = reinterpret_cast(destinationBuffer); - const auto& blendshapeCoefficients = _headData->getBlendshapeCoefficients(); // note: we don't use the blink and average loudness, we just use the numBlendShapes and // compute the procedural info on the client side. faceTrackerInfo->leftEyeBlink = _headData->_leftEyeBlink; @@ -553,26 +578,26 @@ QByteArray AvatarData::toByteArray(AvatarDataDetail dataDetail, quint64 lastSent QReadLocker readLock(&_jointDataLock); jointData = _jointData; } + const int numJoints = jointData.size(); + const int jointBitVectorSize = calcBitVectorSize(numJoints); // If it is connected, pack up the data - if (hasJointData) { + if (packetStateFlags & AvatarDataPacket::PACKET_HAS_JOINT_DATA) { auto startSection = destinationBuffer; // joint rotation data - int numJoints = jointData.size(); *destinationBuffer++ = (uint8_t)numJoints; unsigned char* validityPosition = destinationBuffer; unsigned char validity = 0; int validityBit = 0; - int numValidityBytes = calcBitVectorSize(numJoints); #ifdef WANT_DEBUG int rotationSentCount = 0; unsigned char* beforeRotations = destinationBuffer; #endif - destinationBuffer += numValidityBytes; // Move pointer past the validity bytes + destinationBuffer += jointBitVectorSize; // Move pointer past the validity bytes // sentJointDataOut and lastSentJointData might be the same vector if (sentJointDataOut) { @@ -625,7 +650,7 @@ QByteArray AvatarData::toByteArray(AvatarDataDetail dataDetail, quint64 lastSent unsigned char* beforeTranslations = destinationBuffer; #endif - destinationBuffer += numValidityBytes; // Move pointer past the validity bytes + destinationBuffer += jointBitVectorSize; // Move pointer past the validity bytes float minTranslation = (distanceAdjust && cullSmallChanges) ? getDistanceBasedMinTranslationDistance(viewerPosition) : AVATAR_MIN_TRANSLATION; @@ -718,6 +743,8 @@ QByteArray AvatarData::toByteArray(AvatarDataDetail dataDetail, quint64 lastSent if (outboundDataRateOut) { outboundDataRateOut->farGrabJointRate.increment(numBytes); } + + includedFlags |= AvatarDataPacket::PACKET_HAS_GRAB_JOINTS; } #ifdef WANT_DEBUG @@ -738,13 +765,13 @@ QByteArray AvatarData::toByteArray(AvatarDataDetail dataDetail, quint64 lastSent outboundDataRateOut->jointDataRate.increment(numBytes); } + includedFlags |= AvatarDataPacket::PACKET_HAS_JOINT_DATA; } - - if (hasJointDefaultPoseFlags) { + + IF_AVATAR_SPACE(PACKET_HAS_JOINT_DEFAULT_POSE_FLAGS, 1 + 2 * jointBitVectorSize) { auto startSection = destinationBuffer; // write numJoints - int numJoints = jointData.size(); *destinationBuffer++ = (uint8_t)numJoints; // write rotationIsDefaultPose bits @@ -763,6 +790,10 @@ QByteArray AvatarData::toByteArray(AvatarDataDetail dataDetail, quint64 lastSent } } + memcpy(packetFlagsLocation, &includedFlags, sizeof(includedFlags)); + // Return dropped items. + itemFlags = packetStateFlags & ~includedFlags; + int avatarDataSize = destinationBuffer - startPosition; if (avatarDataSize > (int)byteArraySize) { @@ -771,6 +802,9 @@ QByteArray AvatarData::toByteArray(AvatarDataDetail dataDetail, quint64 lastSent } return avatarDataByteArray.left(avatarDataSize); + +#undef AVATAR_MEMCPY +#undef IF_AVATAR_SPACE } // NOTE: This is never used in a "distanceAdjust" mode, so it's ok that it doesn't use a variable minimum rotation/translation diff --git a/libraries/avatars/src/AvatarData.h b/libraries/avatars/src/AvatarData.h index 056e745370..057b351670 100644 --- a/libraries/avatars/src/AvatarData.h +++ b/libraries/avatars/src/AvatarData.h @@ -450,7 +450,7 @@ public: virtual QByteArray toByteArray(AvatarDataDetail dataDetail, quint64 lastSentTime, const QVector& lastSentJointData, AvatarDataPacket::HasFlags& hasFlagsOut, bool dropFaceTracking, bool distanceAdjust, glm::vec3 viewerPosition, - QVector* sentJointDataOut, AvatarDataRate* outboundDataRateOut = nullptr) const; + QVector* sentJointDataOut, int maxDataSize = 0, AvatarDataRate* outboundDataRateOut = nullptr) const; virtual void doneEncoding(bool cullSmallChanges); From 0c7dee730c60452f262ef507aa89a3d6609d6011 Mon Sep 17 00:00:00 2001 From: Simon Walton Date: Tue, 18 Sep 2018 10:25:59 -0700 Subject: [PATCH 03/22] Pack partial avatar data in bulk packets using space available Keep a single packet in-hand and send when full rather than a packet list. --- .../src/avatars/AvatarMixerSlave.cpp | 60 +++++++++++----- libraries/avatars/src/AvatarData.cpp | 72 ++++++++++--------- 2 files changed, 79 insertions(+), 53 deletions(-) diff --git a/assignment-client/src/avatars/AvatarMixerSlave.cpp b/assignment-client/src/avatars/AvatarMixerSlave.cpp index 20d85a3d4d..c6d8c7f495 100644 --- a/assignment-client/src/avatars/AvatarMixerSlave.cpp +++ b/assignment-client/src/avatars/AvatarMixerSlave.cpp @@ -279,10 +279,6 @@ void AvatarMixerSlave::broadcastAvatarDataToAgent(const SharedNodePointer& node) int minimumBytesPerAvatar = PALIsOpen ? AvatarDataPacket::AVATAR_HAS_FLAGS_SIZE + NUM_BYTES_RFC4122_UUID + sizeof(AvatarDataPacket::AvatarGlobalPosition) + sizeof(AvatarDataPacket::AudioLoudness) : 0; - // setup a PacketList for the avatarPackets - auto avatarPacketList = NLPacketList::create(PacketType::BulkAvatarData); - static auto maxAvatarDataBytes = avatarPacketList->getMaxSegmentSize() - NUM_BYTES_RFC4122_UUID; - // compute node bounding box const float MY_AVATAR_BUBBLE_EXPANSION_FACTOR = 4.0f; // magic number determined emperically AABox nodeBox = computeBubbleBox(avatar, MY_AVATAR_BUBBLE_EXPANSION_FACTOR); @@ -401,6 +397,11 @@ void AvatarMixerSlave::broadcastAvatarDataToAgent(const SharedNodePointer& node) int remainingAvatars = (int)sortedAvatars.size(); auto traitsPacketList = NLPacketList::create(PacketType::BulkAvatarTraits, QByteArray(), true, true); + auto avatarPacket = NLPacket::create(PacketType::BulkAvatarData); + const int avatarPacketCapacity = avatarPacket->getPayloadCapacity(); + int avatarSpaceAvailable = avatarPacketCapacity; + //avatarSpaceAvailable = 100; + int numPacketsSent = 0; const auto& sortedAvatarVector = sortedAvatars.getSortedVector(numToSendEst); for (const auto& sortedAvatar : sortedAvatarVector) { @@ -460,17 +461,40 @@ void AvatarMixerSlave::broadcastAvatarDataToAgent(const SharedNodePointer& node) bool distanceAdjust = true; glm::vec3 viewerPosition = myPosition; - AvatarDataPacket::HasFlags hasFlagsOut = 0; // the result of the toByteArray + AvatarDataPacket::HasFlags includeFlags = 0; // the result of the toByteArray bool dropFaceTracking = false; - auto startSerialize = chrono::high_resolution_clock::now(); - QByteArray bytes = otherAvatar->toByteArray(detail, lastEncodeForOther, lastSentJointsForOther, - hasFlagsOut, dropFaceTracking, distanceAdjust, viewerPosition, - &lastSentJointsForOther, maxAvatarDataBytes); - auto endSerialize = chrono::high_resolution_clock::now(); - _stats.toByteArrayElapsedTime += - (quint64) chrono::duration_cast(endSerialize - startSerialize).count(); + do { + auto startSerialize = chrono::high_resolution_clock::now(); + QByteArray bytes = otherAvatar->toByteArray(detail, lastEncodeForOther, lastSentJointsForOther, + includeFlags, dropFaceTracking, distanceAdjust, viewerPosition, + &lastSentJointsForOther, avatarSpaceAvailable); + auto endSerialize = chrono::high_resolution_clock::now(); + _stats.toByteArrayElapsedTime += + (quint64)chrono::duration_cast(endSerialize - startSerialize).count(); + avatarPacket->write(bytes); + avatarSpaceAvailable -= bytes.size(); + if (includeFlags != 0) { + // Weren't able to fit everything. + nodeList->sendPacket(std::move(avatarPacket), *destinationNode); + ++numPacketsSent; + avatarPacket = NLPacket::create(PacketType::BulkAvatarData); + avatarSpaceAvailable = avatarPacketCapacity; + } + } while (includeFlags != 0); + if (detail != AvatarData::NoData) { + _stats.numOthersIncluded++; + + // increment the number of avatars sent to this reciever + nodeData->incrementNumAvatarsSentLastFrame(); + + // set the last sent sequence number for this sender on the receiver + nodeData->setLastBroadcastSequenceNumber(otherNode->getUUID(), + otherNodeData->getLastReceivedSequenceNumber()); + nodeData->setLastOtherAvatarEncodeTime(otherNode->getUUID(), usecTimestampNow()); + } +#if 0 if (bytes.size() > maxAvatarDataBytes) { qCWarning(avatars) << "otherAvatar.toByteArray() for" << otherNode->getUUID() << "resulted in very large buffer of" << bytes.size() << "bytes - dropping facial data"; @@ -517,6 +541,7 @@ void AvatarMixerSlave::broadcastAvatarDataToAgent(const SharedNodePointer& node) // TODO? this avatar is not included now, and will probably not be included next frame. // It would be nice if we could tweak its future sort priority to put it at the back of the list. } +#endif auto endAvatarDataPacking = chrono::high_resolution_clock::now(); _stats.avatarDataPackingElapsedTime += @@ -534,15 +559,14 @@ void AvatarMixerSlave::broadcastAvatarDataToAgent(const SharedNodePointer& node) quint64 startPacketSending = usecTimestampNow(); - // close the current packet so that we're always sending something - avatarPacketList->closeCurrentPacket(true); + if (avatarPacket->getPayloadSize() != 0) { + nodeList->sendPacket(std::move(avatarPacket), *destinationNode); + ++numPacketsSent; + } - _stats.numPacketsSent += (int)avatarPacketList->getNumPackets(); + _stats.numPacketsSent += numPacketsSent; _stats.numBytesSent += numAvatarDataBytes; - // send the avatar data PacketList - nodeList->sendPacketList(std::move(avatarPacketList), *destinationNode); - // record the bytes sent for other avatar data in the AvatarMixerClientData nodeData->recordSentAvatarData(numAvatarDataBytes); diff --git a/libraries/avatars/src/AvatarData.cpp b/libraries/avatars/src/AvatarData.cpp index fdd5317bce..6e5dc3fc6a 100644 --- a/libraries/avatars/src/AvatarData.cpp +++ b/libraries/avatars/src/AvatarData.cpp @@ -65,7 +65,7 @@ size_t AvatarDataPacket::maxFaceTrackerInfoSize(size_t numBlendshapeCoefficients } size_t AvatarDataPacket::maxJointDataSize(size_t numJoints, bool hasGrabJoints) { - const size_t validityBitsSize = (size_t)std::ceil(numJoints / (float)BITS_IN_BYTE); + const size_t validityBitsSize = (size_t) calcBitVectorSize(numJoints); size_t totalSize = sizeof(uint8_t); // numJoints @@ -270,10 +270,6 @@ QByteArray AvatarData::toByteArray(AvatarDataDetail dataDetail, quint64 lastSent auto parentID = getParentID(); - bool hasJointData = false; - bool hasJointDefaultPoseFlags = false; - bool hasGrabJoints = false; - glm::mat4 leftFarGrabMatrix; glm::mat4 rightFarGrabMatrix; glm::mat4 mouseFarGrabMatrix; @@ -289,6 +285,9 @@ QByteArray AvatarData::toByteArray(AvatarDataDetail dataDetail, quint64 lastSent bool hasLookAtPosition = false; bool hasAudioLoudness = false; bool hasSensorToWorldMatrix = false; + bool hasJointData = false; + bool hasJointDefaultPoseFlags = false; + bool hasGrabJoints = false; bool hasAdditionalFlags = false; // local position, and parent info only apply to avatars that are parented. The local position @@ -354,13 +353,13 @@ QByteArray AvatarData::toByteArray(AvatarDataDetail dataDetail, quint64 lastSent | (hasJointDefaultPoseFlags ? AvatarDataPacket::PACKET_HAS_JOINT_DEFAULT_POSE_FLAGS : 0) | (hasGrabJoints ? AvatarDataPacket::PACKET_HAS_GRAB_JOINTS : 0); + itemFlags = packetStateFlags; } else { packetStateFlags = itemFlags; } - - const size_t byteArraySize = AvatarDataPacket::MAX_CONSTANT_HEADER_SIZE + - (packetStateFlags & AvatarDataPacket::PACKET_HAS_FACE_TRACKER_INFO ? AvatarDataPacket::maxFaceTrackerInfoSize(_headData->getBlendshapeCoefficients().size()) : 0) + + const size_t byteArraySize = AvatarDataPacket::MAX_CONSTANT_HEADER_SIZE + NUM_BYTES_RFC4122_UUID + + AvatarDataPacket::maxFaceTrackerInfoSize(_headData->getBlendshapeCoefficients().size()) + (packetStateFlags & AvatarDataPacket::PACKET_HAS_JOINT_DATA ? AvatarDataPacket::maxJointDataSize(_jointData.size(), true) : 0) + (packetStateFlags & AvatarDataPacket::PACKET_HAS_JOINT_DEFAULT_POSE_FLAGS ? AvatarDataPacket::maxJointDefaultPoseFlagsSize(_jointData.size()) : 0); @@ -371,25 +370,27 @@ QByteArray AvatarData::toByteArray(AvatarDataDetail dataDetail, quint64 lastSent QByteArray avatarDataByteArray((int)byteArraySize, 0); unsigned char* destinationBuffer = reinterpret_cast(avatarDataByteArray.data()); unsigned char* startPosition = destinationBuffer; + const unsigned char * packetEnd = destinationBuffer + maxDataSize; + + AvatarDataPacket::HasFlags includedFlags = 0; + + // Packets always have UUID. + memcpy(destinationBuffer, getSessionUUID().toRfc4122(), NUM_BYTES_RFC4122_UUID); + destinationBuffer += NUM_BYTES_RFC4122_UUID; unsigned char * packetFlagsLocation = destinationBuffer; destinationBuffer += sizeof(packetStateFlags); if (itemFlags == 0) { - itemFlags = packetStateFlags; } - AvatarDataPacket::HasFlags includedFlags = 0; - - const unsigned char * packetEnd = destinationBuffer + maxDataSize; - #define AVATAR_MEMCPY(src) \ memcpy(destinationBuffer, &(src), sizeof(src)); \ destinationBuffer += sizeof(src); // If we want an item and there's sufficient space: -#define IF_AVATAR_SPACE(flag, space) \ - if ((itemFlags & AvatarDataPacket::flag) && (packetEnd - destinationBuffer) >= (space) \ +#define IF_AVATAR_SPACE(flag, space) \ + if ((itemFlags & AvatarDataPacket::flag) && (int)(packetEnd - destinationBuffer) >= (space) \ && (includedFlags |= AvatarDataPacket::flag)) IF_AVATAR_SPACE(PACKET_HAS_AVATAR_GLOBAL_POSITION, sizeof _globalPosition) { @@ -414,7 +415,7 @@ QByteArray AvatarData::toByteArray(AvatarDataDetail dataDetail, quint64 lastSent } } - IF_AVATAR_SPACE(PACKET_HAS_AVATAR_ORIENTATION, 6) { + IF_AVATAR_SPACE(PACKET_HAS_AVATAR_ORIENTATION, sizeof(AvatarDataPacket::SixByteQuat)) { auto startSection = destinationBuffer; auto localOrientation = getOrientationOutbound(); destinationBuffer += packOrientationQuatToSixBytes(destinationBuffer, localOrientation); @@ -552,7 +553,7 @@ QByteArray AvatarData::toByteArray(AvatarDataDetail dataDetail, quint64 lastSent const auto& blendshapeCoefficients = _headData->getBlendshapeCoefficients(); // If it is connected, pack up the data - IF_AVATAR_SPACE(PACKET_HAS_FACE_TRACKER_INFO, blendshapeCoefficients.size() * (int)sizeof(float)) { + IF_AVATAR_SPACE(PACKET_HAS_FACE_TRACKER_INFO, sizeof(AvatarDataPacket::FaceTrackerInfo) + blendshapeCoefficients.size() * sizeof(float)) { auto startSection = destinationBuffer; auto faceTrackerInfo = reinterpret_cast(destinationBuffer); // note: we don't use the blink and average loudness, we just use the numBlendShapes and @@ -574,15 +575,16 @@ QByteArray AvatarData::toByteArray(AvatarDataDetail dataDetail, quint64 lastSent } QVector jointData; - if (hasJointData || hasJointDefaultPoseFlags) { + if (packetStateFlags & (AvatarDataPacket::PACKET_HAS_JOINT_DATA | AvatarDataPacket::PACKET_HAS_JOINT_DEFAULT_POSE_FLAGS)) { QReadLocker readLock(&_jointDataLock); jointData = _jointData; } const int numJoints = jointData.size(); const int jointBitVectorSize = calcBitVectorSize(numJoints); - // If it is connected, pack up the data - if (packetStateFlags & AvatarDataPacket::PACKET_HAS_JOINT_DATA) { + // Check against full size or minimum size: count + two bit-vectors + two controllers + const int approxJointSpace = sendAll ? AvatarDataPacket::maxJointDataSize(numJoints, true) : 1 + 2 * jointBitVectorSize + 2 * (6 + 2); + IF_AVATAR_SPACE(PACKET_HAS_JOINT_DATA, approxJointSpace) { auto startSection = destinationBuffer; // joint rotation data @@ -611,8 +613,8 @@ QByteArray AvatarData::toByteArray(AvatarDataDetail dataDetail, quint64 lastSent const JointData& last = lastSentJointData[i]; if (!data.rotationIsDefaultPose) { - // The dot product for larger rotations is a lower number. - // So if the dot() is less than the value, then the rotation is a larger angle of rotation + // The dot product for larger rotations is a lower number, + // so if the dot() is less than the value, then the rotation is a larger angle of rotation if (sendAll || last.rotationIsDefaultPose || (!cullSmallChanges && last.rotation != data.rotation) || (cullSmallChanges && glm::dot(last.rotation, data.rotation) < minRotationDOT) ) { validity |= (1 << validityBit); @@ -705,7 +707,8 @@ QByteArray AvatarData::toByteArray(AvatarDataDetail dataDetail, quint64 lastSent destinationBuffer += packFloatVec3ToSignedTwoByteFixed(destinationBuffer, controllerRightHandTransform.getTranslation(), TRANSLATION_COMPRESSION_RADIX); - if (hasGrabJoints) { + int numGrabJointBytes = 0; + IF_AVATAR_SPACE(PACKET_HAS_GRAB_JOINTS, sizeof (AvatarDataPacket::FarGrabJoints)) { // the far-grab joints may range further than 3 meters, so we can't use packFloatVec3ToSignedTwoByteFixed etc auto startSection = destinationBuffer; auto data = reinterpret_cast(destinationBuffer); @@ -738,13 +741,7 @@ QByteArray AvatarData::toByteArray(AvatarDataDetail dataDetail, quint64 lastSent data->mouseFarGrabRotation[3] = mouseFarGrabRotation.z; destinationBuffer += sizeof(data->mouseFarGrabRotation); - int numBytes = destinationBuffer - startSection; - - if (outboundDataRateOut) { - outboundDataRateOut->farGrabJointRate.increment(numBytes); - } - - includedFlags |= AvatarDataPacket::PACKET_HAS_GRAB_JOINTS; + int numGrabJointBytes = destinationBuffer - startSection; } #ifdef WANT_DEBUG @@ -760,12 +757,17 @@ QByteArray AvatarData::toByteArray(AvatarDataDetail dataDetail, quint64 lastSent } #endif - int numBytes = destinationBuffer - startSection; - if (outboundDataRateOut) { - outboundDataRateOut->jointDataRate.increment(numBytes); + if (destinationBuffer >= packetEnd) { + // Joint data too large - revert + destinationBuffer = startSection; + includedFlags &= ~(AvatarDataPacket::PACKET_HAS_GRAB_JOINTS | AvatarDataPacket::PACKET_HAS_JOINT_DATA); + } else { + int numBytes = destinationBuffer - startSection; + if (outboundDataRateOut) { + outboundDataRateOut->jointDataRate.increment(numBytes); + outboundDataRateOut->farGrabJointRate.increment(numGrabJointBytes); + } } - - includedFlags |= AvatarDataPacket::PACKET_HAS_JOINT_DATA; } IF_AVATAR_SPACE(PACKET_HAS_JOINT_DEFAULT_POSE_FLAGS, 1 + 2 * jointBitVectorSize) { From 5d91396e91320b354fc24b5e1be30cd2ade97755 Mon Sep 17 00:00:00 2001 From: Simon Walton Date: Tue, 18 Sep 2018 17:49:23 -0700 Subject: [PATCH 04/22] Fixes for client & other clean-up Handle grab-joints better; fix packet size calc; remove dead code; other improvements. --- .../src/avatars/AvatarMixerSlave.cpp | 60 ++------------ libraries/avatars/src/AvatarData.cpp | 82 +++++++++++-------- 2 files changed, 52 insertions(+), 90 deletions(-) diff --git a/assignment-client/src/avatars/AvatarMixerSlave.cpp b/assignment-client/src/avatars/AvatarMixerSlave.cpp index c6d8c7f495..2aa8de6cf4 100644 --- a/assignment-client/src/avatars/AvatarMixerSlave.cpp +++ b/assignment-client/src/avatars/AvatarMixerSlave.cpp @@ -261,7 +261,6 @@ void AvatarMixerSlave::broadcastAvatarDataToAgent(const SharedNodePointer& node) // max number of avatarBytes per frame int maxAvatarBytesPerFrame = int(_maxKbpsPerNode * BYTES_PER_KILOBIT / AVATAR_MIXER_BROADCAST_FRAMES_PER_SECOND); - // keep track of the number of other avatars held back in this frame int numAvatarsHeldBack = 0; @@ -457,23 +456,22 @@ void AvatarMixerSlave::broadcastAvatarDataToAgent(const SharedNodePointer& node) bool includeThisAvatar = true; QVector& lastSentJointsForOther = nodeData->getLastOtherAvatarSentJoints(otherNode->getUUID()); - lastSentJointsForOther.resize(otherAvatar->getJointCount()); - - bool distanceAdjust = true; - glm::vec3 viewerPosition = myPosition; + const bool distanceAdjust = true; + const bool dropFaceTracking = false; AvatarDataPacket::HasFlags includeFlags = 0; // the result of the toByteArray - bool dropFaceTracking = false; do { auto startSerialize = chrono::high_resolution_clock::now(); QByteArray bytes = otherAvatar->toByteArray(detail, lastEncodeForOther, lastSentJointsForOther, - includeFlags, dropFaceTracking, distanceAdjust, viewerPosition, + includeFlags, dropFaceTracking, distanceAdjust, myPosition, &lastSentJointsForOther, avatarSpaceAvailable); auto endSerialize = chrono::high_resolution_clock::now(); _stats.toByteArrayElapsedTime += (quint64)chrono::duration_cast(endSerialize - startSerialize).count(); + avatarPacket->write(bytes); avatarSpaceAvailable -= bytes.size(); + numAvatarDataBytes += bytes.size(); if (includeFlags != 0) { // Weren't able to fit everything. nodeList->sendPacket(std::move(avatarPacket), *destinationNode); @@ -494,54 +492,6 @@ void AvatarMixerSlave::broadcastAvatarDataToAgent(const SharedNodePointer& node) otherNodeData->getLastReceivedSequenceNumber()); nodeData->setLastOtherAvatarEncodeTime(otherNode->getUUID(), usecTimestampNow()); } -#if 0 - if (bytes.size() > maxAvatarDataBytes) { - qCWarning(avatars) << "otherAvatar.toByteArray() for" << otherNode->getUUID() - << "resulted in very large buffer of" << bytes.size() << "bytes - dropping facial data"; - - dropFaceTracking = true; // first try dropping the facial data - bytes = otherAvatar->toByteArray(detail, lastEncodeForOther, lastSentJointsForOther, - hasFlagsOut, dropFaceTracking, distanceAdjust, viewerPosition, &lastSentJointsForOther, maxAvatarDataBytes); - - if (bytes.size() > maxAvatarDataBytes) { - qCWarning(avatars) << "otherAvatar.toByteArray() for" << otherNode->getUUID() - << "without facial data resulted in very large buffer of" << bytes.size() - << "bytes - reducing to MinimumData"; - bytes = otherAvatar->toByteArray(AvatarData::MinimumData, lastEncodeForOther, lastSentJointsForOther, - hasFlagsOut, dropFaceTracking, distanceAdjust, viewerPosition, &lastSentJointsForOther, maxAvatarDataBytes); - - if (bytes.size() > maxAvatarDataBytes, maxAvatarDataBytes) { - qCWarning(avatars) << "otherAvatar.toByteArray() for" << otherNode->getUUID() - << "MinimumData resulted in very large buffer of" << bytes.size() - << "bytes - refusing to send avatar"; - includeThisAvatar = false; - } - } - } - - if (includeThisAvatar) { - // start a new segment in the PacketList for this avatar - avatarPacketList->startSegment(); - numAvatarDataBytes += avatarPacketList->write(otherNode->getUUID().toRfc4122()); - numAvatarDataBytes += avatarPacketList->write(bytes); - avatarPacketList->endSegment(); - - if (detail != AvatarData::NoData) { - _stats.numOthersIncluded++; - - // increment the number of avatars sent to this reciever - nodeData->incrementNumAvatarsSentLastFrame(); - - // set the last sent sequence number for this sender on the receiver - nodeData->setLastBroadcastSequenceNumber(otherNode->getUUID(), - otherNodeData->getLastReceivedSequenceNumber()); - nodeData->setLastOtherAvatarEncodeTime(otherNode->getUUID(), usecTimestampNow()); - } - } else { - // TODO? this avatar is not included now, and will probably not be included next frame. - // It would be nice if we could tweak its future sort priority to put it at the back of the list. - } -#endif auto endAvatarDataPacking = chrono::high_resolution_clock::now(); _stats.avatarDataPackingElapsedTime += diff --git a/libraries/avatars/src/AvatarData.cpp b/libraries/avatars/src/AvatarData.cpp index 6e5dc3fc6a..2b46eb9cdd 100644 --- a/libraries/avatars/src/AvatarData.cpp +++ b/libraries/avatars/src/AvatarData.cpp @@ -65,7 +65,7 @@ size_t AvatarDataPacket::maxFaceTrackerInfoSize(size_t numBlendshapeCoefficients } size_t AvatarDataPacket::maxJointDataSize(size_t numJoints, bool hasGrabJoints) { - const size_t validityBitsSize = (size_t) calcBitVectorSize(numJoints); + const size_t validityBitsSize = calcBitVectorSize((int)numJoints); size_t totalSize = sizeof(uint8_t); // numJoints @@ -74,9 +74,9 @@ size_t AvatarDataPacket::maxJointDataSize(size_t numJoints, bool hasGrabJoints) totalSize += validityBitsSize; // Translations mask totalSize += numJoints * sizeof(SixByteTrans); // Translations - size_t NUM_FAUX_JOINT = 2; - size_t num_grab_joints = (hasGrabJoints ? 2 : 0); - totalSize += (NUM_FAUX_JOINT + num_grab_joints) * (sizeof(SixByteQuat) + sizeof(SixByteTrans)); // faux joints + static const size_t NUM_FAUX_JOINT = 2; + totalSize += NUM_FAUX_JOINT * (sizeof(SixByteQuat) + sizeof(SixByteTrans)); // faux joints + totalSize += hasGrabJoints ? sizeof(FarGrabJoints) : 0; return totalSize; } @@ -222,12 +222,14 @@ float AvatarData::getDistanceBasedMinTranslationDistance(glm::vec3 viewerPositio // we want to track outbound data in this case... QByteArray AvatarData::toByteArrayStateful(AvatarDataDetail dataDetail, bool dropFaceTracking) { - AvatarDataPacket::HasFlags hasFlagsOut; + AvatarDataPacket::HasFlags hasFlagsOut = 0; auto lastSentTime = _lastToByteArray; _lastToByteArray = usecTimestampNow(); - return AvatarData::toByteArray(dataDetail, lastSentTime, getLastSentJointData(), + auto avatarByteArray = AvatarData::toByteArray(dataDetail, lastSentTime, getLastSentJointData(), hasFlagsOut, dropFaceTracking, false, glm::vec3(0), nullptr, 0, &_outboundDataRate); + // Strip UUID + return avatarByteArray.right(avatarByteArray.size() - NUM_BYTES_RFC4122_UUID); } QByteArray AvatarData::toByteArray(AvatarDataDetail dataDetail, quint64 lastSentTime, @@ -245,7 +247,9 @@ QByteArray AvatarData::toByteArray(AvatarDataDetail dataDetail, quint64 lastSent // special case, if we were asked for no data, then just include the flags all set to nothing if (dataDetail == NoData) { AvatarDataPacket::HasFlags packetStateFlags = 0; - QByteArray avatarDataByteArray(reinterpret_cast(&packetStateFlags), sizeof(packetStateFlags)); + + QByteArray avatarDataByteArray(getSessionUUID().toRfc4122().data(), NUM_BYTES_RFC4122_UUID + sizeof(packetStateFlags)); + avatarDataByteArray.append((char*) &packetStateFlags, sizeof packetStateFlags); return avatarDataByteArray; } @@ -287,7 +291,6 @@ QByteArray AvatarData::toByteArray(AvatarDataDetail dataDetail, quint64 lastSent bool hasSensorToWorldMatrix = false; bool hasJointData = false; bool hasJointDefaultPoseFlags = false; - bool hasGrabJoints = false; bool hasAdditionalFlags = false; // local position, and parent info only apply to avatars that are parented. The local position @@ -315,26 +318,8 @@ QByteArray AvatarData::toByteArray(AvatarDataDetail dataDetail, quint64 lastSent hasFaceTrackerInfo = !dropFaceTracking && (hasFaceTracker() || getHasScriptedBlendshapes()) && (sendAll || faceTrackerInfoChangedSince(lastSentTime)); - hasJointData = sendAll || !sendMinimum; + hasJointData = !sendMinimum; hasJointDefaultPoseFlags = hasJointData; - if (hasJointData) { - bool leftValid; - leftFarGrabMatrix = _farGrabLeftMatrixCache.get(leftValid); - if (!leftValid) { - leftFarGrabMatrix = glm::mat4(); - } - bool rightValid; - rightFarGrabMatrix = _farGrabRightMatrixCache.get(rightValid); - if (!rightValid) { - rightFarGrabMatrix = glm::mat4(); - } - bool mouseValid; - mouseFarGrabMatrix = _farGrabMouseMatrixCache.get(mouseValid); - if (!mouseValid) { - mouseFarGrabMatrix = glm::mat4(); - } - hasGrabJoints = (leftValid || rightValid || mouseValid); - } } packetStateFlags = @@ -351,17 +336,45 @@ QByteArray AvatarData::toByteArray(AvatarDataDetail dataDetail, quint64 lastSent | (hasFaceTrackerInfo ? AvatarDataPacket::PACKET_HAS_FACE_TRACKER_INFO : 0) | (hasJointData ? AvatarDataPacket::PACKET_HAS_JOINT_DATA : 0) | (hasJointDefaultPoseFlags ? AvatarDataPacket::PACKET_HAS_JOINT_DEFAULT_POSE_FLAGS : 0) - | (hasGrabJoints ? AvatarDataPacket::PACKET_HAS_GRAB_JOINTS : 0); + | (hasJointData ? AvatarDataPacket::PACKET_HAS_GRAB_JOINTS : 0); itemFlags = packetStateFlags; } else { packetStateFlags = itemFlags; + if (packetStateFlags & AvatarDataPacket::PACKET_HAS_JOINT_DATA) { + // Force all joints upon continuation, as deltas aren't valid. + sendAll = true; + } + if (packetStateFlags & AvatarDataPacket::PACKET_HAS_GRAB_JOINTS) { + packetStateFlags |= AvatarDataPacket::PACKET_HAS_JOINT_DATA; + } + } + + if (packetStateFlags & AvatarDataPacket::PACKET_HAS_GRAB_JOINTS) { + bool leftValid; + leftFarGrabMatrix = _farGrabLeftMatrixCache.get(leftValid); + if (!leftValid) { + leftFarGrabMatrix = glm::mat4(); + } + bool rightValid; + rightFarGrabMatrix = _farGrabRightMatrixCache.get(rightValid); + if (!rightValid) { + rightFarGrabMatrix = glm::mat4(); + } + bool mouseValid; + mouseFarGrabMatrix = _farGrabMouseMatrixCache.get(mouseValid); + if (!mouseValid) { + mouseFarGrabMatrix = glm::mat4(); + } + if (!(leftValid || rightValid || mouseValid)) { + packetStateFlags &= ~AvatarDataPacket::PACKET_HAS_GRAB_JOINTS; + } } const size_t byteArraySize = AvatarDataPacket::MAX_CONSTANT_HEADER_SIZE + NUM_BYTES_RFC4122_UUID + AvatarDataPacket::maxFaceTrackerInfoSize(_headData->getBlendshapeCoefficients().size()) + - (packetStateFlags & AvatarDataPacket::PACKET_HAS_JOINT_DATA ? AvatarDataPacket::maxJointDataSize(_jointData.size(), true) : 0) + - (packetStateFlags & AvatarDataPacket::PACKET_HAS_JOINT_DEFAULT_POSE_FLAGS ? AvatarDataPacket::maxJointDefaultPoseFlagsSize(_jointData.size()) : 0); + AvatarDataPacket::maxJointDataSize(_jointData.size(), true) + + AvatarDataPacket::maxJointDefaultPoseFlagsSize(_jointData.size()); if (maxDataSize == 0) { maxDataSize = (int)byteArraySize; @@ -381,16 +394,13 @@ QByteArray AvatarData::toByteArray(AvatarDataDetail dataDetail, quint64 lastSent unsigned char * packetFlagsLocation = destinationBuffer; destinationBuffer += sizeof(packetStateFlags); - if (itemFlags == 0) { - } - #define AVATAR_MEMCPY(src) \ memcpy(destinationBuffer, &(src), sizeof(src)); \ destinationBuffer += sizeof(src); // If we want an item and there's sufficient space: #define IF_AVATAR_SPACE(flag, space) \ - if ((itemFlags & AvatarDataPacket::flag) && (int)(packetEnd - destinationBuffer) >= (space) \ + if ((packetStateFlags & AvatarDataPacket::flag) && (int)(packetEnd - destinationBuffer) >= (space) \ && (includedFlags |= AvatarDataPacket::flag)) IF_AVATAR_SPACE(PACKET_HAS_AVATAR_GLOBAL_POSITION, sizeof _globalPosition) { @@ -583,7 +593,9 @@ QByteArray AvatarData::toByteArray(AvatarDataDetail dataDetail, quint64 lastSent const int jointBitVectorSize = calcBitVectorSize(numJoints); // Check against full size or minimum size: count + two bit-vectors + two controllers - const int approxJointSpace = sendAll ? AvatarDataPacket::maxJointDataSize(numJoints, true) : 1 + 2 * jointBitVectorSize + 2 * (6 + 2); + const int approxJointSpace = sendAll ? (int)AvatarDataPacket::maxJointDataSize(numJoints, true) : + 1 + 2 * jointBitVectorSize + 2 * (sizeof(AvatarDataPacket::SixByteQuat) + sizeof(AvatarDataPacket::SixByteTrans)); + IF_AVATAR_SPACE(PACKET_HAS_JOINT_DATA, approxJointSpace) { auto startSection = destinationBuffer; From c88a713ef86e4dacf4d46f5a845fef0acc769cad Mon Sep 17 00:00:00 2001 From: Simon Walton Date: Wed, 19 Sep 2018 12:05:06 -0700 Subject: [PATCH 05/22] Use Local ID for last time & sequence # Also try to fix size_t warnings on gcc --- assignment-client/src/avatars/AvatarMixer.cpp | 6 +++--- .../src/avatars/AvatarMixerClientData.cpp | 14 +++++++------- .../src/avatars/AvatarMixerClientData.h | 18 +++++++++--------- .../src/avatars/AvatarMixerSlave.cpp | 17 ++++++++--------- libraries/avatars/src/AvatarData.cpp | 8 ++++---- 5 files changed, 31 insertions(+), 32 deletions(-) diff --git a/assignment-client/src/avatars/AvatarMixer.cpp b/assignment-client/src/avatars/AvatarMixer.cpp index 561afee296..ebd86b749b 100644 --- a/assignment-client/src/avatars/AvatarMixer.cpp +++ b/assignment-client/src/avatars/AvatarMixer.cpp @@ -541,7 +541,7 @@ void AvatarMixer::handleRequestsDomainListDataPacket(QSharedPointersetLastBroadcastTime(node->getUUID(), 0); + nodeData->setLastBroadcastTime(node->getLocalID(), 0); nodeData->resetSentTraitData(node->getLocalID()); } ); @@ -637,7 +637,7 @@ void AvatarMixer::handleNodeIgnoreRequestPacket(QSharedPointer // Reset the lastBroadcastTime for the ignored avatar to 0 // so the AvatarMixer knows it'll have to send identity data about the ignored avatar // to the ignorer if the ignorer unignores. - nodeData->setLastBroadcastTime(ignoredUUID, 0); + nodeData->setLastBroadcastTime(ignoredNode->getLocalID(), 0); nodeData->resetSentTraitData(ignoredNode->getLocalID()); } @@ -647,7 +647,7 @@ void AvatarMixer::handleNodeIgnoreRequestPacket(QSharedPointer // to the ignored if the ignorer unignores. AvatarMixerClientData* ignoredNodeData = reinterpret_cast(ignoredNode->getLinkedData()); if (ignoredNodeData) { - ignoredNodeData->setLastBroadcastTime(senderNode->getUUID(), 0); + ignoredNodeData->setLastBroadcastTime(senderNode->getLocalID(), 0); ignoredNodeData->resetSentTraitData(senderNode->getLocalID()); } } diff --git a/assignment-client/src/avatars/AvatarMixerClientData.cpp b/assignment-client/src/avatars/AvatarMixerClientData.cpp index 6c01e6e02b..2226a4c7a6 100644 --- a/assignment-client/src/avatars/AvatarMixerClientData.cpp +++ b/assignment-client/src/avatars/AvatarMixerClientData.cpp @@ -200,7 +200,7 @@ void AvatarMixerClientData::checkSkeletonURLAgainstWhitelist(const SlaveSharedDa } } -uint64_t AvatarMixerClientData::getLastBroadcastTime(const QUuid& nodeUUID) const { +uint64_t AvatarMixerClientData::getLastBroadcastTime(NLPacket::LocalID nodeUUID) const { // return the matching PacketSequenceNumber, or the default if we don't have it auto nodeMatch = _lastBroadcastTimes.find(nodeUUID); if (nodeMatch != _lastBroadcastTimes.end()) { @@ -209,9 +209,9 @@ uint64_t AvatarMixerClientData::getLastBroadcastTime(const QUuid& nodeUUID) cons return 0; } -uint16_t AvatarMixerClientData::getLastBroadcastSequenceNumber(const QUuid& nodeUUID) const { +uint16_t AvatarMixerClientData::getLastBroadcastSequenceNumber(NLPacket::LocalID nodeID) const { // return the matching PacketSequenceNumber, or the default if we don't have it - auto nodeMatch = _lastBroadcastSequenceNumbers.find(nodeUUID); + auto nodeMatch = _lastBroadcastSequenceNumbers.find(nodeID); if (nodeMatch != _lastBroadcastSequenceNumbers.end()) { return nodeMatch->second; } @@ -232,7 +232,7 @@ void AvatarMixerClientData::ignoreOther(const Node* self, const Node* other) { } else { killPacket->writePrimitive(KillAvatarReason::YourAvatarEnteredTheirBubble); } - setLastBroadcastTime(other->getUUID(), 0); + setLastBroadcastTime(other->getLocalID(), 0); resetSentTraitData(other->getLocalID()); @@ -311,9 +311,9 @@ AvatarMixerClientData::TraitsCheckTimestamp AvatarMixerClientData::getLastOtherA } } -void AvatarMixerClientData::cleanupKilledNode(const QUuid& nodeUUID, Node::LocalID nodeLocalID) { - removeLastBroadcastSequenceNumber(nodeUUID); - removeLastBroadcastTime(nodeUUID); +void AvatarMixerClientData::cleanupKilledNode(const QUuid&, Node::LocalID nodeLocalID) { + removeLastBroadcastSequenceNumber(nodeLocalID); + removeLastBroadcastTime(nodeLocalID); _lastSentTraitsTimestamps.erase(nodeLocalID); _sentTraitVersions.erase(nodeLocalID); } diff --git a/assignment-client/src/avatars/AvatarMixerClientData.h b/assignment-client/src/avatars/AvatarMixerClientData.h index d38a90ef1f..ccb0f4001d 100644 --- a/assignment-client/src/avatars/AvatarMixerClientData.h +++ b/assignment-client/src/avatars/AvatarMixerClientData.h @@ -49,14 +49,14 @@ public: const AvatarData* getConstAvatarData() const { return _avatar.get(); } AvatarSharedPointer getAvatarSharedPointer() const { return _avatar; } - uint16_t getLastBroadcastSequenceNumber(const QUuid& nodeUUID) const; - void setLastBroadcastSequenceNumber(const QUuid& nodeUUID, uint16_t sequenceNumber) - { _lastBroadcastSequenceNumbers[nodeUUID] = sequenceNumber; } - Q_INVOKABLE void removeLastBroadcastSequenceNumber(const QUuid& nodeUUID) { _lastBroadcastSequenceNumbers.erase(nodeUUID); } + uint16_t getLastBroadcastSequenceNumber(NLPacket::LocalID nodeID) const; + void setLastBroadcastSequenceNumber(NLPacket::LocalID nodeID, uint16_t sequenceNumber) + { _lastBroadcastSequenceNumbers[nodeID] = sequenceNumber; } + Q_INVOKABLE void removeLastBroadcastSequenceNumber(NLPacket::LocalID nodeID) { _lastBroadcastSequenceNumbers.erase(nodeID); } - uint64_t getLastBroadcastTime(const QUuid& nodeUUID) const; - void setLastBroadcastTime(const QUuid& nodeUUID, uint64_t broadcastTime) { _lastBroadcastTimes[nodeUUID] = broadcastTime; } - Q_INVOKABLE void removeLastBroadcastTime(const QUuid& nodeUUID) { _lastBroadcastTimes.erase(nodeUUID); } + uint64_t getLastBroadcastTime(NLPacket::LocalID nodeUUID) const; + void setLastBroadcastTime(NLPacket::LocalID nodeUUID, uint64_t broadcastTime) { _lastBroadcastTimes[nodeUUID] = broadcastTime; } + Q_INVOKABLE void removeLastBroadcastTime(NLPacket::LocalID nodeUUID) { _lastBroadcastTimes.erase(nodeUUID); } Q_INVOKABLE void cleanupKilledNode(const QUuid& nodeUUID, Node::LocalID nodeLocalID); @@ -147,8 +147,8 @@ private: AvatarSharedPointer _avatar { new AvatarData() }; uint16_t _lastReceivedSequenceNumber { 0 }; - std::unordered_map _lastBroadcastSequenceNumbers; - std::unordered_map _lastBroadcastTimes; + std::unordered_map _lastBroadcastSequenceNumbers; + std::unordered_map _lastBroadcastTimes; // this is a map of the last time we encoded an "other" avatar for // sending to "this" node diff --git a/assignment-client/src/avatars/AvatarMixerSlave.cpp b/assignment-client/src/avatars/AvatarMixerSlave.cpp index 2aa8de6cf4..ef10cb730a 100644 --- a/assignment-client/src/avatars/AvatarMixerSlave.cpp +++ b/assignment-client/src/avatars/AvatarMixerSlave.cpp @@ -359,7 +359,7 @@ void AvatarMixerSlave::broadcastAvatarDataToAgent(const SharedNodePointer& node) } if (!shouldIgnore) { - AvatarDataSequenceNumber lastSeqToReceiver = nodeData->getLastBroadcastSequenceNumber(avatarNode->getUUID()); + AvatarDataSequenceNumber lastSeqToReceiver = nodeData->getLastBroadcastSequenceNumber(avatarNode->getLocalID()); AvatarDataSequenceNumber lastSeqFromSender = avatarClientNodeData->getLastReceivedSequenceNumber(); // FIXME - This code does appear to be working. But it seems brittle. @@ -435,11 +435,11 @@ void AvatarMixerSlave::broadcastAvatarDataToAgent(const SharedNodePointer& node) // If the time that the mixer sent AVATAR DATA about Avatar B to Avatar A is BEFORE OR EQUAL TO // the time that Avatar B flagged an IDENTITY DATA change, send IDENTITY DATA about Avatar B to Avatar A. if (otherAvatar->hasProcessedFirstIdentity() - && nodeData->getLastBroadcastTime(otherNode->getUUID()) <= otherNodeData->getIdentityChangeTimestamp()) { + && nodeData->getLastBroadcastTime(otherNode->getLocalID()) <= otherNodeData->getIdentityChangeTimestamp()) { identityBytesSent += sendIdentityPacket(otherNodeData, node); // remember the last time we sent identity details about this other node to the receiver - nodeData->setLastBroadcastTime(otherNode->getUUID(), usecTimestampNow()); + nodeData->setLastBroadcastTime(otherNode->getLocalID(), usecTimestampNow()); } // Typically all out-of-view avatars but such avatars' priorities will rise with time: @@ -453,7 +453,6 @@ void AvatarMixerSlave::broadcastAvatarDataToAgent(const SharedNodePointer& node) nodeData->incrementAvatarInView(); } - bool includeThisAvatar = true; QVector& lastSentJointsForOther = nodeData->getLastOtherAvatarSentJoints(otherNode->getUUID()); const bool distanceAdjust = true; @@ -484,11 +483,11 @@ void AvatarMixerSlave::broadcastAvatarDataToAgent(const SharedNodePointer& node) if (detail != AvatarData::NoData) { _stats.numOthersIncluded++; - // increment the number of avatars sent to this reciever + // increment the number of avatars sent to this receiver nodeData->incrementNumAvatarsSentLastFrame(); // set the last sent sequence number for this sender on the receiver - nodeData->setLastBroadcastSequenceNumber(otherNode->getUUID(), + nodeData->setLastBroadcastSequenceNumber(otherNode->getLocalID(), otherNodeData->getLastReceivedSequenceNumber()); nodeData->setLastOtherAvatarEncodeTime(otherNode->getUUID(), usecTimestampNow()); } @@ -582,11 +581,11 @@ void AvatarMixerSlave::broadcastAvatarDataToDownstreamMixer(const SharedNodePoin quint64 end = usecTimestampNow(); _stats.toByteArrayElapsedTime += (end - start); - auto lastBroadcastTime = nodeData->getLastBroadcastTime(agentNode->getUUID()); + auto lastBroadcastTime = nodeData->getLastBroadcastTime(agentNode->getLocalID()); if (lastBroadcastTime <= agentNodeData->getIdentityChangeTimestamp() || (start - lastBroadcastTime) >= REBROADCAST_IDENTITY_TO_DOWNSTREAM_EVERY_US) { sendReplicatedIdentityPacket(*agentNode, agentNodeData, *node); - nodeData->setLastBroadcastTime(agentNode->getUUID(), start); + nodeData->setLastBroadcastTime(agentNode->getLocalID(), start); } // figure out how large our avatar byte array can be to fit in the packet list @@ -620,7 +619,7 @@ void AvatarMixerSlave::broadcastAvatarDataToDownstreamMixer(const SharedNodePoin nodeData->incrementNumAvatarsSentLastFrame(); // set the last sent sequence number for this sender on the receiver - nodeData->setLastBroadcastSequenceNumber(agentNode->getUUID(), + nodeData->setLastBroadcastSequenceNumber(agentNode->getLocalID(), agentNodeData->getLastReceivedSequenceNumber()); // increment the number of avatars sent to this reciever diff --git a/libraries/avatars/src/AvatarData.cpp b/libraries/avatars/src/AvatarData.cpp index 2b46eb9cdd..5c50805e3b 100644 --- a/libraries/avatars/src/AvatarData.cpp +++ b/libraries/avatars/src/AvatarData.cpp @@ -400,7 +400,7 @@ QByteArray AvatarData::toByteArray(AvatarDataDetail dataDetail, quint64 lastSent // If we want an item and there's sufficient space: #define IF_AVATAR_SPACE(flag, space) \ - if ((packetStateFlags & AvatarDataPacket::flag) && (int)(packetEnd - destinationBuffer) >= (space) \ + if ((packetStateFlags & AvatarDataPacket::flag) && (size_t)(packetEnd - destinationBuffer) >= (size_t)(space) \ && (includedFlags |= AvatarDataPacket::flag)) IF_AVATAR_SPACE(PACKET_HAS_AVATAR_GLOBAL_POSITION, sizeof _globalPosition) { @@ -563,7 +563,7 @@ QByteArray AvatarData::toByteArray(AvatarDataDetail dataDetail, quint64 lastSent const auto& blendshapeCoefficients = _headData->getBlendshapeCoefficients(); // If it is connected, pack up the data - IF_AVATAR_SPACE(PACKET_HAS_FACE_TRACKER_INFO, sizeof(AvatarDataPacket::FaceTrackerInfo) + blendshapeCoefficients.size() * sizeof(float)) { + IF_AVATAR_SPACE(PACKET_HAS_FACE_TRACKER_INFO, sizeof(AvatarDataPacket::FaceTrackerInfo) + (size_t)blendshapeCoefficients.size() * sizeof(float)) { auto startSection = destinationBuffer; auto faceTrackerInfo = reinterpret_cast(destinationBuffer); // note: we don't use the blink and average loudness, we just use the numBlendShapes and @@ -593,7 +593,7 @@ QByteArray AvatarData::toByteArray(AvatarDataDetail dataDetail, quint64 lastSent const int jointBitVectorSize = calcBitVectorSize(numJoints); // Check against full size or minimum size: count + two bit-vectors + two controllers - const int approxJointSpace = sendAll ? (int)AvatarDataPacket::maxJointDataSize(numJoints, true) : + const size_t approxJointSpace = sendAll ? AvatarDataPacket::maxJointDataSize(numJoints, true) : 1 + 2 * jointBitVectorSize + 2 * (sizeof(AvatarDataPacket::SixByteQuat) + sizeof(AvatarDataPacket::SixByteTrans)); IF_AVATAR_SPACE(PACKET_HAS_JOINT_DATA, approxJointSpace) { @@ -753,7 +753,7 @@ QByteArray AvatarData::toByteArray(AvatarDataDetail dataDetail, quint64 lastSent data->mouseFarGrabRotation[3] = mouseFarGrabRotation.z; destinationBuffer += sizeof(data->mouseFarGrabRotation); - int numGrabJointBytes = destinationBuffer - startSection; + numGrabJointBytes = destinationBuffer - startSection; } #ifdef WANT_DEBUG From 04c47943ba61cf73dd34b2f4a266bc384d2ed7ef Mon Sep 17 00:00:00 2001 From: Simon Walton Date: Wed, 19 Sep 2018 13:45:08 -0700 Subject: [PATCH 06/22] Handle small packet space correctly --- assignment-client/src/avatars/AvatarMixerSlave.cpp | 2 +- libraries/avatars/src/AvatarData.cpp | 2 ++ libraries/avatars/src/AvatarData.h | 2 ++ 3 files changed, 5 insertions(+), 1 deletion(-) diff --git a/assignment-client/src/avatars/AvatarMixerSlave.cpp b/assignment-client/src/avatars/AvatarMixerSlave.cpp index ef10cb730a..ee7dfd85f3 100644 --- a/assignment-client/src/avatars/AvatarMixerSlave.cpp +++ b/assignment-client/src/avatars/AvatarMixerSlave.cpp @@ -471,7 +471,7 @@ void AvatarMixerSlave::broadcastAvatarDataToAgent(const SharedNodePointer& node) avatarPacket->write(bytes); avatarSpaceAvailable -= bytes.size(); numAvatarDataBytes += bytes.size(); - if (includeFlags != 0) { + if (includeFlags != 0 || avatarSpaceAvailable < AvatarDataPacket::MIN_BULK_PACKET_SIZE) { // Weren't able to fit everything. nodeList->sendPacket(std::move(avatarPacket), *destinationNode); ++numPacketsSent; diff --git a/libraries/avatars/src/AvatarData.cpp b/libraries/avatars/src/AvatarData.cpp index 5c50805e3b..20456012a3 100644 --- a/libraries/avatars/src/AvatarData.cpp +++ b/libraries/avatars/src/AvatarData.cpp @@ -243,10 +243,12 @@ QByteArray AvatarData::toByteArray(AvatarDataDetail dataDetail, quint64 lastSent bool sendPALMinimum = (dataDetail == PALMinimum); lazyInitHeadData(); + ASSERT(maxDataSize == 0 || maxDataSize >= AvatarDataPacket::MIN_BULK_PACKET_SIZE); // special case, if we were asked for no data, then just include the flags all set to nothing if (dataDetail == NoData) { AvatarDataPacket::HasFlags packetStateFlags = 0; + itemFlags = packetStateFlags; QByteArray avatarDataByteArray(getSessionUUID().toRfc4122().data(), NUM_BYTES_RFC4122_UUID + sizeof(packetStateFlags)); avatarDataByteArray.append((char*) &packetStateFlags, sizeof packetStateFlags); diff --git a/libraries/avatars/src/AvatarData.h b/libraries/avatars/src/AvatarData.h index 057b351670..df7129b7a5 100644 --- a/libraries/avatars/src/AvatarData.h +++ b/libraries/avatars/src/AvatarData.h @@ -296,6 +296,8 @@ namespace AvatarDataPacket { } PACKED_END; const size_t FAR_GRAB_JOINTS_SIZE = 84; static_assert(sizeof(FarGrabJoints) == FAR_GRAB_JOINTS_SIZE, "AvatarDataPacket::FarGrabJoints size doesn't match."); + + const size_t MIN_BULK_PACKET_SIZE = NUM_BYTES_RFC4122_UUID + HEADER_SIZE; } const float MAX_AUDIO_LOUDNESS = 1000.0f; // close enough for mouth animation From f1b7097edb69ce6f0801538470f020919a481d89 Mon Sep 17 00:00:00 2001 From: Simon Walton Date: Wed, 19 Sep 2018 15:32:56 -0700 Subject: [PATCH 07/22] Priority experiment --- assignment-client/src/avatars/AvatarMixerSlave.cpp | 2 +- libraries/avatars/src/AvatarData.cpp | 2 +- libraries/shared/src/PrioritySortUtil.h | 5 ++++- 3 files changed, 6 insertions(+), 3 deletions(-) diff --git a/assignment-client/src/avatars/AvatarMixerSlave.cpp b/assignment-client/src/avatars/AvatarMixerSlave.cpp index ee7dfd85f3..15330ab5c3 100644 --- a/assignment-client/src/avatars/AvatarMixerSlave.cpp +++ b/assignment-client/src/avatars/AvatarMixerSlave.cpp @@ -471,7 +471,7 @@ void AvatarMixerSlave::broadcastAvatarDataToAgent(const SharedNodePointer& node) avatarPacket->write(bytes); avatarSpaceAvailable -= bytes.size(); numAvatarDataBytes += bytes.size(); - if (includeFlags != 0 || avatarSpaceAvailable < AvatarDataPacket::MIN_BULK_PACKET_SIZE) { + if (includeFlags != 0 || avatarSpaceAvailable < (int)AvatarDataPacket::MIN_BULK_PACKET_SIZE) { // Weren't able to fit everything. nodeList->sendPacket(std::move(avatarPacket), *destinationNode); ++numPacketsSent; diff --git a/libraries/avatars/src/AvatarData.cpp b/libraries/avatars/src/AvatarData.cpp index 20456012a3..4007856a73 100644 --- a/libraries/avatars/src/AvatarData.cpp +++ b/libraries/avatars/src/AvatarData.cpp @@ -243,7 +243,7 @@ QByteArray AvatarData::toByteArray(AvatarDataDetail dataDetail, quint64 lastSent bool sendPALMinimum = (dataDetail == PALMinimum); lazyInitHeadData(); - ASSERT(maxDataSize == 0 || maxDataSize >= AvatarDataPacket::MIN_BULK_PACKET_SIZE); + ASSERT(maxDataSize == 0 || (size_t)maxDataSize >= AvatarDataPacket::MIN_BULK_PACKET_SIZE); // special case, if we were asked for no data, then just include the flags all set to nothing if (dataDetail == NoData) { diff --git a/libraries/shared/src/PrioritySortUtil.h b/libraries/shared/src/PrioritySortUtil.h index 3a40fead71..c3d0f352ce 100644 --- a/libraries/shared/src/PrioritySortUtil.h +++ b/libraries/shared/src/PrioritySortUtil.h @@ -23,7 +23,7 @@ const float OUT_OF_VIEW_THRESHOLD = 0.5f * OUT_OF_VIEW_PENALTY; namespace PrioritySortUtil { - constexpr float DEFAULT_ANGULAR_COEF { 1.0f }; + constexpr float DEFAULT_ANGULAR_COEF { 2.5f }; constexpr float DEFAULT_CENTER_COEF { 0.5f }; constexpr float DEFAULT_AGE_COEF { 0.25f }; @@ -103,6 +103,9 @@ namespace PrioritySortUtil { float radius = glm::max(thing.getRadius(), MIN_RADIUS); // Other item's angle from view centre: float cosineAngle = glm::dot(offset, view.getDirection()) / distance; + if (cosineAngle > 0.0f) { + cosineAngle = std::sqrt(cosineAngle); + } float age = float((_usecCurrentTime - thing.getTimestamp()) / USECS_PER_SECOND); // the "age" term accumulates at the sum of all weights From 7bba4d4badaea550a67b3ffe4fac9416203a5e0c Mon Sep 17 00:00:00 2001 From: Simon Walton Date: Wed, 19 Sep 2018 18:38:20 -0700 Subject: [PATCH 08/22] Change priorities in the correct place --- libraries/avatars/src/AvatarData.cpp | 2 +- libraries/shared/src/PrioritySortUtil.h | 2 +- 2 files changed, 2 insertions(+), 2 deletions(-) diff --git a/libraries/avatars/src/AvatarData.cpp b/libraries/avatars/src/AvatarData.cpp index 4007856a73..73525f69a7 100644 --- a/libraries/avatars/src/AvatarData.cpp +++ b/libraries/avatars/src/AvatarData.cpp @@ -2879,7 +2879,7 @@ void RayToAvatarIntersectionResultFromScriptValue(const QScriptValue& object, Ra value.extraInfo = object.property("extraInfo").toVariant().toMap(); } -float AvatarData::_avatarSortCoefficientSize { 8.0f }; +float AvatarData::_avatarSortCoefficientSize { 20.0f }; float AvatarData::_avatarSortCoefficientCenter { 4.0f }; float AvatarData::_avatarSortCoefficientAge { 1.0f }; diff --git a/libraries/shared/src/PrioritySortUtil.h b/libraries/shared/src/PrioritySortUtil.h index c3d0f352ce..8020bde09d 100644 --- a/libraries/shared/src/PrioritySortUtil.h +++ b/libraries/shared/src/PrioritySortUtil.h @@ -23,7 +23,7 @@ const float OUT_OF_VIEW_THRESHOLD = 0.5f * OUT_OF_VIEW_PENALTY; namespace PrioritySortUtil { - constexpr float DEFAULT_ANGULAR_COEF { 2.5f }; + constexpr float DEFAULT_ANGULAR_COEF { 1.0f }; constexpr float DEFAULT_CENTER_COEF { 0.5f }; constexpr float DEFAULT_AGE_COEF { 0.25f }; From 328ed8d9766b2c2e1a418f00df8d0ceaca49a72c Mon Sep 17 00:00:00 2001 From: Simon Walton Date: Thu, 20 Sep 2018 09:31:01 -0700 Subject: [PATCH 09/22] Remove unused variable --- assignment-client/src/avatars/AvatarMixerSlave.cpp | 5 ----- 1 file changed, 5 deletions(-) diff --git a/assignment-client/src/avatars/AvatarMixerSlave.cpp b/assignment-client/src/avatars/AvatarMixerSlave.cpp index 15330ab5c3..c8cc4d0c8a 100644 --- a/assignment-client/src/avatars/AvatarMixerSlave.cpp +++ b/assignment-client/src/avatars/AvatarMixerSlave.cpp @@ -250,9 +250,6 @@ void AvatarMixerSlave::broadcastAvatarDataToAgent(const SharedNodePointer& node) // reset the number of sent avatars nodeData->resetNumAvatarsSentLastFrame(); - // keep a counter of the number of considered avatars - int numOtherAvatars = 0; - // keep track of outbound data rate specifically for avatar data int numAvatarDataBytes = 0; int identityBytesSent = 0; @@ -427,8 +424,6 @@ void AvatarMixerSlave::broadcastAvatarDataToAgent(const SharedNodePointer& node) auto startAvatarDataPacking = chrono::high_resolution_clock::now(); - ++numOtherAvatars; - const AvatarMixerClientData* otherNodeData = reinterpret_cast(otherNode->getLinkedData()); const AvatarData* otherAvatar = otherNodeData->getConstAvatarData(); From f2e69d5c818363e58566ceeca5a3aa75be209bfd Mon Sep 17 00:00:00 2001 From: Simon Walton Date: Thu, 20 Sep 2018 17:37:39 -0700 Subject: [PATCH 10/22] Make default bubble-box AvatarData property for efficiency Also a use of getClientGlobalPosition(), etc --- .../src/avatars/AvatarMixerClientData.h | 2 +- .../src/avatars/AvatarMixerSlave.cpp | 6 ++--- libraries/avatars/src/AvatarData.cpp | 24 +++++++++++++++++-- libraries/avatars/src/AvatarData.h | 6 +++++ 4 files changed, 31 insertions(+), 7 deletions(-) diff --git a/assignment-client/src/avatars/AvatarMixerClientData.h b/assignment-client/src/avatars/AvatarMixerClientData.h index ccb0f4001d..492d21a807 100644 --- a/assignment-client/src/avatars/AvatarMixerClientData.h +++ b/assignment-client/src/avatars/AvatarMixerClientData.h @@ -90,7 +90,7 @@ public: void loadJSONStats(QJsonObject& jsonObject) const; - glm::vec3 getPosition() const { return _avatar ? _avatar->getWorldPosition() : glm::vec3(0); } + glm::vec3 getPosition() const { return _avatar ? _avatar->getClientGlobalPosition() : glm::vec3(0); } bool isRadiusIgnoring(const QUuid& other) const; void addToRadiusIgnoringSet(const QUuid& other); void removeFromRadiusIgnoringSet(const QUuid& other); diff --git a/assignment-client/src/avatars/AvatarMixerSlave.cpp b/assignment-client/src/avatars/AvatarMixerSlave.cpp index c8cc4d0c8a..f83bb2d3cb 100644 --- a/assignment-client/src/avatars/AvatarMixerSlave.cpp +++ b/assignment-client/src/avatars/AvatarMixerSlave.cpp @@ -244,7 +244,7 @@ void AvatarMixerSlave::broadcastAvatarDataToAgent(const SharedNodePointer& node) // reset the internal state for correct random number distribution distribution.reset(); - // Base number to sort on number previously sent. + // Estimate number to sort on number sent last frame. const int numToSendEst = std::max(nodeData->getNumAvatarsSentLastFrame() * 2, 20); // reset the number of sent avatars @@ -342,8 +342,7 @@ void AvatarMixerSlave::broadcastAvatarDataToAgent(const SharedNodePointer& node) // Don't bother with these checks if the other avatar has their bubble enabled and we're gettingAnyIgnored if (destinationNode->isIgnoreRadiusEnabled() || (avatarNode->isIgnoreRadiusEnabled() && !getsAnyIgnored)) { // Perform the collision check between the two bounding boxes - const float OTHER_AVATAR_BUBBLE_EXPANSION_FACTOR = 2.4f; // magic number determined empirically - AABox otherNodeBox = computeBubbleBox(avatarClientNodeData->getAvatar(), OTHER_AVATAR_BUBBLE_EXPANSION_FACTOR); + AABox otherNodeBox = avatarClientNodeData->getAvatar().getDefaultBubbleBox(); if (nodeBox.touches(otherNodeBox)) { nodeData->ignoreOther(destinationNode, avatarNode); shouldIgnore = !getsAnyIgnored; @@ -396,7 +395,6 @@ void AvatarMixerSlave::broadcastAvatarDataToAgent(const SharedNodePointer& node) auto avatarPacket = NLPacket::create(PacketType::BulkAvatarData); const int avatarPacketCapacity = avatarPacket->getPayloadCapacity(); int avatarSpaceAvailable = avatarPacketCapacity; - //avatarSpaceAvailable = 100; int numPacketsSent = 0; const auto& sortedAvatarVector = sortedAvatars.getSortedVector(numToSendEst); diff --git a/libraries/avatars/src/AvatarData.cpp b/libraries/avatars/src/AvatarData.cpp index 73525f69a7..782483a6c1 100644 --- a/libraries/avatars/src/AvatarData.cpp +++ b/libraries/avatars/src/AvatarData.cpp @@ -384,8 +384,8 @@ QByteArray AvatarData::toByteArray(AvatarDataDetail dataDetail, quint64 lastSent QByteArray avatarDataByteArray((int)byteArraySize, 0); unsigned char* destinationBuffer = reinterpret_cast(avatarDataByteArray.data()); - unsigned char* startPosition = destinationBuffer; - const unsigned char * packetEnd = destinationBuffer + maxDataSize; + const unsigned char* const startPosition = destinationBuffer; + const unsigned char* const packetEnd = destinationBuffer + maxDataSize; AvatarDataPacket::HasFlags includedFlags = 0; @@ -971,6 +971,8 @@ int AvatarData::parseDataFromBuffer(const QByteArray& buffer) { _avatarBoundingBoxChanged = now; } + _defaultBubbleBox = computeBubbleBox(); + sourceBuffer += sizeof(AvatarDataPacket::AvatarBoundingBox); int numBytesRead = sourceBuffer - startSection; _avatarBoundingBoxRate.increment(numBytesRead); @@ -2916,3 +2918,21 @@ void AvatarEntityMapFromScriptValue(const QScriptValue& object, AvatarEntityMap& value[EntityID] = binaryEntityProperties; } } + +const float AvatarData::DEFAULT_BUBBLE_SCALE = 2.4f; // magic number determined empirically + +AABox AvatarData::computeBubbleBox(float bubbleScale) const { + AABox box = AABox(_globalBoundingBoxOffset - _globalBoundingBoxDimensions, _globalBoundingBoxDimensions); + glm::vec3 size = box.getScale(); + size *= bubbleScale; + const glm::vec3 MIN_BUBBLE_SCALE(0.3f, 1.3f, 0.3); + size= glm::max(size, MIN_BUBBLE_SCALE); + box.setScaleStayCentered(size); + return box; +} + +AABox AvatarData::getDefaultBubbleBox() const { + AABox bubbleBox(_defaultBubbleBox); + bubbleBox.translate(_globalPosition); + return bubbleBox; +} diff --git a/libraries/avatars/src/AvatarData.h b/libraries/avatars/src/AvatarData.h index df7129b7a5..3ea7631e0c 100644 --- a/libraries/avatars/src/AvatarData.h +++ b/libraries/avatars/src/AvatarData.h @@ -1100,6 +1100,7 @@ public: glm::vec3 getClientGlobalPosition() const { return _globalPosition; } AABox getGlobalBoundingBox() const { return AABox(_globalPosition + _globalBoundingBoxOffset - _globalBoundingBoxDimensions, _globalBoundingBoxDimensions); } + AABox getDefaultBubbleBox() const; /**jsdoc * @function MyAvatar.getAvatarEntityData @@ -1192,6 +1193,9 @@ public: void setReplicaIndex(int replicaIndex) { _replicaIndex = replicaIndex; } int getReplicaIndex() { return _replicaIndex; } + static const float DEFAULT_BUBBLE_SCALE; /* = 2.4 */ + AABox computeBubbleBox(float bubbleScale = DEFAULT_BUBBLE_SCALE) const; + signals: /**jsdoc @@ -1425,6 +1429,8 @@ protected: glm::vec3 _globalBoundingBoxDimensions; glm::vec3 _globalBoundingBoxOffset; + AABox _defaultBubbleBox; + mutable ReadWriteLockable _avatarEntitiesLock; AvatarEntityIDs _avatarEntityDetached; // recently detached from this avatar AvatarEntityIDs _avatarEntityForRecording; // create new entities id for avatar recording From 82b81c907670a0eb26053648bc551045219c6841 Mon Sep 17 00:00:00 2001 From: Simon Walton Date: Mon, 24 Sep 2018 11:48:48 -0700 Subject: [PATCH 11/22] Limit no. of joints sent per avatar to prevent lock-up With this the avatar mixer will sent the full bit-vector but only the first 111 actual joints. --- libraries/avatars/src/AvatarData.cpp | 10 +++++++--- libraries/avatars/src/AvatarData.h | 3 ++- 2 files changed, 9 insertions(+), 4 deletions(-) diff --git a/libraries/avatars/src/AvatarData.cpp b/libraries/avatars/src/AvatarData.cpp index 782483a6c1..31687a7f27 100644 --- a/libraries/avatars/src/AvatarData.cpp +++ b/libraries/avatars/src/AvatarData.cpp @@ -592,10 +592,12 @@ QByteArray AvatarData::toByteArray(AvatarDataDetail dataDetail, quint64 lastSent jointData = _jointData; } const int numJoints = jointData.size(); + assert(numJoints <= 255); const int jointBitVectorSize = calcBitVectorSize(numJoints); + const int cappedNumJoints = std::min(numJoints, AvatarDataPacket::MAX_NUM_JOINTS); // Check against full size or minimum size: count + two bit-vectors + two controllers - const size_t approxJointSpace = sendAll ? AvatarDataPacket::maxJointDataSize(numJoints, true) : + const size_t approxJointSpace = sendAll ? AvatarDataPacket::maxJointDataSize(cappedNumJoints, false) : 1 + 2 * jointBitVectorSize + 2 * (sizeof(AvatarDataPacket::SixByteQuat) + sizeof(AvatarDataPacket::SixByteTrans)); IF_AVATAR_SPACE(PACKET_HAS_JOINT_DATA, approxJointSpace) { @@ -605,6 +607,7 @@ QByteArray AvatarData::toByteArray(AvatarDataDetail dataDetail, quint64 lastSent *destinationBuffer++ = (uint8_t)numJoints; unsigned char* validityPosition = destinationBuffer; + memset(validityPosition, 0, jointBitVectorSize); unsigned char validity = 0; int validityBit = 0; @@ -622,7 +625,7 @@ QByteArray AvatarData::toByteArray(AvatarDataDetail dataDetail, quint64 lastSent float minRotationDOT = (distanceAdjust && cullSmallChanges) ? getDistanceBasedMinRotationDOT(viewerPosition) : AVATAR_MIN_ROTATION_DOT; - for (int i = 0; i < jointData.size(); i++) { + for (int i = 0; i < cappedNumJoints; i++) { const JointData& data = jointData[i]; const JointData& last = lastSentJointData[i]; @@ -666,12 +669,13 @@ QByteArray AvatarData::toByteArray(AvatarDataDetail dataDetail, quint64 lastSent unsigned char* beforeTranslations = destinationBuffer; #endif + memset(destinationBuffer, 0, jointBitVectorSize); destinationBuffer += jointBitVectorSize; // Move pointer past the validity bytes float minTranslation = (distanceAdjust && cullSmallChanges) ? getDistanceBasedMinTranslationDistance(viewerPosition) : AVATAR_MIN_TRANSLATION; float maxTranslationDimension = 0.0; - for (int i = 0; i < jointData.size(); i++) { + for (int i = 0; i < cappedNumJoints; i++) { const JointData& data = jointData[i]; const JointData& last = lastSentJointData[i]; diff --git a/libraries/avatars/src/AvatarData.h b/libraries/avatars/src/AvatarData.h index 3ea7631e0c..5b5d49feea 100644 --- a/libraries/avatars/src/AvatarData.h +++ b/libraries/avatars/src/AvatarData.h @@ -297,7 +297,8 @@ namespace AvatarDataPacket { const size_t FAR_GRAB_JOINTS_SIZE = 84; static_assert(sizeof(FarGrabJoints) == FAR_GRAB_JOINTS_SIZE, "AvatarDataPacket::FarGrabJoints size doesn't match."); - const size_t MIN_BULK_PACKET_SIZE = NUM_BYTES_RFC4122_UUID + HEADER_SIZE; + static const size_t MIN_BULK_PACKET_SIZE = NUM_BYTES_RFC4122_UUID + HEADER_SIZE; + static const int MAX_NUM_JOINTS = 111; } const float MAX_AUDIO_LOUDNESS = 1000.0f; // close enough for mouth animation From 07bdaeede794271a98b56275e9e687b930d0022a Mon Sep 17 00:00:00 2001 From: Simon Walton Date: Mon, 24 Sep 2018 18:23:49 -0700 Subject: [PATCH 12/22] Split avatar joint-data across multiple packets if necessary --- .../src/avatars/AvatarMixerSlave.cpp | 16 +-- libraries/avatars/src/AvatarData.cpp | 136 ++++++++---------- libraries/avatars/src/AvatarData.h | 11 +- 3 files changed, 80 insertions(+), 83 deletions(-) diff --git a/assignment-client/src/avatars/AvatarMixerSlave.cpp b/assignment-client/src/avatars/AvatarMixerSlave.cpp index f83bb2d3cb..3810456f71 100644 --- a/assignment-client/src/avatars/AvatarMixerSlave.cpp +++ b/assignment-client/src/avatars/AvatarMixerSlave.cpp @@ -450,12 +450,12 @@ void AvatarMixerSlave::broadcastAvatarDataToAgent(const SharedNodePointer& node) const bool distanceAdjust = true; const bool dropFaceTracking = false; - AvatarDataPacket::HasFlags includeFlags = 0; // the result of the toByteArray + AvatarDataPacket::SendStatus sendStatus; do { auto startSerialize = chrono::high_resolution_clock::now(); QByteArray bytes = otherAvatar->toByteArray(detail, lastEncodeForOther, lastSentJointsForOther, - includeFlags, dropFaceTracking, distanceAdjust, myPosition, + sendStatus, dropFaceTracking, distanceAdjust, myPosition, &lastSentJointsForOther, avatarSpaceAvailable); auto endSerialize = chrono::high_resolution_clock::now(); _stats.toByteArrayElapsedTime += @@ -464,14 +464,14 @@ void AvatarMixerSlave::broadcastAvatarDataToAgent(const SharedNodePointer& node) avatarPacket->write(bytes); avatarSpaceAvailable -= bytes.size(); numAvatarDataBytes += bytes.size(); - if (includeFlags != 0 || avatarSpaceAvailable < (int)AvatarDataPacket::MIN_BULK_PACKET_SIZE) { + if (sendStatus.itemFlags != 0 || avatarSpaceAvailable < (int)AvatarDataPacket::MIN_BULK_PACKET_SIZE) { // Weren't able to fit everything. nodeList->sendPacket(std::move(avatarPacket), *destinationNode); ++numPacketsSent; avatarPacket = NLPacket::create(PacketType::BulkAvatarData); avatarSpaceAvailable = avatarPacketCapacity; } - } while (includeFlags != 0); + } while (!sendStatus); if (detail != AvatarData::NoData) { _stats.numOthersIncluded++; @@ -565,12 +565,12 @@ void AvatarMixerSlave::broadcastAvatarDataToDownstreamMixer(const SharedNodePoin // so we always send a full update for this avatar quint64 start = usecTimestampNow(); - AvatarDataPacket::HasFlags flagsOut; + AvatarDataPacket::SendStatus sendStatus; QVector emptyLastJointSendData { otherAvatar->getJointCount() }; QByteArray avatarByteArray = otherAvatar->toByteArray(AvatarData::SendAllData, 0, emptyLastJointSendData, - flagsOut, false, false, glm::vec3(0), nullptr, 0); + sendStatus, false, false, glm::vec3(0), nullptr, 0); quint64 end = usecTimestampNow(); _stats.toByteArrayElapsedTime += (end - start); @@ -596,14 +596,14 @@ void AvatarMixerSlave::broadcastAvatarDataToDownstreamMixer(const SharedNodePoin << "-" << avatarByteArray.size() << "bytes"; avatarByteArray = otherAvatar->toByteArray(AvatarData::SendAllData, 0, emptyLastJointSendData, - flagsOut, true, false, glm::vec3(0), nullptr, 0); + sendStatus, true, false, glm::vec3(0), nullptr, 0); if (avatarByteArray.size() > maxAvatarByteArraySize) { qCWarning(avatars) << "Replicated avatar data without facial data still too large for" << otherAvatar->getSessionUUID() << "-" << avatarByteArray.size() << "bytes"; avatarByteArray = otherAvatar->toByteArray(AvatarData::MinimumData, 0, emptyLastJointSendData, - flagsOut, true, false, glm::vec3(0), nullptr, 0); + sendStatus, true, false, glm::vec3(0), nullptr, 0); } } diff --git a/libraries/avatars/src/AvatarData.cpp b/libraries/avatars/src/AvatarData.cpp index 31687a7f27..f393ca9ba8 100644 --- a/libraries/avatars/src/AvatarData.cpp +++ b/libraries/avatars/src/AvatarData.cpp @@ -225,16 +225,16 @@ QByteArray AvatarData::toByteArrayStateful(AvatarDataDetail dataDetail, bool dro AvatarDataPacket::HasFlags hasFlagsOut = 0; auto lastSentTime = _lastToByteArray; _lastToByteArray = usecTimestampNow(); + AvatarDataPacket::SendStatus sendStatus; auto avatarByteArray = AvatarData::toByteArray(dataDetail, lastSentTime, getLastSentJointData(), - hasFlagsOut, dropFaceTracking, false, glm::vec3(0), nullptr, - 0, &_outboundDataRate); + sendStatus, dropFaceTracking, false, glm::vec3(0), nullptr, 0, &_outboundDataRate); // Strip UUID return avatarByteArray.right(avatarByteArray.size() - NUM_BYTES_RFC4122_UUID); } QByteArray AvatarData::toByteArray(AvatarDataDetail dataDetail, quint64 lastSentTime, const QVector& lastSentJointData, - AvatarDataPacket::HasFlags& itemFlags, bool dropFaceTracking, bool distanceAdjust, + AvatarDataPacket::SendStatus& sendStatus, bool dropFaceTracking, bool distanceAdjust, glm::vec3 viewerPosition, QVector* sentJointDataOut, int maxDataSize, AvatarDataRate* outboundDataRateOut) const { bool cullSmallChanges = (dataDetail == CullSmallData); @@ -248,7 +248,7 @@ QByteArray AvatarData::toByteArray(AvatarDataDetail dataDetail, quint64 lastSent // special case, if we were asked for no data, then just include the flags all set to nothing if (dataDetail == NoData) { AvatarDataPacket::HasFlags packetStateFlags = 0; - itemFlags = packetStateFlags; + sendStatus.itemFlags = packetStateFlags; QByteArray avatarDataByteArray(getSessionUUID().toRfc4122().data(), NUM_BYTES_RFC4122_UUID + sizeof(packetStateFlags)); avatarDataByteArray.append((char*) &packetStateFlags, sizeof packetStateFlags); @@ -274,7 +274,7 @@ QByteArray AvatarData::toByteArray(AvatarDataDetail dataDetail, quint64 lastSent // 3 translations * 6 bytes = 6.48kbps // - auto parentID = getParentID(); + QUuid parentID; glm::mat4 leftFarGrabMatrix; glm::mat4 rightFarGrabMatrix; @@ -283,7 +283,7 @@ QByteArray AvatarData::toByteArray(AvatarDataDetail dataDetail, quint64 lastSent // Leading flags, to indicate how much data is actually included in the packet... AvatarDataPacket::HasFlags packetStateFlags = 0; - if (itemFlags == 0) { + if (sendStatus.itemFlags == 0) { bool hasAvatarGlobalPosition = true; // always include global position bool hasAvatarOrientation = false; bool hasAvatarBoundingBox = false; @@ -340,13 +340,11 @@ QByteArray AvatarData::toByteArray(AvatarDataDetail dataDetail, quint64 lastSent | (hasJointDefaultPoseFlags ? AvatarDataPacket::PACKET_HAS_JOINT_DEFAULT_POSE_FLAGS : 0) | (hasJointData ? AvatarDataPacket::PACKET_HAS_GRAB_JOINTS : 0); - itemFlags = packetStateFlags; + sendStatus.itemFlags = packetStateFlags; + sendStatus.rotationsSent = 0; + sendStatus.translationsSent = 0; } else { - packetStateFlags = itemFlags; - if (packetStateFlags & AvatarDataPacket::PACKET_HAS_JOINT_DATA) { - // Force all joints upon continuation, as deltas aren't valid. - sendAll = true; - } + packetStateFlags = sendStatus.itemFlags; if (packetStateFlags & AvatarDataPacket::PACKET_HAS_GRAB_JOINTS) { packetStateFlags |= AvatarDataPacket::PACKET_HAS_JOINT_DATA; } @@ -372,6 +370,9 @@ QByteArray AvatarData::toByteArray(AvatarDataDetail dataDetail, quint64 lastSent packetStateFlags &= ~AvatarDataPacket::PACKET_HAS_GRAB_JOINTS; } } + if (packetStateFlags & (AvatarDataPacket::PACKET_HAS_ADDITIONAL_FLAGS | AvatarDataPacket::PACKET_HAS_PARENT_INFO)) { + parentID = getParentID(); + } const size_t byteArraySize = AvatarDataPacket::MAX_CONSTANT_HEADER_SIZE + NUM_BYTES_RFC4122_UUID + AvatarDataPacket::maxFaceTrackerInfoSize(_headData->getBlendshapeCoefficients().size()) + @@ -388,6 +389,7 @@ QByteArray AvatarData::toByteArray(AvatarDataDetail dataDetail, quint64 lastSent const unsigned char* const packetEnd = destinationBuffer + maxDataSize; AvatarDataPacket::HasFlags includedFlags = 0; + AvatarDataPacket::HasFlags extraReturnedFlags = 0; // Packets always have UUID. memcpy(destinationBuffer, getSessionUUID().toRfc4122(), NUM_BYTES_RFC4122_UUID); @@ -594,13 +596,11 @@ QByteArray AvatarData::toByteArray(AvatarDataDetail dataDetail, quint64 lastSent const int numJoints = jointData.size(); assert(numJoints <= 255); const int jointBitVectorSize = calcBitVectorSize(numJoints); - const int cappedNumJoints = std::min(numJoints, AvatarDataPacket::MAX_NUM_JOINTS); - // Check against full size or minimum size: count + two bit-vectors + two controllers - const size_t approxJointSpace = sendAll ? AvatarDataPacket::maxJointDataSize(cappedNumJoints, false) : - 1 + 2 * jointBitVectorSize + 2 * (sizeof(AvatarDataPacket::SixByteQuat) + sizeof(AvatarDataPacket::SixByteTrans)); - - IF_AVATAR_SPACE(PACKET_HAS_JOINT_DATA, approxJointSpace) { + IF_AVATAR_SPACE(PACKET_HAS_JOINT_DATA, 1 + 2 * jointBitVectorSize + AvatarDataPacket::FAUX_JOINT_SIZE) { + // Allow for faux joints + translation bit-vector: + static const ptrdiff_t MIN_SIZE_FOR_JOINT = sizeof(AvatarDataPacket::SixByteQuat) + + jointBitVectorSize + AvatarDataPacket::FAUX_JOINT_SIZE; auto startSection = destinationBuffer; // joint rotation data @@ -608,8 +608,6 @@ QByteArray AvatarData::toByteArray(AvatarDataDetail dataDetail, quint64 lastSent unsigned char* validityPosition = destinationBuffer; memset(validityPosition, 0, jointBitVectorSize); - unsigned char validity = 0; - int validityBit = 0; #ifdef WANT_DEBUG int rotationSentCount = 0; @@ -625,44 +623,41 @@ QByteArray AvatarData::toByteArray(AvatarDataDetail dataDetail, quint64 lastSent float minRotationDOT = (distanceAdjust && cullSmallChanges) ? getDistanceBasedMinRotationDOT(viewerPosition) : AVATAR_MIN_ROTATION_DOT; - for (int i = 0; i < cappedNumJoints; i++) { + int i = sendStatus.rotationsSent; + for (; i < numJoints; ++i) { const JointData& data = jointData[i]; const JointData& last = lastSentJointData[i]; - if (!data.rotationIsDefaultPose) { - // The dot product for larger rotations is a lower number, - // so if the dot() is less than the value, then the rotation is a larger angle of rotation - if (sendAll || last.rotationIsDefaultPose || (!cullSmallChanges && last.rotation != data.rotation) - || (cullSmallChanges && glm::dot(last.rotation, data.rotation) < minRotationDOT) ) { - validity |= (1 << validityBit); + if ((packetEnd - destinationBuffer) >= MIN_SIZE_FOR_JOINT) { + if (!data.rotationIsDefaultPose) { + // The dot product for larger rotations is a lower number, + // so if the dot() is less than the value, then the rotation is a larger angle of rotation + if (sendAll || last.rotationIsDefaultPose || (!cullSmallChanges && last.rotation != data.rotation) + || (cullSmallChanges && glm::dot(last.rotation, data.rotation) < minRotationDOT)) { + validityPosition[i / BITS_IN_BYTE] |= 1 << (i % BITS_IN_BYTE); #ifdef WANT_DEBUG - rotationSentCount++; + rotationSentCount++; #endif - destinationBuffer += packOrientationQuatToSixBytes(destinationBuffer, data.rotation); + destinationBuffer += packOrientationQuatToSixBytes(destinationBuffer, data.rotation); - if (sentJointDataOut) { - (*sentJointDataOut)[i].rotation = data.rotation; + if (sentJointDataOut) { + (*sentJointDataOut)[i].rotation = data.rotation; + } } } + } else { + break; } if (sentJointDataOut) { (*sentJointDataOut)[i].rotationIsDefaultPose = data.rotationIsDefaultPose; } - if (++validityBit == BITS_IN_BYTE) { - *validityPosition++ = validity; - validityBit = validity = 0; - } - } - if (validityBit != 0) { - *validityPosition++ = validity; } + sendStatus.rotationsSent = i; // joint translation data validityPosition = destinationBuffer; - validity = 0; - validityBit = 0; #ifdef WANT_DEBUG int translationSentCount = 0; @@ -675,44 +670,41 @@ QByteArray AvatarData::toByteArray(AvatarDataDetail dataDetail, quint64 lastSent float minTranslation = (distanceAdjust && cullSmallChanges) ? getDistanceBasedMinTranslationDistance(viewerPosition) : AVATAR_MIN_TRANSLATION; float maxTranslationDimension = 0.0; - for (int i = 0; i < cappedNumJoints; i++) { + i = sendStatus.translationsSent; + for (; i < numJoints; ++i) { const JointData& data = jointData[i]; const JointData& last = lastSentJointData[i]; - if (!data.translationIsDefaultPose) { - if (sendAll || last.translationIsDefaultPose || (!cullSmallChanges && last.translation != data.translation) - || (cullSmallChanges && glm::distance(data.translation, lastSentJointData[i].translation) > minTranslation)) { - - validity |= (1 << validityBit); + if (packetEnd - destinationBuffer > MIN_SIZE_FOR_JOINT) { + if (!data.translationIsDefaultPose) { + if (sendAll || last.translationIsDefaultPose || (!cullSmallChanges && last.translation != data.translation) + || (cullSmallChanges && glm::distance(data.translation, lastSentJointData[i].translation) > minTranslation)) { + validityPosition[i / BITS_IN_BYTE] |= 1 << (i % BITS_IN_BYTE); #ifdef WANT_DEBUG - translationSentCount++; + translationSentCount++; #endif - maxTranslationDimension = glm::max(fabsf(data.translation.x), maxTranslationDimension); - maxTranslationDimension = glm::max(fabsf(data.translation.y), maxTranslationDimension); - maxTranslationDimension = glm::max(fabsf(data.translation.z), maxTranslationDimension); + maxTranslationDimension = glm::max(fabsf(data.translation.x), maxTranslationDimension); + maxTranslationDimension = glm::max(fabsf(data.translation.y), maxTranslationDimension); + maxTranslationDimension = glm::max(fabsf(data.translation.z), maxTranslationDimension); - destinationBuffer += - packFloatVec3ToSignedTwoByteFixed(destinationBuffer, data.translation, TRANSLATION_COMPRESSION_RADIX); + destinationBuffer += + packFloatVec3ToSignedTwoByteFixed(destinationBuffer, data.translation, TRANSLATION_COMPRESSION_RADIX); - if (sentJointDataOut) { - (*sentJointDataOut)[i].translation = data.translation; + if (sentJointDataOut) { + (*sentJointDataOut)[i].translation = data.translation; + } } } + } else { + break; } if (sentJointDataOut) { (*sentJointDataOut)[i].translationIsDefaultPose = data.translationIsDefaultPose; } - if (++validityBit == BITS_IN_BYTE) { - *validityPosition++ = validity; - validityBit = validity = 0; - } - } - - if (validityBit != 0) { - *validityPosition++ = validity; } + sendStatus.translationsSent = i; // faux joints Transform controllerLeftHandTransform = Transform(getControllerLeftHandMatrix()); @@ -775,16 +767,14 @@ QByteArray AvatarData::toByteArray(AvatarDataDetail dataDetail, quint64 lastSent } #endif - if (destinationBuffer >= packetEnd) { - // Joint data too large - revert - destinationBuffer = startSection; - includedFlags &= ~(AvatarDataPacket::PACKET_HAS_GRAB_JOINTS | AvatarDataPacket::PACKET_HAS_JOINT_DATA); - } else { - int numBytes = destinationBuffer - startSection; - if (outboundDataRateOut) { - outboundDataRateOut->jointDataRate.increment(numBytes); - outboundDataRateOut->farGrabJointRate.increment(numGrabJointBytes); - } + if (sendStatus.rotationsSent != numJoints || sendStatus.translationsSent != numJoints) { + extraReturnedFlags |= AvatarDataPacket::PACKET_HAS_JOINT_DATA; + } + + int numBytes = destinationBuffer - startSection; + if (outboundDataRateOut) { + outboundDataRateOut->jointDataRate.increment(numBytes); + outboundDataRateOut->farGrabJointRate.increment(numGrabJointBytes); } } @@ -812,7 +802,7 @@ QByteArray AvatarData::toByteArray(AvatarDataDetail dataDetail, quint64 lastSent memcpy(packetFlagsLocation, &includedFlags, sizeof(includedFlags)); // Return dropped items. - itemFlags = packetStateFlags & ~includedFlags; + sendStatus.itemFlags = (packetStateFlags & ~includedFlags) | extraReturnedFlags; int avatarDataSize = destinationBuffer - startPosition; diff --git a/libraries/avatars/src/AvatarData.h b/libraries/avatars/src/AvatarData.h index 5b5d49feea..3ed90c059a 100644 --- a/libraries/avatars/src/AvatarData.h +++ b/libraries/avatars/src/AvatarData.h @@ -298,7 +298,14 @@ namespace AvatarDataPacket { static_assert(sizeof(FarGrabJoints) == FAR_GRAB_JOINTS_SIZE, "AvatarDataPacket::FarGrabJoints size doesn't match."); static const size_t MIN_BULK_PACKET_SIZE = NUM_BYTES_RFC4122_UUID + HEADER_SIZE; - static const int MAX_NUM_JOINTS = 111; + static const size_t FAUX_JOINT_SIZE = 2 * (sizeof(SixByteQuat) + sizeof(SixByteTrans)); + + struct SendStatus { + HasFlags itemFlags { 0 }; + int rotationsSent { 0 }; + int translationsSent { 0 }; + operator bool() { return itemFlags == 0; } + }; } const float MAX_AUDIO_LOUDNESS = 1000.0f; // close enough for mouth animation @@ -452,7 +459,7 @@ public: virtual QByteArray toByteArrayStateful(AvatarDataDetail dataDetail, bool dropFaceTracking = false); virtual QByteArray toByteArray(AvatarDataDetail dataDetail, quint64 lastSentTime, const QVector& lastSentJointData, - AvatarDataPacket::HasFlags& hasFlagsOut, bool dropFaceTracking, bool distanceAdjust, glm::vec3 viewerPosition, + AvatarDataPacket::SendStatus& sendStatus, bool dropFaceTracking, bool distanceAdjust, glm::vec3 viewerPosition, QVector* sentJointDataOut, int maxDataSize = 0, AvatarDataRate* outboundDataRateOut = nullptr) const; virtual void doneEncoding(bool cullSmallChanges); From f95ab1b040aca9562fe222b08321a4b5e05cc56a Mon Sep 17 00:00:00 2001 From: Simon Walton Date: Tue, 25 Sep 2018 12:07:59 -0700 Subject: [PATCH 13/22] Clean-up & other small changes --- .../src/avatars/AvatarMixerSlave.cpp | 4 +- libraries/avatars/src/AvatarData.cpp | 51 ++++++++++--------- libraries/avatars/src/AvatarData.h | 2 +- 3 files changed, 29 insertions(+), 28 deletions(-) diff --git a/assignment-client/src/avatars/AvatarMixerSlave.cpp b/assignment-client/src/avatars/AvatarMixerSlave.cpp index 3810456f71..6a1241756c 100644 --- a/assignment-client/src/avatars/AvatarMixerSlave.cpp +++ b/assignment-client/src/avatars/AvatarMixerSlave.cpp @@ -244,7 +244,7 @@ void AvatarMixerSlave::broadcastAvatarDataToAgent(const SharedNodePointer& node) // reset the internal state for correct random number distribution distribution.reset(); - // Estimate number to sort on number sent last frame. + // Estimate number to sort on number sent last frame (with min. of 20). const int numToSendEst = std::max(nodeData->getNumAvatarsSentLastFrame() * 2, 20); // reset the number of sent avatars @@ -464,7 +464,7 @@ void AvatarMixerSlave::broadcastAvatarDataToAgent(const SharedNodePointer& node) avatarPacket->write(bytes); avatarSpaceAvailable -= bytes.size(); numAvatarDataBytes += bytes.size(); - if (sendStatus.itemFlags != 0 || avatarSpaceAvailable < (int)AvatarDataPacket::MIN_BULK_PACKET_SIZE) { + if (!sendStatus || avatarSpaceAvailable < (int)AvatarDataPacket::MIN_BULK_PACKET_SIZE) { // Weren't able to fit everything. nodeList->sendPacket(std::move(avatarPacket), *destinationNode); ++numPacketsSent; diff --git a/libraries/avatars/src/AvatarData.cpp b/libraries/avatars/src/AvatarData.cpp index f393ca9ba8..4295eb66a7 100644 --- a/libraries/avatars/src/AvatarData.cpp +++ b/libraries/avatars/src/AvatarData.cpp @@ -245,13 +245,17 @@ QByteArray AvatarData::toByteArray(AvatarDataDetail dataDetail, quint64 lastSent lazyInitHeadData(); ASSERT(maxDataSize == 0 || (size_t)maxDataSize >= AvatarDataPacket::MIN_BULK_PACKET_SIZE); + // Leading flags, to indicate how much data is actually included in the packet... + AvatarDataPacket::HasFlags wantedFlags = 0; + AvatarDataPacket::HasFlags includedFlags = 0; + AvatarDataPacket::HasFlags extraReturnedFlags = 0; // For partial joint data. + // special case, if we were asked for no data, then just include the flags all set to nothing if (dataDetail == NoData) { - AvatarDataPacket::HasFlags packetStateFlags = 0; - sendStatus.itemFlags = packetStateFlags; + sendStatus.itemFlags = wantedFlags; - QByteArray avatarDataByteArray(getSessionUUID().toRfc4122().data(), NUM_BYTES_RFC4122_UUID + sizeof(packetStateFlags)); - avatarDataByteArray.append((char*) &packetStateFlags, sizeof packetStateFlags); + QByteArray avatarDataByteArray(getSessionUUID().toRfc4122().data(), NUM_BYTES_RFC4122_UUID + sizeof wantedFlags); + avatarDataByteArray.append((char*) &wantedFlags, sizeof wantedFlags); return avatarDataByteArray; } @@ -280,10 +284,8 @@ QByteArray AvatarData::toByteArray(AvatarDataDetail dataDetail, quint64 lastSent glm::mat4 rightFarGrabMatrix; glm::mat4 mouseFarGrabMatrix; - // Leading flags, to indicate how much data is actually included in the packet... - AvatarDataPacket::HasFlags packetStateFlags = 0; - if (sendStatus.itemFlags == 0) { + // New avatar ... bool hasAvatarGlobalPosition = true; // always include global position bool hasAvatarOrientation = false; bool hasAvatarBoundingBox = false; @@ -324,7 +326,7 @@ QByteArray AvatarData::toByteArray(AvatarDataDetail dataDetail, quint64 lastSent hasJointDefaultPoseFlags = hasJointData; } - packetStateFlags = + wantedFlags = (hasAvatarGlobalPosition ? AvatarDataPacket::PACKET_HAS_AVATAR_GLOBAL_POSITION : 0) | (hasAvatarBoundingBox ? AvatarDataPacket::PACKET_HAS_AVATAR_BOUNDING_BOX : 0) | (hasAvatarOrientation ? AvatarDataPacket::PACKET_HAS_AVATAR_ORIENTATION : 0) @@ -340,17 +342,18 @@ QByteArray AvatarData::toByteArray(AvatarDataDetail dataDetail, quint64 lastSent | (hasJointDefaultPoseFlags ? AvatarDataPacket::PACKET_HAS_JOINT_DEFAULT_POSE_FLAGS : 0) | (hasJointData ? AvatarDataPacket::PACKET_HAS_GRAB_JOINTS : 0); - sendStatus.itemFlags = packetStateFlags; + sendStatus.itemFlags = wantedFlags; sendStatus.rotationsSent = 0; sendStatus.translationsSent = 0; - } else { - packetStateFlags = sendStatus.itemFlags; - if (packetStateFlags & AvatarDataPacket::PACKET_HAS_GRAB_JOINTS) { - packetStateFlags |= AvatarDataPacket::PACKET_HAS_JOINT_DATA; + } else { // Continuing avatar ... + wantedFlags = sendStatus.itemFlags; + if (wantedFlags & AvatarDataPacket::PACKET_HAS_GRAB_JOINTS) { + // Must send joints for grab joints - + wantedFlags |= AvatarDataPacket::PACKET_HAS_JOINT_DATA; } } - if (packetStateFlags & AvatarDataPacket::PACKET_HAS_GRAB_JOINTS) { + if (wantedFlags & AvatarDataPacket::PACKET_HAS_GRAB_JOINTS) { bool leftValid; leftFarGrabMatrix = _farGrabLeftMatrixCache.get(leftValid); if (!leftValid) { @@ -367,10 +370,10 @@ QByteArray AvatarData::toByteArray(AvatarDataDetail dataDetail, quint64 lastSent mouseFarGrabMatrix = glm::mat4(); } if (!(leftValid || rightValid || mouseValid)) { - packetStateFlags &= ~AvatarDataPacket::PACKET_HAS_GRAB_JOINTS; + wantedFlags &= ~AvatarDataPacket::PACKET_HAS_GRAB_JOINTS; } } - if (packetStateFlags & (AvatarDataPacket::PACKET_HAS_ADDITIONAL_FLAGS | AvatarDataPacket::PACKET_HAS_PARENT_INFO)) { + if (wantedFlags & (AvatarDataPacket::PACKET_HAS_ADDITIONAL_FLAGS | AvatarDataPacket::PACKET_HAS_PARENT_INFO)) { parentID = getParentID(); } @@ -388,24 +391,22 @@ QByteArray AvatarData::toByteArray(AvatarDataDetail dataDetail, quint64 lastSent const unsigned char* const startPosition = destinationBuffer; const unsigned char* const packetEnd = destinationBuffer + maxDataSize; - AvatarDataPacket::HasFlags includedFlags = 0; - AvatarDataPacket::HasFlags extraReturnedFlags = 0; - // Packets always have UUID. memcpy(destinationBuffer, getSessionUUID().toRfc4122(), NUM_BYTES_RFC4122_UUID); destinationBuffer += NUM_BYTES_RFC4122_UUID; unsigned char * packetFlagsLocation = destinationBuffer; - destinationBuffer += sizeof(packetStateFlags); + destinationBuffer += sizeof(wantedFlags); #define AVATAR_MEMCPY(src) \ memcpy(destinationBuffer, &(src), sizeof(src)); \ destinationBuffer += sizeof(src); // If we want an item and there's sufficient space: -#define IF_AVATAR_SPACE(flag, space) \ - if ((packetStateFlags & AvatarDataPacket::flag) && (size_t)(packetEnd - destinationBuffer) >= (size_t)(space) \ - && (includedFlags |= AvatarDataPacket::flag)) +#define IF_AVATAR_SPACE(flag, space) \ + if ((wantedFlags & AvatarDataPacket::flag) \ + && (size_t)(packetEnd - destinationBuffer) >= (size_t)(space) \ + && (includedFlags |= AvatarDataPacket::flag)) IF_AVATAR_SPACE(PACKET_HAS_AVATAR_GLOBAL_POSITION, sizeof _globalPosition) { auto startSection = destinationBuffer; @@ -589,7 +590,7 @@ QByteArray AvatarData::toByteArray(AvatarDataDetail dataDetail, quint64 lastSent } QVector jointData; - if (packetStateFlags & (AvatarDataPacket::PACKET_HAS_JOINT_DATA | AvatarDataPacket::PACKET_HAS_JOINT_DEFAULT_POSE_FLAGS)) { + if (wantedFlags & (AvatarDataPacket::PACKET_HAS_JOINT_DATA | AvatarDataPacket::PACKET_HAS_JOINT_DEFAULT_POSE_FLAGS)) { QReadLocker readLock(&_jointDataLock); jointData = _jointData; } @@ -802,7 +803,7 @@ QByteArray AvatarData::toByteArray(AvatarDataDetail dataDetail, quint64 lastSent memcpy(packetFlagsLocation, &includedFlags, sizeof(includedFlags)); // Return dropped items. - sendStatus.itemFlags = (packetStateFlags & ~includedFlags) | extraReturnedFlags; + sendStatus.itemFlags = (wantedFlags & ~includedFlags) | extraReturnedFlags; int avatarDataSize = destinationBuffer - startPosition; diff --git a/libraries/avatars/src/AvatarData.h b/libraries/avatars/src/AvatarData.h index 3ed90c059a..40334f9b86 100644 --- a/libraries/avatars/src/AvatarData.h +++ b/libraries/avatars/src/AvatarData.h @@ -302,7 +302,7 @@ namespace AvatarDataPacket { struct SendStatus { HasFlags itemFlags { 0 }; - int rotationsSent { 0 }; + int rotationsSent { 0 }; // ie: index of next unsent joint int translationsSent { 0 }; operator bool() { return itemFlags == 0; } }; From 13890f1d16bf2c33945c9ce2aef0c668b276ae09 Mon Sep 17 00:00:00 2001 From: Simon Walton Date: Wed, 26 Sep 2018 09:47:26 -0700 Subject: [PATCH 14/22] Use local ID in place of UUID in a couple more cases --- .../src/avatars/AvatarMixerClientData.cpp | 10 +++++----- assignment-client/src/avatars/AvatarMixerClientData.h | 10 +++++----- assignment-client/src/avatars/AvatarMixerSlave.cpp | 6 +++--- libraries/avatars/src/AvatarData.cpp | 1 - 4 files changed, 13 insertions(+), 14 deletions(-) diff --git a/assignment-client/src/avatars/AvatarMixerClientData.cpp b/assignment-client/src/avatars/AvatarMixerClientData.cpp index 2226a4c7a6..801ef61abf 100644 --- a/assignment-client/src/avatars/AvatarMixerClientData.cpp +++ b/assignment-client/src/avatars/AvatarMixerClientData.cpp @@ -26,20 +26,20 @@ AvatarMixerClientData::AvatarMixerClientData(const QUuid& nodeID, Node::LocalID _avatar->setID(nodeID); } -uint64_t AvatarMixerClientData::getLastOtherAvatarEncodeTime(QUuid otherAvatar) const { - std::unordered_map::const_iterator itr = _lastOtherAvatarEncodeTime.find(otherAvatar); +uint64_t AvatarMixerClientData::getLastOtherAvatarEncodeTime(NLPacket::LocalID otherAvatar) const { + const auto itr = _lastOtherAvatarEncodeTime.find(otherAvatar); if (itr != _lastOtherAvatarEncodeTime.end()) { return itr->second; } return 0; } -void AvatarMixerClientData::setLastOtherAvatarEncodeTime(const QUuid& otherAvatar, uint64_t time) { - std::unordered_map::iterator itr = _lastOtherAvatarEncodeTime.find(otherAvatar); +void AvatarMixerClientData::setLastOtherAvatarEncodeTime(NLPacket::LocalID otherAvatar, uint64_t time) { + auto itr = _lastOtherAvatarEncodeTime.find(otherAvatar); if (itr != _lastOtherAvatarEncodeTime.end()) { itr->second = time; } else { - _lastOtherAvatarEncodeTime.emplace(std::pair(otherAvatar, time)); + _lastOtherAvatarEncodeTime.emplace(std::pair(otherAvatar, time)); } } diff --git a/assignment-client/src/avatars/AvatarMixerClientData.h b/assignment-client/src/avatars/AvatarMixerClientData.h index 492d21a807..634c202611 100644 --- a/assignment-client/src/avatars/AvatarMixerClientData.h +++ b/assignment-client/src/avatars/AvatarMixerClientData.h @@ -111,10 +111,10 @@ public: const ConicalViewFrustums& getViewFrustums() const { return _currentViewFrustums; } - uint64_t getLastOtherAvatarEncodeTime(QUuid otherAvatar) const; - void setLastOtherAvatarEncodeTime(const QUuid& otherAvatar, uint64_t time); + uint64_t getLastOtherAvatarEncodeTime(NLPacket::LocalID otherAvatar) const; + void setLastOtherAvatarEncodeTime(NLPacket::LocalID otherAvatar, uint64_t time); - QVector& getLastOtherAvatarSentJoints(QUuid otherAvatar) { return _lastOtherAvatarSentJoints[otherAvatar]; } + QVector& getLastOtherAvatarSentJoints(NLPacket::LocalID otherAvatar) { return _lastOtherAvatarSentJoints[otherAvatar]; } void queuePacket(QSharedPointer message, SharedNodePointer node); int processPackets(const SlaveSharedData& slaveSharedData); // returns number of packets processed @@ -152,8 +152,8 @@ private: // this is a map of the last time we encoded an "other" avatar for // sending to "this" node - std::unordered_map _lastOtherAvatarEncodeTime; - std::unordered_map> _lastOtherAvatarSentJoints; + std::unordered_map _lastOtherAvatarEncodeTime; + std::unordered_map> _lastOtherAvatarSentJoints; uint64_t _identityChangeTimestamp; bool _avatarSessionDisplayNameMustChange{ true }; diff --git a/assignment-client/src/avatars/AvatarMixerSlave.cpp b/assignment-client/src/avatars/AvatarMixerSlave.cpp index 6a1241756c..5e1e1019e4 100644 --- a/assignment-client/src/avatars/AvatarMixerSlave.cpp +++ b/assignment-client/src/avatars/AvatarMixerSlave.cpp @@ -382,7 +382,7 @@ void AvatarMixerSlave::broadcastAvatarDataToAgent(const SharedNodePointer& node) if (!shouldIgnore) { // sort this one for later const AvatarData* avatarNodeData = avatarClientNodeData->getConstAvatarData(); - auto lastEncodeTime = nodeData->getLastOtherAvatarEncodeTime(avatarNodeData->getSessionUUID()); + auto lastEncodeTime = nodeData->getLastOtherAvatarEncodeTime(avatarNode->getLocalID()); sortedAvatars.push(SortableAvatar(avatarNodeData, avatarNode, lastEncodeTime)); } @@ -446,7 +446,7 @@ void AvatarMixerSlave::broadcastAvatarDataToAgent(const SharedNodePointer& node) nodeData->incrementAvatarInView(); } - QVector& lastSentJointsForOther = nodeData->getLastOtherAvatarSentJoints(otherNode->getUUID()); + QVector& lastSentJointsForOther = nodeData->getLastOtherAvatarSentJoints(otherNode->getLocalID()); const bool distanceAdjust = true; const bool dropFaceTracking = false; @@ -482,7 +482,7 @@ void AvatarMixerSlave::broadcastAvatarDataToAgent(const SharedNodePointer& node) // set the last sent sequence number for this sender on the receiver nodeData->setLastBroadcastSequenceNumber(otherNode->getLocalID(), otherNodeData->getLastReceivedSequenceNumber()); - nodeData->setLastOtherAvatarEncodeTime(otherNode->getUUID(), usecTimestampNow()); + nodeData->setLastOtherAvatarEncodeTime(otherNode->getLocalID(), usecTimestampNow()); } auto endAvatarDataPacking = chrono::high_resolution_clock::now(); diff --git a/libraries/avatars/src/AvatarData.cpp b/libraries/avatars/src/AvatarData.cpp index 2282745905..91e7939d9c 100644 --- a/libraries/avatars/src/AvatarData.cpp +++ b/libraries/avatars/src/AvatarData.cpp @@ -222,7 +222,6 @@ float AvatarData::getDistanceBasedMinTranslationDistance(glm::vec3 viewerPositio // we want to track outbound data in this case... QByteArray AvatarData::toByteArrayStateful(AvatarDataDetail dataDetail, bool dropFaceTracking) { - AvatarDataPacket::HasFlags hasFlagsOut = 0; auto lastSentTime = _lastToByteArray; _lastToByteArray = usecTimestampNow(); AvatarDataPacket::SendStatus sendStatus; From 1e79329e834bd8f4e50a4e71f67a40694f6f9c84 Mon Sep 17 00:00:00 2001 From: Simon Walton Date: Wed, 26 Sep 2018 14:23:52 -0700 Subject: [PATCH 15/22] Access joint data via C pointer instead of operator[] QVector::detach() was showing up in the linux profiles - it's related to the copy-on-write capability. Const QVectors don't use it though. Also a bug fix for the minimum joint space needed. --- libraries/avatars/src/AvatarData.cpp | 39 +++++++++++++++------------- libraries/avatars/src/AvatarData.h | 2 +- 2 files changed, 22 insertions(+), 19 deletions(-) diff --git a/libraries/avatars/src/AvatarData.cpp b/libraries/avatars/src/AvatarData.cpp index 91e7939d9c..7fb0f786ba 100644 --- a/libraries/avatars/src/AvatarData.cpp +++ b/libraries/avatars/src/AvatarData.cpp @@ -402,9 +402,9 @@ QByteArray AvatarData::toByteArray(AvatarDataDetail dataDetail, quint64 lastSent destinationBuffer += sizeof(src); // If we want an item and there's sufficient space: -#define IF_AVATAR_SPACE(flag, space) \ - if ((wantedFlags & AvatarDataPacket::flag) \ - && (size_t)(packetEnd - destinationBuffer) >= (size_t)(space) \ +#define IF_AVATAR_SPACE(flag, space) \ + if ((wantedFlags & AvatarDataPacket::flag) \ + && (packetEnd - destinationBuffer) >= (ptrdiff_t)(space) \ && (includedFlags |= AvatarDataPacket::flag)) IF_AVATAR_SPACE(PACKET_HAS_AVATAR_GLOBAL_POSITION, sizeof _globalPosition) { @@ -597,10 +597,11 @@ QByteArray AvatarData::toByteArray(AvatarDataDetail dataDetail, quint64 lastSent assert(numJoints <= 255); const int jointBitVectorSize = calcBitVectorSize(numJoints); - IF_AVATAR_SPACE(PACKET_HAS_JOINT_DATA, 1 + 2 * jointBitVectorSize + AvatarDataPacket::FAUX_JOINT_SIZE) { + // Start joints if room for at least the faux joints. + IF_AVATAR_SPACE(PACKET_HAS_JOINT_DATA, 1 + 2 * jointBitVectorSize + AvatarDataPacket::FAUX_JOINTS_SIZE) { // Allow for faux joints + translation bit-vector: - static const ptrdiff_t MIN_SIZE_FOR_JOINT = sizeof(AvatarDataPacket::SixByteQuat) - + jointBitVectorSize + AvatarDataPacket::FAUX_JOINT_SIZE; + const ptrdiff_t minSizeForJoint = sizeof(AvatarDataPacket::SixByteQuat) + + jointBitVectorSize + AvatarDataPacket::FAUX_JOINTS_SIZE; auto startSection = destinationBuffer; // joint rotation data @@ -620,15 +621,17 @@ QByteArray AvatarData::toByteArray(AvatarDataDetail dataDetail, quint64 lastSent if (sentJointDataOut) { sentJointDataOut->resize(numJoints); // Make sure the destination is resized before using it } + const JointData *const joints = jointData.data(); + JointData *const sentJoints = sentJointDataOut ? sentJointDataOut->data() : nullptr; float minRotationDOT = (distanceAdjust && cullSmallChanges) ? getDistanceBasedMinRotationDOT(viewerPosition) : AVATAR_MIN_ROTATION_DOT; int i = sendStatus.rotationsSent; for (; i < numJoints; ++i) { - const JointData& data = jointData[i]; + const JointData& data = joints[i]; const JointData& last = lastSentJointData[i]; - if ((packetEnd - destinationBuffer) >= MIN_SIZE_FOR_JOINT) { + if (packetEnd - destinationBuffer >= minSizeForJoint) { if (!data.rotationIsDefaultPose) { // The dot product for larger rotations is a lower number, // so if the dot() is less than the value, then the rotation is a larger angle of rotation @@ -640,8 +643,8 @@ QByteArray AvatarData::toByteArray(AvatarDataDetail dataDetail, quint64 lastSent #endif destinationBuffer += packOrientationQuatToSixBytes(destinationBuffer, data.rotation); - if (sentJointDataOut) { - (*sentJointDataOut)[i].rotation = data.rotation; + if (sentJoints) { + sentJoints[i].rotation = data.rotation; } } } @@ -649,8 +652,8 @@ QByteArray AvatarData::toByteArray(AvatarDataDetail dataDetail, quint64 lastSent break; } - if (sentJointDataOut) { - (*sentJointDataOut)[i].rotationIsDefaultPose = data.rotationIsDefaultPose; + if (sentJoints) { + sentJoints[i].rotationIsDefaultPose = data.rotationIsDefaultPose; } } @@ -672,10 +675,10 @@ QByteArray AvatarData::toByteArray(AvatarDataDetail dataDetail, quint64 lastSent float maxTranslationDimension = 0.0; i = sendStatus.translationsSent; for (; i < numJoints; ++i) { - const JointData& data = jointData[i]; + const JointData& data = joints[i]; const JointData& last = lastSentJointData[i]; - if (packetEnd - destinationBuffer > MIN_SIZE_FOR_JOINT) { + if (packetEnd - destinationBuffer >= minSizeForJoint) { if (!data.translationIsDefaultPose) { if (sendAll || last.translationIsDefaultPose || (!cullSmallChanges && last.translation != data.translation) || (cullSmallChanges && glm::distance(data.translation, lastSentJointData[i].translation) > minTranslation)) { @@ -690,8 +693,8 @@ QByteArray AvatarData::toByteArray(AvatarDataDetail dataDetail, quint64 lastSent destinationBuffer += packFloatVec3ToSignedTwoByteFixed(destinationBuffer, data.translation, TRANSLATION_COMPRESSION_RADIX); - if (sentJointDataOut) { - (*sentJointDataOut)[i].translation = data.translation; + if (sentJoints) { + sentJoints[i].translation = data.translation; } } } @@ -699,8 +702,8 @@ QByteArray AvatarData::toByteArray(AvatarDataDetail dataDetail, quint64 lastSent break; } - if (sentJointDataOut) { - (*sentJointDataOut)[i].translationIsDefaultPose = data.translationIsDefaultPose; + if (sentJoints) { + sentJoints[i].translationIsDefaultPose = data.translationIsDefaultPose; } } diff --git a/libraries/avatars/src/AvatarData.h b/libraries/avatars/src/AvatarData.h index a4dfaa594e..3ca8bd4775 100644 --- a/libraries/avatars/src/AvatarData.h +++ b/libraries/avatars/src/AvatarData.h @@ -298,7 +298,7 @@ namespace AvatarDataPacket { static_assert(sizeof(FarGrabJoints) == FAR_GRAB_JOINTS_SIZE, "AvatarDataPacket::FarGrabJoints size doesn't match."); static const size_t MIN_BULK_PACKET_SIZE = NUM_BYTES_RFC4122_UUID + HEADER_SIZE; - static const size_t FAUX_JOINT_SIZE = 2 * (sizeof(SixByteQuat) + sizeof(SixByteTrans)); + static const size_t FAUX_JOINTS_SIZE = 2 * (sizeof(SixByteQuat) + sizeof(SixByteTrans)); struct SendStatus { HasFlags itemFlags { 0 }; From 7a0043c01002e359e291b931eaaa166de87eb846 Mon Sep 17 00:00:00 2001 From: Simon Walton Date: Fri, 28 Sep 2018 10:26:52 -0700 Subject: [PATCH 16/22] Send AvatarIdentity in NLPacketList; be more selective --- .../src/avatars/AvatarMixerSlave.cpp | 37 +++++++++++-------- .../src/avatars/AvatarMixerSlave.h | 2 +- 2 files changed, 23 insertions(+), 16 deletions(-) diff --git a/assignment-client/src/avatars/AvatarMixerSlave.cpp b/assignment-client/src/avatars/AvatarMixerSlave.cpp index 5e1e1019e4..3834516b80 100644 --- a/assignment-client/src/avatars/AvatarMixerSlave.cpp +++ b/assignment-client/src/avatars/AvatarMixerSlave.cpp @@ -68,13 +68,13 @@ void AvatarMixerSlave::processIncomingPackets(const SharedNodePointer& node) { _stats.processIncomingPacketsElapsedTime += (end - start); } -int AvatarMixerSlave::sendIdentityPacket(const AvatarMixerClientData* nodeData, const SharedNodePointer& destinationNode) { - if (destinationNode->getType() == NodeType::Agent && !destinationNode->isUpstream()) { +int AvatarMixerSlave::sendIdentityPacket(NLPacketList& packetList, const AvatarMixerClientData* nodeData, const Node& destinationNode) { + if (destinationNode.getType() == NodeType::Agent && !destinationNode.isUpstream()) { QByteArray individualData = nodeData->getConstAvatarData()->identityByteArray(); individualData.replace(0, NUM_BYTES_RFC4122_UUID, nodeData->getNodeID().toRfc4122()); // FIXME, this looks suspicious - auto identityPackets = NLPacketList::create(PacketType::AvatarIdentity, QByteArray(), true, true); - identityPackets->write(individualData); - DependencyManager::get()->sendPacketList(std::move(identityPackets), *destinationNode); + packetList.startSegment(); + packetList.write(individualData); + packetList.endSegment(); _stats.numIdentityPackets++; return individualData.size(); } else { @@ -396,6 +396,7 @@ void AvatarMixerSlave::broadcastAvatarDataToAgent(const SharedNodePointer& node) const int avatarPacketCapacity = avatarPacket->getPayloadCapacity(); int avatarSpaceAvailable = avatarPacketCapacity; int numPacketsSent = 0; + auto identityPacketList = NLPacketList::create(PacketType::AvatarIdentity, QByteArray(), true, true); const auto& sortedAvatarVector = sortedAvatars.getSortedVector(numToSendEst); for (const auto& sortedAvatar : sortedAvatarVector) { @@ -425,16 +426,6 @@ void AvatarMixerSlave::broadcastAvatarDataToAgent(const SharedNodePointer& node) const AvatarMixerClientData* otherNodeData = reinterpret_cast(otherNode->getLinkedData()); const AvatarData* otherAvatar = otherNodeData->getConstAvatarData(); - // If the time that the mixer sent AVATAR DATA about Avatar B to Avatar A is BEFORE OR EQUAL TO - // the time that Avatar B flagged an IDENTITY DATA change, send IDENTITY DATA about Avatar B to Avatar A. - if (otherAvatar->hasProcessedFirstIdentity() - && nodeData->getLastBroadcastTime(otherNode->getLocalID()) <= otherNodeData->getIdentityChangeTimestamp()) { - identityBytesSent += sendIdentityPacket(otherNodeData, node); - - // remember the last time we sent identity details about this other node to the receiver - nodeData->setLastBroadcastTime(otherNode->getLocalID(), usecTimestampNow()); - } - // Typically all out-of-view avatars but such avatars' priorities will rise with time: bool isLowerPriority = sortedAvatar.getPriority() <= OUT_OF_VIEW_THRESHOLD; @@ -444,6 +435,16 @@ void AvatarMixerSlave::broadcastAvatarDataToAgent(const SharedNodePointer& node) } else if (!overBudget) { detail = distribution(generator) < AVATAR_SEND_FULL_UPDATE_RATIO ? AvatarData::SendAllData : AvatarData::CullSmallData; nodeData->incrementAvatarInView(); + + // If the time that the mixer sent AVATAR DATA about Avatar B to Avatar A is BEFORE OR EQUAL TO + // the time that Avatar B flagged an IDENTITY DATA change, send IDENTITY DATA about Avatar B to Avatar A. + if (otherAvatar->hasProcessedFirstIdentity() + && nodeData->getLastBroadcastTime(otherNode->getLocalID()) <= otherNodeData->getIdentityChangeTimestamp()) { + identityBytesSent += sendIdentityPacket(*identityPacketList, otherNodeData, *destinationNode); + + // remember the last time we sent identity details about this other node to the receiver + nodeData->setLastBroadcastTime(otherNode->getLocalID(), usecTimestampNow()); + } } QVector& lastSentJointsForOther = nodeData->getLastOtherAvatarSentJoints(otherNode->getLocalID()); @@ -520,6 +521,12 @@ void AvatarMixerSlave::broadcastAvatarDataToAgent(const SharedNodePointer& node) nodeList->sendPacketList(std::move(traitsPacketList), *destinationNode); } + // Send any AvatarIdentity packets: + identityPacketList->closeCurrentPacket(); + if (identityBytesSent > 0) { + nodeList->sendPacketList(std::move(identityPacketList), *destinationNode); + } + // record the number of avatars held back this frame nodeData->recordNumOtherAvatarStarves(numAvatarsHeldBack); nodeData->recordNumOtherAvatarSkips(numAvatarsWithSkippedFrames); diff --git a/assignment-client/src/avatars/AvatarMixerSlave.h b/assignment-client/src/avatars/AvatarMixerSlave.h index bcb70f8743..2ef90af38e 100644 --- a/assignment-client/src/avatars/AvatarMixerSlave.h +++ b/assignment-client/src/avatars/AvatarMixerSlave.h @@ -101,7 +101,7 @@ public: void harvestStats(AvatarMixerSlaveStats& stats); private: - int sendIdentityPacket(const AvatarMixerClientData* nodeData, const SharedNodePointer& destinationNode); + int sendIdentityPacket(NLPacketList& packet, const AvatarMixerClientData* nodeData, const Node& destinationNode); int sendReplicatedIdentityPacket(const Node& agentNode, const AvatarMixerClientData* nodeData, const Node& destinationNode); qint64 addChangedTraitsToBulkPacket(AvatarMixerClientData* listeningNodeData, From 18c1371321cea741e2c9b9ec6f49eef0759ca3b7 Mon Sep 17 00:00:00 2001 From: Simon Walton Date: Fri, 28 Sep 2018 17:53:06 -0700 Subject: [PATCH 17/22] Process multiple avatars in an AvatarIdentity message --- assignment-client/src/avatars/AvatarMixer.cpp | 2 +- libraries/avatars/src/AvatarData.cpp | 21 +++--- libraries/avatars/src/AvatarData.h | 2 +- libraries/avatars/src/AvatarHashMap.cpp | 70 ++++++++++--------- 4 files changed, 50 insertions(+), 45 deletions(-) diff --git a/assignment-client/src/avatars/AvatarMixer.cpp b/assignment-client/src/avatars/AvatarMixer.cpp index ebd86b749b..168c95cb77 100644 --- a/assignment-client/src/avatars/AvatarMixer.cpp +++ b/assignment-client/src/avatars/AvatarMixer.cpp @@ -565,7 +565,7 @@ void AvatarMixer::handleAvatarIdentityPacket(QSharedPointer mes // parse the identity packet and update the change timestamp if appropriate bool identityChanged = false; bool displayNameChanged = false; - avatar.processAvatarIdentity(message->getMessage(), identityChanged, displayNameChanged); + avatar.processAvatarIdentity(QDataStream(message->getMessage()), identityChanged, displayNameChanged); if (identityChanged) { QMutexLocker nodeDataLocker(&nodeData->getMutex()); diff --git a/libraries/avatars/src/AvatarData.cpp b/libraries/avatars/src/AvatarData.cpp index 7fb0f786ba..07751d03e3 100644 --- a/libraries/avatars/src/AvatarData.cpp +++ b/libraries/avatars/src/AvatarData.cpp @@ -1767,11 +1767,9 @@ glm::quat AvatarData::getOrientationOutbound() const { return (getLocalOrientation()); } -void AvatarData::processAvatarIdentity(const QByteArray& identityData, bool& identityChanged, +void AvatarData::processAvatarIdentity(QDataStream& packetStream, bool& identityChanged, bool& displayNameChanged) { - QDataStream packetStream(identityData); - QUuid avatarSessionID; // peek the sequence number, this will tell us if we should be processing this identity packet at all @@ -1786,17 +1784,18 @@ void AvatarData::processAvatarIdentity(const QByteArray& identityData, bool& ide << (udt::SequenceNumber::Type) incomingSequenceNumber; } - if (incomingSequenceNumber > _identitySequenceNumber) { - Identity identity; + Identity identity; - packetStream - >> identity.attachmentData - >> identity.displayName - >> identity.sessionDisplayName - >> identity.isReplicated - >> identity.lookAtSnappingEnabled + packetStream + >> identity.attachmentData + >> identity.displayName + >> identity.sessionDisplayName + >> identity.isReplicated + >> identity.lookAtSnappingEnabled ; + if (incomingSequenceNumber > _identitySequenceNumber) { + // set the store identity sequence number to match the incoming identity _identitySequenceNumber = incomingSequenceNumber; diff --git a/libraries/avatars/src/AvatarData.h b/libraries/avatars/src/AvatarData.h index 3ca8bd4775..76b699a3d2 100644 --- a/libraries/avatars/src/AvatarData.h +++ b/libraries/avatars/src/AvatarData.h @@ -967,7 +967,7 @@ public: // identityChanged returns true if identity has changed, false otherwise. // identityChanged returns true if identity has changed, false otherwise. Similarly for displayNameChanged and skeletonModelUrlChange. - void processAvatarIdentity(const QByteArray& identityData, bool& identityChanged, bool& displayNameChanged); + void processAvatarIdentity(QDataStream& packetStream, bool& identityChanged, bool& displayNameChanged); qint64 packTrait(AvatarTraits::TraitType traitType, ExtendedIODevice& destination, AvatarTraits::TraitVersion traitVersion = AvatarTraits::NULL_TRAIT_VERSION); diff --git a/libraries/avatars/src/AvatarHashMap.cpp b/libraries/avatars/src/AvatarHashMap.cpp index d205a915f8..79ba5d06f9 100644 --- a/libraries/avatars/src/AvatarHashMap.cpp +++ b/libraries/avatars/src/AvatarHashMap.cpp @@ -70,7 +70,7 @@ void AvatarReplicas::processAvatarIdentity(const QUuid& parentID, const QByteArr if (_replicasMap.find(parentID) != _replicasMap.end()) { auto &replicas = _replicasMap[parentID]; for (auto avatar : replicas) { - avatar->processAvatarIdentity(identityData, identityChanged, displayNameChanged); + avatar->processAvatarIdentity(QDataStream(identityData), identityChanged, displayNameChanged); } } } @@ -266,41 +266,47 @@ AvatarSharedPointer AvatarHashMap::parseAvatarData(QSharedPointer message, SharedNodePointer sendingNode) { + QDataStream avatarIdentityStream(message->getMessage()); - // peek the avatar UUID from the incoming packet - QUuid identityUUID = QUuid::fromRfc4122(message->peek(NUM_BYTES_RFC4122_UUID)); + while (!avatarIdentityStream.atEnd()) { + // peek the avatar UUID from the incoming packet + avatarIdentityStream.startTransaction(); + QUuid identityUUID; + avatarIdentityStream >> identityUUID; + avatarIdentityStream.rollbackTransaction(); - if (identityUUID.isNull()) { - qCDebug(avatars) << "Refusing to process identity packet for null avatar ID"; - return; - } - - // make sure this isn't for an ignored avatar - auto nodeList = DependencyManager::get(); - static auto EMPTY = QUuid(); - - { - QReadLocker locker(&_hashLock); - _pendingAvatars.remove(identityUUID); - auto me = _avatarHash.find(EMPTY); - if ((me != _avatarHash.end()) && (identityUUID == me.value()->getSessionUUID())) { - // We add MyAvatar to _avatarHash with an empty UUID. Code relies on this. In order to correctly handle an - // identity packet for ourself (such as when we are assigned a sessionDisplayName by the mixer upon joining), - // we make things match here. - identityUUID = EMPTY; + if (identityUUID.isNull()) { + qCDebug(avatars) << "Refusing to process identity packet for null avatar ID"; + return; } - } - - if (!nodeList->isIgnoringNode(identityUUID) || nodeList->getRequestsDomainListData()) { - // mesh URL for a UUID, find avatar in our list - bool isNewAvatar; - auto avatar = newOrExistingAvatar(identityUUID, sendingNode, isNewAvatar); - bool identityChanged = false; - bool displayNameChanged = false; - // In this case, the "sendingNode" is the Avatar Mixer. - avatar->processAvatarIdentity(message->getMessage(), identityChanged, displayNameChanged); - _replicas.processAvatarIdentity(identityUUID, message->getMessage(), identityChanged, displayNameChanged); + // make sure this isn't for an ignored avatar + auto nodeList = DependencyManager::get(); + static auto EMPTY = QUuid(); + + { + QReadLocker locker(&_hashLock); + _pendingAvatars.remove(identityUUID); + auto me = _avatarHash.find(EMPTY); + if ((me != _avatarHash.end()) && (identityUUID == me.value()->getSessionUUID())) { + // We add MyAvatar to _avatarHash with an empty UUID. Code relies on this. In order to correctly handle an + // identity packet for ourself (such as when we are assigned a sessionDisplayName by the mixer upon joining), + // we make things match here. + identityUUID = EMPTY; + } + } + + if (!nodeList->isIgnoringNode(identityUUID) || nodeList->getRequestsDomainListData()) { + // mesh URL for a UUID, find avatar in our list + bool isNewAvatar; + auto avatar = newOrExistingAvatar(identityUUID, sendingNode, isNewAvatar); + bool identityChanged = false; + bool displayNameChanged = false; + // In this case, the "sendingNode" is the Avatar Mixer. + avatar->processAvatarIdentity(avatarIdentityStream, identityChanged, displayNameChanged); + _replicas.processAvatarIdentity(identityUUID, message->getMessage(), identityChanged, displayNameChanged); + + } } } From 4534b87ea07c934e927b70a6fd23c8d7f3d3dc84 Mon Sep 17 00:00:00 2001 From: Simon Walton Date: Fri, 28 Sep 2018 18:24:59 -0700 Subject: [PATCH 18/22] Fix error passing rvalue ref as lvalue --- libraries/avatars/src/AvatarHashMap.cpp | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/libraries/avatars/src/AvatarHashMap.cpp b/libraries/avatars/src/AvatarHashMap.cpp index 79ba5d06f9..e453f5d298 100644 --- a/libraries/avatars/src/AvatarHashMap.cpp +++ b/libraries/avatars/src/AvatarHashMap.cpp @@ -69,8 +69,9 @@ void AvatarReplicas::removeReplicas(const QUuid& parentID) { void AvatarReplicas::processAvatarIdentity(const QUuid& parentID, const QByteArray& identityData, bool& identityChanged, bool& displayNameChanged) { if (_replicasMap.find(parentID) != _replicasMap.end()) { auto &replicas = _replicasMap[parentID]; + QDataStream identityDataStream(identityData); for (auto avatar : replicas) { - avatar->processAvatarIdentity(QDataStream(identityData), identityChanged, displayNameChanged); + avatar->processAvatarIdentity(identityDataStream, identityChanged, displayNameChanged); } } } From 756d1a6fc4fd8fab92e060cce70bc201165a20f2 Mon Sep 17 00:00:00 2001 From: Simon Walton Date: Sat, 29 Sep 2018 21:15:10 -0700 Subject: [PATCH 19/22] Fix another bind-to-temporary --- assignment-client/src/avatars/AvatarMixer.cpp | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/assignment-client/src/avatars/AvatarMixer.cpp b/assignment-client/src/avatars/AvatarMixer.cpp index 168c95cb77..48f3edb94f 100644 --- a/assignment-client/src/avatars/AvatarMixer.cpp +++ b/assignment-client/src/avatars/AvatarMixer.cpp @@ -565,7 +565,8 @@ void AvatarMixer::handleAvatarIdentityPacket(QSharedPointer mes // parse the identity packet and update the change timestamp if appropriate bool identityChanged = false; bool displayNameChanged = false; - avatar.processAvatarIdentity(QDataStream(message->getMessage()), identityChanged, displayNameChanged); + QDataStream avatarIdentityStream(message->getMessage()); + avatar.processAvatarIdentity(avatarIdentityStream, identityChanged, displayNameChanged); if (identityChanged) { QMutexLocker nodeDataLocker(&nodeData->getMutex()); From 0ca112f7b91c379e0599fc113b388dfcea9cc170 Mon Sep 17 00:00:00 2001 From: Simon Walton Date: Thu, 4 Oct 2018 16:40:44 -0700 Subject: [PATCH 20/22] Restore missing abs() to quaternion dot-product See https://github.com/highfidelity/hifi/pull/14138 --- libraries/avatars/src/AvatarData.cpp | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/libraries/avatars/src/AvatarData.cpp b/libraries/avatars/src/AvatarData.cpp index 07751d03e3..57d36e1f0f 100644 --- a/libraries/avatars/src/AvatarData.cpp +++ b/libraries/avatars/src/AvatarData.cpp @@ -636,7 +636,7 @@ QByteArray AvatarData::toByteArray(AvatarDataDetail dataDetail, quint64 lastSent // The dot product for larger rotations is a lower number, // so if the dot() is less than the value, then the rotation is a larger angle of rotation if (sendAll || last.rotationIsDefaultPose || (!cullSmallChanges && last.rotation != data.rotation) - || (cullSmallChanges && glm::dot(last.rotation, data.rotation) < minRotationDOT)) { + || (cullSmallChanges && fabsf(glm::dot(last.rotation, data.rotation)) < minRotationDOT)) { validityPosition[i / BITS_IN_BYTE] |= 1 << (i % BITS_IN_BYTE); #ifdef WANT_DEBUG rotationSentCount++; From 0b77eb5e9ce7e5ebbab42f0db90dece9764a0e82 Mon Sep 17 00:00:00 2001 From: Simon Walton Date: Tue, 16 Oct 2018 14:25:38 -0700 Subject: [PATCH 21/22] Control UUID inclusion in avatar data with new param Increase estimate of avatars to be sent; remove start/end segment in avatar identity writes. --- .../src/avatars/AvatarMixerSlave.cpp | 5 ++-- libraries/avatars/src/AvatarData.cpp | 25 +++++++++++-------- libraries/avatars/src/AvatarData.h | 1 + 3 files changed, 17 insertions(+), 14 deletions(-) diff --git a/assignment-client/src/avatars/AvatarMixerSlave.cpp b/assignment-client/src/avatars/AvatarMixerSlave.cpp index 0e0802bf38..75d36783a5 100644 --- a/assignment-client/src/avatars/AvatarMixerSlave.cpp +++ b/assignment-client/src/avatars/AvatarMixerSlave.cpp @@ -72,9 +72,7 @@ int AvatarMixerSlave::sendIdentityPacket(NLPacketList& packetList, const AvatarM if (destinationNode.getType() == NodeType::Agent && !destinationNode.isUpstream()) { QByteArray individualData = nodeData->getConstAvatarData()->identityByteArray(); individualData.replace(0, NUM_BYTES_RFC4122_UUID, nodeData->getNodeID().toRfc4122()); // FIXME, this looks suspicious - packetList.startSegment(); packetList.write(individualData); - packetList.endSegment(); _stats.numIdentityPackets++; return individualData.size(); } else { @@ -248,7 +246,7 @@ void AvatarMixerSlave::broadcastAvatarDataToAgent(const SharedNodePointer& node) distribution.reset(); // Estimate number to sort on number sent last frame (with min. of 20). - const int numToSendEst = std::max(nodeData->getNumAvatarsSentLastFrame() * 2, 20); + const int numToSendEst = std::max(int(nodeData->getNumAvatarsSentLastFrame() * 2.5f), 20); // reset the number of sent avatars nodeData->resetNumAvatarsSentLastFrame(); @@ -455,6 +453,7 @@ void AvatarMixerSlave::broadcastAvatarDataToAgent(const SharedNodePointer& node) const bool distanceAdjust = true; const bool dropFaceTracking = false; AvatarDataPacket::SendStatus sendStatus; + sendStatus.sendUUID = true; do { auto startSerialize = chrono::high_resolution_clock::now(); diff --git a/libraries/avatars/src/AvatarData.cpp b/libraries/avatars/src/AvatarData.cpp index 422db2f2ed..3ac8b32d48 100644 --- a/libraries/avatars/src/AvatarData.cpp +++ b/libraries/avatars/src/AvatarData.cpp @@ -229,8 +229,7 @@ QByteArray AvatarData::toByteArrayStateful(AvatarDataDetail dataDetail, bool dro AvatarDataPacket::SendStatus sendStatus; auto avatarByteArray = AvatarData::toByteArray(dataDetail, lastSentTime, getLastSentJointData(), sendStatus, dropFaceTracking, false, glm::vec3(0), nullptr, 0, &_outboundDataRate); - // Strip UUID - return avatarByteArray.right(avatarByteArray.size() - NUM_BYTES_RFC4122_UUID); + return avatarByteArray; } QByteArray AvatarData::toByteArray(AvatarDataDetail dataDetail, quint64 lastSentTime, @@ -255,7 +254,11 @@ QByteArray AvatarData::toByteArray(AvatarDataDetail dataDetail, quint64 lastSent if (dataDetail == NoData) { sendStatus.itemFlags = wantedFlags; - QByteArray avatarDataByteArray(getSessionUUID().toRfc4122().data(), NUM_BYTES_RFC4122_UUID + sizeof wantedFlags); + QByteArray avatarDataByteArray; + if (sendStatus.sendUUID) { + avatarDataByteArray.append(getSessionUUID().toRfc4122().data(), NUM_BYTES_RFC4122_UUID); + } + avatarDataByteArray.append((char*) &wantedFlags, sizeof wantedFlags); return avatarDataByteArray; } @@ -392,13 +395,6 @@ QByteArray AvatarData::toByteArray(AvatarDataDetail dataDetail, quint64 lastSent const unsigned char* const startPosition = destinationBuffer; const unsigned char* const packetEnd = destinationBuffer + maxDataSize; - // Packets always have UUID. - memcpy(destinationBuffer, getSessionUUID().toRfc4122(), NUM_BYTES_RFC4122_UUID); - destinationBuffer += NUM_BYTES_RFC4122_UUID; - - unsigned char * packetFlagsLocation = destinationBuffer; - destinationBuffer += sizeof(wantedFlags); - #define AVATAR_MEMCPY(src) \ memcpy(destinationBuffer, &(src), sizeof(src)); \ destinationBuffer += sizeof(src); @@ -409,6 +405,14 @@ QByteArray AvatarData::toByteArray(AvatarDataDetail dataDetail, quint64 lastSent && (packetEnd - destinationBuffer) >= (ptrdiff_t)(space) \ && (includedFlags |= AvatarDataPacket::flag)) + if (sendStatus.sendUUID) { + memcpy(destinationBuffer, getSessionUUID().toRfc4122(), NUM_BYTES_RFC4122_UUID); + destinationBuffer += NUM_BYTES_RFC4122_UUID; + } + + unsigned char * packetFlagsLocation = destinationBuffer; + destinationBuffer += sizeof(wantedFlags); + IF_AVATAR_SPACE(PACKET_HAS_AVATAR_GLOBAL_POSITION, sizeof _globalPosition) { auto startSection = destinationBuffer; if (_overrideGlobalPosition) { @@ -417,7 +421,6 @@ QByteArray AvatarData::toByteArray(AvatarDataDetail dataDetail, quint64 lastSent AVATAR_MEMCPY(_globalPosition); } - int numBytes = destinationBuffer - startSection; if (outboundDataRateOut) { diff --git a/libraries/avatars/src/AvatarData.h b/libraries/avatars/src/AvatarData.h index aa3d3d328c..1f5b7093b9 100644 --- a/libraries/avatars/src/AvatarData.h +++ b/libraries/avatars/src/AvatarData.h @@ -302,6 +302,7 @@ namespace AvatarDataPacket { struct SendStatus { HasFlags itemFlags { 0 }; + bool sendUUID { false }; int rotationsSent { 0 }; // ie: index of next unsent joint int translationsSent { 0 }; operator bool() { return itemFlags == 0; } From b6f259fc42aa7ed03b2001056e46c2b71bfc2d83 Mon Sep 17 00:00:00 2001 From: Simon Walton Date: Wed, 24 Oct 2018 13:17:45 -0700 Subject: [PATCH 22/22] Fix incorrect merge-resolution --- libraries/avatars/src/AvatarData.cpp | 6 +----- 1 file changed, 1 insertion(+), 5 deletions(-) diff --git a/libraries/avatars/src/AvatarData.cpp b/libraries/avatars/src/AvatarData.cpp index e08f7fb6f8..7f42289a9b 100644 --- a/libraries/avatars/src/AvatarData.cpp +++ b/libraries/avatars/src/AvatarData.cpp @@ -419,11 +419,7 @@ QByteArray AvatarData::toByteArray(AvatarDataDetail dataDetail, quint64 lastSent IF_AVATAR_SPACE(PACKET_HAS_AVATAR_GLOBAL_POSITION, sizeof _globalPosition) { auto startSection = destinationBuffer; - if (_overrideGlobalPosition) { - AVATAR_MEMCPY(_globalPositionOverride); - } else { - AVATAR_MEMCPY(_globalPosition); - } + AVATAR_MEMCPY(_globalPosition); int numBytes = destinationBuffer - startSection;