diff --git a/assignment-client/src/Agent.cpp b/assignment-client/src/Agent.cpp index 806608cd5f..a458719346 100644 --- a/assignment-client/src/Agent.cpp +++ b/assignment-client/src/Agent.cpp @@ -546,7 +546,7 @@ void Agent::processAgentAvatar() { auto scriptedAvatar = DependencyManager::get(); AvatarData::AvatarDataDetail dataDetail = (randFloat() < AVATAR_SEND_FULL_UPDATE_RATIO) ? AvatarData::SendAllData : AvatarData::CullSmallData; - QByteArray avatarByteArray = scriptedAvatar->toByteArray(dataDetail, 0, scriptedAvatar->getLastSentJointData()); + QByteArray avatarByteArray = scriptedAvatar->toByteArray(dataDetail); scriptedAvatar->doneEncoding(true); static AvatarDataSequenceNumber sequenceNumber = 0; diff --git a/assignment-client/src/avatars/AvatarMixer.cpp b/assignment-client/src/avatars/AvatarMixer.cpp index db158c861c..51f6067e6a 100644 --- a/assignment-client/src/avatars/AvatarMixer.cpp +++ b/assignment-client/src/avatars/AvatarMixer.cpp @@ -125,12 +125,18 @@ void AvatarMixer::start() { // // DONE --- 1) only sleep for remainder // DONE --- 2) clean up stats, add slave stats - // 3a) out of view??? is it broken? - // 3) Error in PacketList::writeData - attempted to write a segment to an unordered packet that is larger than the payload size. - // 4) fix two different versions of toByteArray() - // 5) throttling?? - // 6) audit the locking and side-effects to node, otherNode, and nodeData - // 7) delete dead code from mixer (now that it's in slave) + // DONE --- 3) out of view??? is it broken? - verified - it's working + // 4) Error in PacketList::writeData - attempted to write a segment to an unordered packet that is larger than the payload size. + // DONE --- 4a) hack to not send face data mostly seems to work... + // 4b) some kind of a better approach to handling otherAvatar.toByteArray() for content that is larger than MTU + // 5) fix two different versions of toByteArray() + // 6) throttling?? + // 7) audit the locking and side-effects to node, otherNode, and nodeData + // 8) delete dead code from mixer (now that it's in slave) + // 9) better stats in the nodes: + // how many avatars are actually "in view" for the avtar in question (even if they are over bandwidth budget) + // + // ////////////////////////////////////////////////////////////////////////////////////////////////////////////////// ////////////////////////////////////////////////////////////////////////////////////////////////////////////////// diff --git a/assignment-client/src/avatars/AvatarMixerClientData.cpp b/assignment-client/src/avatars/AvatarMixerClientData.cpp index b5d4e390bb..6fdd2fd23e 100644 --- a/assignment-client/src/avatars/AvatarMixerClientData.cpp +++ b/assignment-client/src/avatars/AvatarMixerClientData.cpp @@ -58,6 +58,7 @@ int AvatarMixerClientData::parseData(ReceivedMessage& message) { return _avatar->parseDataFromBuffer(message.readWithoutCopy(message.getBytesLeftToRead())); } +// FIXME -- this needs a mutex in new model. bool AvatarMixerClientData::checkAndSetHasReceivedFirstPacketsFrom(const QUuid& uuid) { if (_hasReceivedFirstPacketsFrom.find(uuid) == _hasReceivedFirstPacketsFrom.end()) { _hasReceivedFirstPacketsFrom.insert(uuid); diff --git a/assignment-client/src/avatars/AvatarMixerClientData.h b/assignment-client/src/avatars/AvatarMixerClientData.h index 8cd72050f7..87f7bb9cc9 100644 --- a/assignment-client/src/avatars/AvatarMixerClientData.h +++ b/assignment-client/src/avatars/AvatarMixerClientData.h @@ -42,6 +42,7 @@ public: int parseData(ReceivedMessage& message) override; AvatarData& getAvatar() { return *_avatar; } + const AvatarData* getConstAvatarData() const { return _avatar.get(); } bool checkAndSetHasReceivedFirstPacketsFrom(const QUuid& uuid); diff --git a/assignment-client/src/avatars/AvatarMixerSlave.cpp b/assignment-client/src/avatars/AvatarMixerSlave.cpp index ca2db55cdd..2a7f9a4d8d 100644 --- a/assignment-client/src/avatars/AvatarMixerSlave.cpp +++ b/assignment-client/src/avatars/AvatarMixerSlave.cpp @@ -245,17 +245,12 @@ void AvatarMixerSlave::broadcastAvatarData(const SharedNodePointer& node) { ++numOtherAvatars; - AvatarMixerClientData* otherNodeData = reinterpret_cast(otherNode->getLinkedData()); - //MutexTryLocker lock(otherNodeData->getMutex()); - - // FIXME -- might want to track this lock failed... - //if (!lock.isLocked()) { - //qDebug() << __FUNCTION__ << "inner loop, node:" << node << "otherNode:" << otherNode << " failed to lock... would BAIL??... line:" << __LINE__; - //return; - //} + const AvatarMixerClientData* otherNodeData = reinterpret_cast(otherNode->getLinkedData()); // make sure we send out identity packets to and from new arrivals. - bool forceSend = !otherNodeData->checkAndSetHasReceivedFirstPacketsFrom(node->getUUID()); + // FIXME this is where our crash was on friday!! + // this is getting called on multiple threads... needs mutex of better solution + bool forceSend = !nodeData->checkAndSetHasReceivedFirstPacketsFrom(otherNode->getUUID()); if (otherNodeData->getIdentityChangeTimestamp().time_since_epoch().count() > 0 && (forceSend @@ -265,7 +260,7 @@ void AvatarMixerSlave::broadcastAvatarData(const SharedNodePointer& node) { // FIXME --- used to be.../ mixer data dependency //sendIdentityPacket(otherNodeData, node); - QByteArray individualData = otherNodeData->getAvatar().identityByteArray(); + QByteArray individualData = otherNodeData->getConstAvatarData()->identityByteArray(); auto identityPacket = NLPacket::create(PacketType::AvatarIdentity, individualData.size()); individualData.replace(0, NUM_BYTES_RFC4122_UUID, otherNodeData->getNodeID().toRfc4122()); identityPacket->write(individualData); @@ -273,12 +268,12 @@ void AvatarMixerSlave::broadcastAvatarData(const SharedNodePointer& node) { //qDebug() << __FUNCTION__ << "inner loop, node:" << node << "otherNode:" << otherNode << " sending itentity packet for otherNode to node..."; } - const AvatarData& otherAvatar = otherNodeData->getAvatar(); + const AvatarData* otherAvatar = otherNodeData->getConstAvatarData(); // Decide whether to send this avatar's data based on it's distance from us // The full rate distance is the distance at which EVERY update will be sent for this avatar // at twice the full rate distance, there will be a 50% chance of sending this avatar's update - glm::vec3 otherPosition = otherAvatar.getClientGlobalPosition(); + glm::vec3 otherPosition = otherAvatar->getClientGlobalPosition(); float distanceToAvatar = glm::length(myPosition - otherPosition); // potentially update the max full rate distance for this frame @@ -299,10 +294,13 @@ void AvatarMixerSlave::broadcastAvatarData(const SharedNodePointer& node) { AvatarDataSequenceNumber lastSeqToReceiver = nodeData->getLastBroadcastSequenceNumber(otherNode->getUUID()); AvatarDataSequenceNumber lastSeqFromSender = otherNodeData->getLastReceivedSequenceNumber(); + /*** + // FIXME this does not belong here... it should be in the packet processing if (lastSeqToReceiver > lastSeqFromSender && lastSeqToReceiver != UINT16_MAX) { // we got out out of order packets from the sender, track it otherNodeData->incrementNumOutOfOrderSends(); } + ***/ // make sure we haven't already sent this data from this sender to this receiver // or that somehow we haven't sent @@ -365,7 +363,7 @@ void AvatarMixerSlave::broadcastAvatarData(const SharedNodePointer& node) { bool dropFaceTracking = true; // this is a hack for now... always drop face tracking quint64 start = usecTimestampNow(); - QByteArray bytes = otherAvatar.toByteArray(detail, lastEncodeForOther, lastSentJointsForOther, + QByteArray bytes = otherAvatar->toByteArray(detail, lastEncodeForOther, lastSentJointsForOther, hasFlagsOut, dropFaceTracking, distanceAdjust, viewerPosition, &lastSentJointsForOther); quint64 end = usecTimestampNow(); _stats.toByteArrayElapsedTime += (end - start); diff --git a/assignment-client/src/avatars/ScriptableAvatar.cpp b/assignment-client/src/avatars/ScriptableAvatar.cpp index 95bcbb587e..cdda070da2 100644 --- a/assignment-client/src/avatars/ScriptableAvatar.cpp +++ b/assignment-client/src/avatars/ScriptableAvatar.cpp @@ -14,10 +14,9 @@ #include #include "ScriptableAvatar.h" -QByteArray ScriptableAvatar::toByteArray(AvatarDataDetail dataDetail, quint64 lastSentTime, const QVector& lastSentJointData, - bool distanceAdjust, glm::vec3 viewerPosition, QVector* sentJointDataOut) { +QByteArray ScriptableAvatar::toByteArray(AvatarDataDetail dataDetail) { _globalPosition = getPosition(); - return AvatarData::toByteArray(dataDetail, lastSentTime, lastSentJointData, distanceAdjust, viewerPosition, sentJointDataOut); + return AvatarData::toByteArray(dataDetail); } diff --git a/assignment-client/src/avatars/ScriptableAvatar.h b/assignment-client/src/avatars/ScriptableAvatar.h index be7a90adf9..b6804da43d 100644 --- a/assignment-client/src/avatars/ScriptableAvatar.h +++ b/assignment-client/src/avatars/ScriptableAvatar.h @@ -28,8 +28,7 @@ public: Q_INVOKABLE AnimationDetails getAnimationDetails(); virtual void setSkeletonModelURL(const QUrl& skeletonModelURL) override; - virtual QByteArray toByteArray(AvatarDataDetail dataDetail, quint64 lastSentTime, const QVector& lastSentJointData, - bool distanceAdjust = false, glm::vec3 viewerPosition = glm::vec3(0), QVector* sentJointDataOut = nullptr) override; + virtual QByteArray toByteArray(AvatarDataDetail dataDetail) override; private slots: diff --git a/interface/src/avatar/MyAvatar.cpp b/interface/src/avatar/MyAvatar.cpp index 29f41c89fd..21b75e4e71 100644 --- a/interface/src/avatar/MyAvatar.cpp +++ b/interface/src/avatar/MyAvatar.cpp @@ -227,8 +227,7 @@ void MyAvatar::simulateAttachments(float deltaTime) { // don't update attachments here, do it in harvestResultsFromPhysicsSimulation() } -QByteArray MyAvatar::toByteArray(AvatarDataDetail dataDetail, quint64 lastSentTime, const QVector& lastSentJointData, - bool distanceAdjust, glm::vec3 viewerPosition, QVector* sentJointDataOut) { +QByteArray MyAvatar::toByteArray(AvatarDataDetail dataDetail) { CameraMode mode = qApp->getCamera()->getMode(); _globalPosition = getPosition(); _globalBoundingBoxDimensions.x = _characterController.getCapsuleRadius(); @@ -239,12 +238,12 @@ QByteArray MyAvatar::toByteArray(AvatarDataDetail dataDetail, quint64 lastSentTi // fake the avatar position that is sent up to the AvatarMixer glm::vec3 oldPosition = getPosition(); setPosition(getSkeletonPosition()); - QByteArray array = AvatarData::toByteArray(dataDetail, lastSentTime, lastSentJointData, distanceAdjust, viewerPosition, sentJointDataOut); + QByteArray array = AvatarData::toByteArray(dataDetail); // copy the correct position back setPosition(oldPosition); return array; } - return AvatarData::toByteArray(dataDetail, lastSentTime, lastSentJointData, distanceAdjust, viewerPosition, sentJointDataOut); + return AvatarData::toByteArray(dataDetail); } void MyAvatar::centerBody() { diff --git a/interface/src/avatar/MyAvatar.h b/interface/src/avatar/MyAvatar.h index c4fe86356d..982f0505ff 100644 --- a/interface/src/avatar/MyAvatar.h +++ b/interface/src/avatar/MyAvatar.h @@ -338,8 +338,7 @@ private: glm::quat getWorldBodyOrientation() const; - virtual QByteArray toByteArray(AvatarDataDetail dataDetail, quint64 lastSentTime, const QVector& lastSentJointData, - bool distanceAdjust = false, glm::vec3 viewerPosition = glm::vec3(0), QVector* sentJointDataOut = nullptr) override; + virtual QByteArray toByteArray(AvatarDataDetail dataDetail) override; void simulate(float deltaTime); void updateFromTrackers(float deltaTime); diff --git a/libraries/avatars/src/AvatarData.cpp b/libraries/avatars/src/AvatarData.cpp index 66d1fb39bf..f48b31881a 100644 --- a/libraries/avatars/src/AvatarData.cpp +++ b/libraries/avatars/src/AvatarData.cpp @@ -168,414 +168,19 @@ float AvatarData::getDistanceBasedMinTranslationDistance(glm::vec3 viewerPositio } -QByteArray AvatarData::toByteArray(AvatarDataDetail dataDetail, quint64 lastSentTime, const QVector& lastSentJointData, - bool distanceAdjust, glm::vec3 viewerPosition, QVector* sentJointDataOut) { - - // if no timestamp was included, then assume the avatarData is single instance - // and is tracking its own last encoding time. - if (lastSentTime == 0) { - lastSentTime = _lastToByteArray; - _lastToByteArray = usecTimestampNow(); - } - - bool cullSmallChanges = (dataDetail == CullSmallData); - bool sendAll = (dataDetail == SendAllData); - bool sendMinimum = (dataDetail == MinimumData); - - lazyInitHeadData(); - - QByteArray avatarDataByteArray(udt::MAX_PACKET_SIZE, 0); - unsigned char* destinationBuffer = reinterpret_cast(avatarDataByteArray.data()); - unsigned char* startPosition = destinationBuffer; - - // FIXME - - // - // BUG -- if you enter a space bubble, and then back away, the avatar has wrong orientation until "send all" happens... - // this is an iFrame issue... what to do about that? - // - // BUG -- Resizing avatar seems to "take too long"... the avatar doesn't redraw at smaller size right away - // - // TODO consider these additional optimizations in the future - // 1) SensorToWorld - should we only send this for avatars with attachments?? - 20 bytes - 7.20 kbps - // 2) GUIID for the session change to 2byte index (savings) - 14 bytes - 5.04 kbps - // 3) Improve Joints -- currently we use rotational tolerances, but if we had skeleton/bone length data - // we could do a better job of determining if the change in joints actually translates to visible - // changes at distance. - // - // Potential savings: - // 63 rotations * 6 bytes = 136kbps - // 3 translations * 6 bytes = 6.48kbps - // - - auto parentID = getParentID(); - - bool hasAvatarGlobalPosition = true; // always include global position - bool hasAvatarOrientation = sendAll || rotationChangedSince(lastSentTime); - bool hasAvatarBoundingBox = sendAll || avatarBoundingBoxChangedSince(lastSentTime); - bool hasAvatarScale = sendAll || avatarScaleChangedSince(lastSentTime); - bool hasLookAtPosition = sendAll || lookAtPositionChangedSince(lastSentTime); - bool hasAudioLoudness = sendAll || audioLoudnessChangedSince(lastSentTime); - bool hasSensorToWorldMatrix = sendAll || sensorToWorldMatrixChangedSince(lastSentTime); - bool hasAdditionalFlags = sendAll || additionalFlagsChangedSince(lastSentTime); - - // local position, and parent info only apply to avatars that are parented. The local position - // and the parent info can change independently though, so we track their "changed since" - // separately - bool hasParentInfo = sendAll || parentInfoChangedSince(lastSentTime); - bool hasAvatarLocalPosition = hasParent() && (sendAll || - tranlationChangedSince(lastSentTime) || - parentInfoChangedSince(lastSentTime)); - - bool hasFaceTrackerInfo = hasFaceTracker() && (sendAll || faceTrackerInfoChangedSince(lastSentTime)); - bool hasJointData = sendAll || !sendMinimum; - - // Leading flags, to indicate how much data is actually included in the packet... - AvatarDataPacket::HasFlags packetStateFlags = - (hasAvatarGlobalPosition ? AvatarDataPacket::PACKET_HAS_AVATAR_GLOBAL_POSITION : 0) - | (hasAvatarBoundingBox ? AvatarDataPacket::PACKET_HAS_AVATAR_BOUNDING_BOX : 0) - | (hasAvatarOrientation ? AvatarDataPacket::PACKET_HAS_AVATAR_ORIENTATION : 0) - | (hasAvatarScale ? AvatarDataPacket::PACKET_HAS_AVATAR_SCALE : 0) - | (hasLookAtPosition ? AvatarDataPacket::PACKET_HAS_LOOK_AT_POSITION : 0) - | (hasAudioLoudness ? AvatarDataPacket::PACKET_HAS_AUDIO_LOUDNESS : 0) - | (hasSensorToWorldMatrix ? AvatarDataPacket::PACKET_HAS_SENSOR_TO_WORLD_MATRIX : 0) - | (hasAdditionalFlags ? AvatarDataPacket::PACKET_HAS_ADDITIONAL_FLAGS : 0) - | (hasParentInfo ? AvatarDataPacket::PACKET_HAS_PARENT_INFO : 0) - | (hasAvatarLocalPosition ? AvatarDataPacket::PACKET_HAS_AVATAR_LOCAL_POSITION : 0) - | (hasFaceTrackerInfo ? AvatarDataPacket::PACKET_HAS_FACE_TRACKER_INFO : 0) - | (hasJointData ? AvatarDataPacket::PACKET_HAS_JOINT_DATA : 0); - - memcpy(destinationBuffer, &packetStateFlags, sizeof(packetStateFlags)); - destinationBuffer += sizeof(packetStateFlags); - - if (hasAvatarGlobalPosition) { - auto startSection = destinationBuffer; - auto data = reinterpret_cast(destinationBuffer); - data->globalPosition[0] = _globalPosition.x; - data->globalPosition[1] = _globalPosition.y; - data->globalPosition[2] = _globalPosition.z; - destinationBuffer += sizeof(AvatarDataPacket::AvatarGlobalPosition); - - int numBytes = destinationBuffer - startSection; - - _globalPositionRateOutbound.increment(numBytes); - } - - if (hasAvatarBoundingBox) { - auto startSection = destinationBuffer; - auto data = reinterpret_cast(destinationBuffer); - - data->avatarDimensions[0] = _globalBoundingBoxDimensions.x; - data->avatarDimensions[1] = _globalBoundingBoxDimensions.y; - data->avatarDimensions[2] = _globalBoundingBoxDimensions.z; - - data->boundOriginOffset[0] = _globalBoundingBoxOffset.x; - data->boundOriginOffset[1] = _globalBoundingBoxOffset.y; - data->boundOriginOffset[2] = _globalBoundingBoxOffset.z; - - destinationBuffer += sizeof(AvatarDataPacket::AvatarBoundingBox); - - int numBytes = destinationBuffer - startSection; - _avatarBoundingBoxRateOutbound.increment(numBytes); - } - - if (hasAvatarOrientation) { - auto startSection = destinationBuffer; - auto localOrientation = getLocalOrientation(); - destinationBuffer += packOrientationQuatToSixBytes(destinationBuffer, localOrientation); - - int numBytes = destinationBuffer - startSection; - _avatarOrientationRateOutbound.increment(numBytes); - } - - if (hasAvatarScale) { - auto startSection = destinationBuffer; - auto data = reinterpret_cast(destinationBuffer); - auto scale = getDomainLimitedScale(); - packFloatRatioToTwoByte((uint8_t*)(&data->scale), scale); - destinationBuffer += sizeof(AvatarDataPacket::AvatarScale); - - int numBytes = destinationBuffer - startSection; - _avatarScaleRateOutbound.increment(numBytes); - } - - if (hasLookAtPosition) { - auto startSection = destinationBuffer; - auto data = reinterpret_cast(destinationBuffer); - auto lookAt = _headData->getLookAtPosition(); - data->lookAtPosition[0] = lookAt.x; - data->lookAtPosition[1] = lookAt.y; - data->lookAtPosition[2] = lookAt.z; - destinationBuffer += sizeof(AvatarDataPacket::LookAtPosition); - - int numBytes = destinationBuffer - startSection; - _lookAtPositionRateOutbound.increment(numBytes); - } - - if (hasAudioLoudness) { - auto startSection = destinationBuffer; - auto data = reinterpret_cast(destinationBuffer); - data->audioLoudness = packFloatGainToByte(_headData->getAudioLoudness() / AUDIO_LOUDNESS_SCALE); - destinationBuffer += sizeof(AvatarDataPacket::AudioLoudness); - - int numBytes = destinationBuffer - startSection; - _audioLoudnessRateOutbound.increment(numBytes); - } - - if (hasSensorToWorldMatrix) { - auto startSection = destinationBuffer; - auto data = reinterpret_cast(destinationBuffer); - glm::mat4 sensorToWorldMatrix = getSensorToWorldMatrix(); - packOrientationQuatToSixBytes(data->sensorToWorldQuat, glmExtractRotation(sensorToWorldMatrix)); - glm::vec3 scale = extractScale(sensorToWorldMatrix); - packFloatScalarToSignedTwoByteFixed((uint8_t*)&data->sensorToWorldScale, scale.x, SENSOR_TO_WORLD_SCALE_RADIX); - data->sensorToWorldTrans[0] = sensorToWorldMatrix[3][0]; - data->sensorToWorldTrans[1] = sensorToWorldMatrix[3][1]; - data->sensorToWorldTrans[2] = sensorToWorldMatrix[3][2]; - destinationBuffer += sizeof(AvatarDataPacket::SensorToWorldMatrix); - - int numBytes = destinationBuffer - startSection; - _sensorToWorldRateOutbound.increment(numBytes); - } - - if (hasAdditionalFlags) { - auto startSection = destinationBuffer; - auto data = reinterpret_cast(destinationBuffer); - - uint8_t flags { 0 }; - - setSemiNibbleAt(flags, KEY_STATE_START_BIT, _keyState); - - // hand state - bool isFingerPointing = _handState & IS_FINGER_POINTING_FLAG; - setSemiNibbleAt(flags, HAND_STATE_START_BIT, _handState & ~IS_FINGER_POINTING_FLAG); - if (isFingerPointing) { - setAtBit(flags, HAND_STATE_FINGER_POINTING_BIT); - } - // faceshift state - if (_headData->_isFaceTrackerConnected) { - setAtBit(flags, IS_FACESHIFT_CONNECTED); - } - // eye tracker state - if (_headData->_isEyeTrackerConnected) { - setAtBit(flags, IS_EYE_TRACKER_CONNECTED); - } - // referential state - if (!parentID.isNull()) { - setAtBit(flags, HAS_REFERENTIAL); - } - data->flags = flags; - destinationBuffer += sizeof(AvatarDataPacket::AdditionalFlags); - - int numBytes = destinationBuffer - startSection; - _additionalFlagsRateOutbound.increment(numBytes); - } - - if (hasParentInfo) { - auto startSection = destinationBuffer; - auto parentInfo = reinterpret_cast(destinationBuffer); - QByteArray referentialAsBytes = parentID.toRfc4122(); - memcpy(parentInfo->parentUUID, referentialAsBytes.data(), referentialAsBytes.size()); - parentInfo->parentJointIndex = getParentJointIndex(); - destinationBuffer += sizeof(AvatarDataPacket::ParentInfo); - - int numBytes = destinationBuffer - startSection; - _parentInfoRateOutbound.increment(numBytes); - } - - if (hasAvatarLocalPosition) { - auto startSection = destinationBuffer; - auto data = reinterpret_cast(destinationBuffer); - auto localPosition = getLocalPosition(); - data->localPosition[0] = localPosition.x; - data->localPosition[1] = localPosition.y; - data->localPosition[2] = localPosition.z; - destinationBuffer += sizeof(AvatarDataPacket::AvatarLocalPosition); - - int numBytes = destinationBuffer - startSection; - _localPositionRateOutbound.increment(numBytes); - } - - // If it is connected, pack up the data - if (hasFaceTrackerInfo) { - auto startSection = destinationBuffer; - auto faceTrackerInfo = reinterpret_cast(destinationBuffer); - - faceTrackerInfo->leftEyeBlink = _headData->_leftEyeBlink; - faceTrackerInfo->rightEyeBlink = _headData->_rightEyeBlink; - faceTrackerInfo->averageLoudness = _headData->_averageLoudness; - faceTrackerInfo->browAudioLift = _headData->_browAudioLift; - faceTrackerInfo->numBlendshapeCoefficients = _headData->_blendshapeCoefficients.size(); - destinationBuffer += sizeof(AvatarDataPacket::FaceTrackerInfo); - - // followed by a variable number of float coefficients - memcpy(destinationBuffer, _headData->_blendshapeCoefficients.data(), _headData->_blendshapeCoefficients.size() * sizeof(float)); - destinationBuffer += _headData->_blendshapeCoefficients.size() * sizeof(float); - - int numBytes = destinationBuffer - startSection; - _faceTrackerRateOutbound.increment(numBytes); - } - - // If it is connected, pack up the data - if (hasJointData) { - auto startSection = destinationBuffer; - QReadLocker readLock(&_jointDataLock); - - // joint rotation data - int numJoints = _jointData.size(); - *destinationBuffer++ = (uint8_t)numJoints; - - unsigned char* validityPosition = destinationBuffer; - unsigned char validity = 0; - int validityBit = 0; - -#ifdef WANT_DEBUG - int rotationSentCount = 0; - unsigned char* beforeRotations = destinationBuffer; -#endif - - if (sentJointDataOut) { - sentJointDataOut->resize(_jointData.size()); // Make sure the destination is resized before using it - } - float minRotationDOT = !distanceAdjust ? AVATAR_MIN_ROTATION_DOT : getDistanceBasedMinRotationDOT(viewerPosition); - - for (int i = 0; i < _jointData.size(); i++) { - const JointData& data = _jointData[i]; - - // The dot product for smaller rotations is a smaller number. - // So if the dot() is less than the value, then the rotation is a larger angle of rotation - bool largeEnoughRotation = fabsf(glm::dot(data.rotation, lastSentJointData[i].rotation)) < minRotationDOT; - - if (sendAll || lastSentJointData[i].rotation != data.rotation) { - if (sendAll || !cullSmallChanges || largeEnoughRotation) { - if (data.rotationSet) { - validity |= (1 << validityBit); -#ifdef WANT_DEBUG - rotationSentCount++; -#endif - if (sentJointDataOut) { - auto jointDataOut = *sentJointDataOut; - jointDataOut[i].rotation = data.rotation; - } - - } - } - } - if (++validityBit == BITS_IN_BYTE) { - *destinationBuffer++ = validity; - validityBit = validity = 0; - } - } - if (validityBit != 0) { - *destinationBuffer++ = validity; - } - - validityBit = 0; - validity = *validityPosition++; - for (int i = 0; i < _jointData.size(); i++) { - const JointData& data = _jointData[i]; - if (validity & (1 << validityBit)) { - destinationBuffer += packOrientationQuatToSixBytes(destinationBuffer, data.rotation); - } - if (++validityBit == BITS_IN_BYTE) { - validityBit = 0; - validity = *validityPosition++; - } - } - - - // joint translation data - validityPosition = destinationBuffer; - validity = 0; - validityBit = 0; - -#ifdef WANT_DEBUG - int translationSentCount = 0; - unsigned char* beforeTranslations = destinationBuffer; -#endif - - float minTranslation = !distanceAdjust ? AVATAR_MIN_TRANSLATION : getDistanceBasedMinTranslationDistance(viewerPosition); - - float maxTranslationDimension = 0.0; - for (int i = 0; i < _jointData.size(); i++) { - const JointData& data = _jointData[i]; - if (sendAll || lastSentJointData[i].translation != data.translation) { - if (sendAll || - !cullSmallChanges || - glm::distance(data.translation, lastSentJointData[i].translation) > minTranslation) { - if (data.translationSet) { - validity |= (1 << validityBit); - #ifdef WANT_DEBUG - translationSentCount++; - #endif - maxTranslationDimension = glm::max(fabsf(data.translation.x), maxTranslationDimension); - maxTranslationDimension = glm::max(fabsf(data.translation.y), maxTranslationDimension); - maxTranslationDimension = glm::max(fabsf(data.translation.z), maxTranslationDimension); - - if (sentJointDataOut) { - auto jointDataOut = *sentJointDataOut; - jointDataOut[i].translation = data.translation; - } - - } - } - } - if (++validityBit == BITS_IN_BYTE) { - *destinationBuffer++ = validity; - validityBit = validity = 0; - } - } - - if (validityBit != 0) { - *destinationBuffer++ = validity; - } - - validityBit = 0; - validity = *validityPosition++; - for (int i = 0; i < _jointData.size(); i++) { - const JointData& data = _jointData[i]; - if (validity & (1 << validityBit)) { - destinationBuffer += - packFloatVec3ToSignedTwoByteFixed(destinationBuffer, data.translation, TRANSLATION_COMPRESSION_RADIX); - } - if (++validityBit == BITS_IN_BYTE) { - validityBit = 0; - validity = *validityPosition++; - } - } - - // faux joints - Transform controllerLeftHandTransform = Transform(getControllerLeftHandMatrix()); - destinationBuffer += packOrientationQuatToSixBytes(destinationBuffer, controllerLeftHandTransform.getRotation()); - destinationBuffer += packFloatVec3ToSignedTwoByteFixed(destinationBuffer, controllerLeftHandTransform.getTranslation(), - TRANSLATION_COMPRESSION_RADIX); - Transform controllerRightHandTransform = Transform(getControllerRightHandMatrix()); - destinationBuffer += packOrientationQuatToSixBytes(destinationBuffer, controllerRightHandTransform.getRotation()); - destinationBuffer += packFloatVec3ToSignedTwoByteFixed(destinationBuffer, controllerRightHandTransform.getTranslation(), - TRANSLATION_COMPRESSION_RADIX); - -#ifdef WANT_DEBUG - if (sendAll) { - qCDebug(avatars) << "AvatarData::toByteArray" << cullSmallChanges << sendAll - << "rotations:" << rotationSentCount << "translations:" << translationSentCount - << "largest:" << maxTranslationDimension - << "size:" - << (beforeRotations - startPosition) << "+" - << (beforeTranslations - beforeRotations) << "+" - << (destinationBuffer - beforeTranslations) << "=" - << (destinationBuffer - startPosition); - } -#endif - - int numBytes = destinationBuffer - startSection; - _jointDataRateOutbound.increment(numBytes); - } - - int avatarDataSize = destinationBuffer - startPosition; - return avatarDataByteArray.left(avatarDataSize); +// we want to track outbound data in this case... +QByteArray AvatarData::toByteArray(AvatarDataDetail dataDetail) { + AvatarDataPacket::HasFlags hasFlagsOut; + auto lastSentTime = _lastToByteArray; + _lastToByteArray = usecTimestampNow(); + return AvatarData::toByteArray(dataDetail, lastSentTime, getLastSentJointData(), + hasFlagsOut, false, false, glm::vec3(0), nullptr, + &_outboundDataRate); } QByteArray AvatarData::toByteArray(AvatarDataDetail dataDetail, quint64 lastSentTime, const QVector& lastSentJointData, - AvatarDataPacket::HasFlags& hasFlagsOut, bool dropFaceTracking, bool distanceAdjust, glm::vec3 viewerPosition, QVector* sentJointDataOut) const { + AvatarDataPacket::HasFlags& hasFlagsOut, bool dropFaceTracking, bool distanceAdjust, + glm::vec3 viewerPosition, QVector* sentJointDataOut, AvatarDataRate* outboundDataRateOut) const { bool cullSmallChanges = (dataDetail == CullSmallData); bool sendAll = (dataDetail == SendAllData); @@ -656,7 +261,9 @@ QByteArray AvatarData::toByteArray(AvatarDataDetail dataDetail, quint64 lastSent int numBytes = destinationBuffer - startSection; - //_globalPositionRateOutbound.increment(numBytes); + if (outboundDataRateOut) { + outboundDataRateOut->globalPositionRate.increment(numBytes); + } } if (hasAvatarBoundingBox) { @@ -674,7 +281,9 @@ QByteArray AvatarData::toByteArray(AvatarDataDetail dataDetail, quint64 lastSent destinationBuffer += sizeof(AvatarDataPacket::AvatarBoundingBox); int numBytes = destinationBuffer - startSection; - //_avatarBoundingBoxRateOutbound.increment(numBytes); + if (outboundDataRateOut) { + outboundDataRateOut->avatarBoundingBoxRate.increment(numBytes); + } } if (hasAvatarOrientation) { @@ -683,7 +292,9 @@ QByteArray AvatarData::toByteArray(AvatarDataDetail dataDetail, quint64 lastSent destinationBuffer += packOrientationQuatToSixBytes(destinationBuffer, localOrientation); int numBytes = destinationBuffer - startSection; - //_avatarOrientationRateOutbound.increment(numBytes); + if (outboundDataRateOut) { + outboundDataRateOut->avatarOrientationRate.increment(numBytes); + } } if (hasAvatarScale) { @@ -694,7 +305,9 @@ QByteArray AvatarData::toByteArray(AvatarDataDetail dataDetail, quint64 lastSent destinationBuffer += sizeof(AvatarDataPacket::AvatarScale); int numBytes = destinationBuffer - startSection; - //_avatarScaleRateOutbound.increment(numBytes); + if (outboundDataRateOut) { + outboundDataRateOut->avatarScaleRate.increment(numBytes); + } } if (hasLookAtPosition) { @@ -707,7 +320,9 @@ QByteArray AvatarData::toByteArray(AvatarDataDetail dataDetail, quint64 lastSent destinationBuffer += sizeof(AvatarDataPacket::LookAtPosition); int numBytes = destinationBuffer - startSection; - //_lookAtPositionRateOutbound.increment(numBytes); + if (outboundDataRateOut) { + outboundDataRateOut->lookAtPositionRate.increment(numBytes); + } } if (hasAudioLoudness) { @@ -717,7 +332,9 @@ QByteArray AvatarData::toByteArray(AvatarDataDetail dataDetail, quint64 lastSent destinationBuffer += sizeof(AvatarDataPacket::AudioLoudness); int numBytes = destinationBuffer - startSection; - //_audioLoudnessRateOutbound.increment(numBytes); + if (outboundDataRateOut) { + outboundDataRateOut->audioLoudnessRate.increment(numBytes); + } } if (hasSensorToWorldMatrix) { @@ -733,7 +350,9 @@ QByteArray AvatarData::toByteArray(AvatarDataDetail dataDetail, quint64 lastSent destinationBuffer += sizeof(AvatarDataPacket::SensorToWorldMatrix); int numBytes = destinationBuffer - startSection; - //_sensorToWorldRateOutbound.increment(numBytes); + if (outboundDataRateOut) { + outboundDataRateOut->sensorToWorldRate.increment(numBytes); + } } if (hasAdditionalFlags) { @@ -766,7 +385,9 @@ QByteArray AvatarData::toByteArray(AvatarDataDetail dataDetail, quint64 lastSent destinationBuffer += sizeof(AvatarDataPacket::AdditionalFlags); int numBytes = destinationBuffer - startSection; - //_additionalFlagsRateOutbound.increment(numBytes); + if (outboundDataRateOut) { + outboundDataRateOut->additionalFlagsRate.increment(numBytes); + } } if (hasParentInfo) { @@ -778,7 +399,9 @@ QByteArray AvatarData::toByteArray(AvatarDataDetail dataDetail, quint64 lastSent destinationBuffer += sizeof(AvatarDataPacket::ParentInfo); int numBytes = destinationBuffer - startSection; - //_parentInfoRateOutbound.increment(numBytes); + if (outboundDataRateOut) { + outboundDataRateOut->parentInfoRate.increment(numBytes); + } } if (hasAvatarLocalPosition) { @@ -791,7 +414,9 @@ QByteArray AvatarData::toByteArray(AvatarDataDetail dataDetail, quint64 lastSent destinationBuffer += sizeof(AvatarDataPacket::AvatarLocalPosition); int numBytes = destinationBuffer - startSection; - //_localPositionRateOutbound.increment(numBytes); + if (outboundDataRateOut) { + outboundDataRateOut->localPositionRate.increment(numBytes); + } } // If it is connected, pack up the data @@ -811,7 +436,9 @@ QByteArray AvatarData::toByteArray(AvatarDataDetail dataDetail, quint64 lastSent destinationBuffer += _headData->_blendshapeCoefficients.size() * sizeof(float); int numBytes = destinationBuffer - startSection; - //_faceTrackerRateOutbound.increment(numBytes); + if (outboundDataRateOut) { + outboundDataRateOut->faceTrackerRate.increment(numBytes); + } } // If it is connected, pack up the data @@ -966,7 +593,9 @@ QByteArray AvatarData::toByteArray(AvatarDataDetail dataDetail, quint64 lastSent #endif int numBytes = destinationBuffer - startSection; - //_jointDataRateOutbound.increment(numBytes); + if (outboundDataRateOut) { + outboundDataRateOut->jointDataRate.increment(numBytes); + } } int avatarDataSize = destinationBuffer - startPosition; @@ -1451,29 +1080,29 @@ float AvatarData::getDataRate(const QString& rateName) const { } else if (rateName == "jointData") { return _jointDataRate.rate() / BYTES_PER_KILOBIT; } else if (rateName == "globalPositionOutbound") { - return _globalPositionRateOutbound.rate() / BYTES_PER_KILOBIT; + return _outboundDataRate.globalPositionRate.rate() / BYTES_PER_KILOBIT; } else if (rateName == "localPositionOutbound") { - return _localPositionRateOutbound.rate() / BYTES_PER_KILOBIT; + return _outboundDataRate.localPositionRate.rate() / BYTES_PER_KILOBIT; } else if (rateName == "avatarBoundingBoxOutbound") { - return _avatarBoundingBoxRateOutbound.rate() / BYTES_PER_KILOBIT; + return _outboundDataRate.avatarBoundingBoxRate.rate() / BYTES_PER_KILOBIT; } else if (rateName == "avatarOrientationOutbound") { - return _avatarOrientationRateOutbound.rate() / BYTES_PER_KILOBIT; + return _outboundDataRate.avatarOrientationRate.rate() / BYTES_PER_KILOBIT; } else if (rateName == "avatarScaleOutbound") { - return _avatarScaleRateOutbound.rate() / BYTES_PER_KILOBIT; + return _outboundDataRate.avatarScaleRate.rate() / BYTES_PER_KILOBIT; } else if (rateName == "lookAtPositionOutbound") { - return _lookAtPositionRateOutbound.rate() / BYTES_PER_KILOBIT; + return _outboundDataRate.lookAtPositionRate.rate() / BYTES_PER_KILOBIT; } else if (rateName == "audioLoudnessOutbound") { - return _audioLoudnessRateOutbound.rate() / BYTES_PER_KILOBIT; + return _outboundDataRate.audioLoudnessRate.rate() / BYTES_PER_KILOBIT; } else if (rateName == "sensorToWorkMatrixOutbound") { - return _sensorToWorldRateOutbound.rate() / BYTES_PER_KILOBIT; + return _outboundDataRate.sensorToWorldRate.rate() / BYTES_PER_KILOBIT; } else if (rateName == "additionalFlagsOutbound") { - return _additionalFlagsRateOutbound.rate() / BYTES_PER_KILOBIT; + return _outboundDataRate.additionalFlagsRate.rate() / BYTES_PER_KILOBIT; } else if (rateName == "parentInfoOutbound") { - return _parentInfoRateOutbound.rate() / BYTES_PER_KILOBIT; + return _outboundDataRate.parentInfoRate.rate() / BYTES_PER_KILOBIT; } else if (rateName == "faceTrackerOutbound") { - return _faceTrackerRateOutbound.rate() / BYTES_PER_KILOBIT; + return _outboundDataRate.faceTrackerRate.rate() / BYTES_PER_KILOBIT; } else if (rateName == "jointDataOutbound") { - return _jointDataRateOutbound.rate() / BYTES_PER_KILOBIT; + return _outboundDataRate.jointDataRate.rate() / BYTES_PER_KILOBIT; } return 0.0f; } @@ -1807,7 +1436,7 @@ void AvatarData::parseAvatarIdentityPacket(const QByteArray& data, Identity& ide } static const QUrl emptyURL(""); -const QUrl& AvatarData::cannonicalSkeletonModelURL(const QUrl& emptyURL) { +QUrl AvatarData::cannonicalSkeletonModelURL(const QUrl& emptyURL) const { // We don't put file urls on the wire, but instead convert to empty. return _skeletonModelURL.scheme() == "file" ? emptyURL : _skeletonModelURL; } @@ -1845,7 +1474,7 @@ void AvatarData::processAvatarIdentity(const Identity& identity, bool& identityC } } -QByteArray AvatarData::identityByteArray() { +QByteArray AvatarData::identityByteArray() const { QByteArray identityData; QDataStream identityStream(&identityData, QIODevice::Append); const QUrl& urlToSend = cannonicalSkeletonModelURL(emptyURL); @@ -2008,13 +1637,7 @@ void AvatarData::sendAvatarDataPacket() { bool cullSmallData = (randFloat() < AVATAR_SEND_FULL_UPDATE_RATIO); auto dataDetail = cullSmallData ? SendAllData : CullSmallData; - QVector lastSentJointData; - { - QReadLocker readLock(&_jointDataLock); - _lastSentJointData.resize(_jointData.size()); - lastSentJointData = _lastSentJointData; - } - QByteArray avatarByteArray = toByteArray(dataDetail, 0, lastSentJointData); + QByteArray avatarByteArray = toByteArray(dataDetail); doneEncoding(cullSmallData); static AvatarDataSequenceNumber sequenceNumber = 0; diff --git a/libraries/avatars/src/AvatarData.h b/libraries/avatars/src/AvatarData.h index 64c0e11577..14348b0622 100644 --- a/libraries/avatars/src/AvatarData.h +++ b/libraries/avatars/src/AvatarData.h @@ -288,6 +288,23 @@ class AttachmentData; class Transform; using TransformPointer = std::shared_ptr; +class AvatarDataRate { +public: + RateCounter<> globalPositionRate; + RateCounter<> localPositionRate; + RateCounter<> avatarBoundingBoxRate; + RateCounter<> avatarOrientationRate; + RateCounter<> avatarScaleRate; + RateCounter<> lookAtPositionRate; + RateCounter<> audioLoudnessRate; + RateCounter<> sensorToWorldRate; + RateCounter<> additionalFlagsRate; + RateCounter<> parentInfoRate; + RateCounter<> faceTrackerRate; + RateCounter<> jointDataRate; +}; + + class AvatarData : public QObject, public SpatiallyNestable { Q_OBJECT @@ -351,12 +368,12 @@ public: SendAllData } AvatarDataDetail; - virtual QByteArray toByteArray(AvatarDataDetail dataDetail, quint64 lastSentTime, const QVector& lastSentJointData, - bool distanceAdjust = false, glm::vec3 viewerPosition = glm::vec3(0), QVector* sentJointDataOut = nullptr); + virtual QByteArray toByteArray(AvatarDataDetail dataDetail); // FIXME virtual QByteArray toByteArray(AvatarDataDetail dataDetail, quint64 lastSentTime, const QVector& lastSentJointData, - AvatarDataPacket::HasFlags& hasFlagsOut, bool dropFaceTracking, bool distanceAdjust, glm::vec3 viewerPosition, QVector* sentJointDataOut) const; + AvatarDataPacket::HasFlags& hasFlagsOut, bool dropFaceTracking, bool distanceAdjust, glm::vec3 viewerPosition, + QVector* sentJointDataOut, AvatarDataRate* outboundDataRateOut = nullptr) const; virtual void doneEncoding(bool cullSmallChanges); @@ -497,7 +514,7 @@ public: // displayNameChanged returns true if displayName has changed, false otherwise. void processAvatarIdentity(const Identity& identity, bool& identityChanged, bool& displayNameChanged); - QByteArray identityByteArray(); + QByteArray identityByteArray() const; const QUrl& getSkeletonModelURL() const { return _skeletonModelURL; } const QString& getDisplayName() const { return _displayName; } @@ -626,7 +643,7 @@ protected: QVector _attachmentData; QString _displayName; QString _sessionDisplayName { }; - const QUrl& cannonicalSkeletonModelURL(const QUrl& empty); + QUrl cannonicalSkeletonModelURL(const QUrl& empty) const; float _displayNameTargetAlpha; float _displayNameAlpha; @@ -697,18 +714,7 @@ protected: RateCounter<> _jointDataUpdateRate; // Some rate data for outgoing data - RateCounter<> _globalPositionRateOutbound; - RateCounter<> _localPositionRateOutbound; - RateCounter<> _avatarBoundingBoxRateOutbound; - RateCounter<> _avatarOrientationRateOutbound; - RateCounter<> _avatarScaleRateOutbound; - RateCounter<> _lookAtPositionRateOutbound; - RateCounter<> _audioLoudnessRateOutbound; - RateCounter<> _sensorToWorldRateOutbound; - RateCounter<> _additionalFlagsRateOutbound; - RateCounter<> _parentInfoRateOutbound; - RateCounter<> _faceTrackerRateOutbound; - RateCounter<> _jointDataRateOutbound; + AvatarDataRate _outboundDataRate; glm::vec3 _globalBoundingBoxDimensions; glm::vec3 _globalBoundingBoxOffset;