From a7750501c6751473f8749befbf96c90c2158d883 Mon Sep 17 00:00:00 2001 From: Brad Hefta-Gaub Date: Tue, 13 Dec 2016 18:58:11 -0800 Subject: [PATCH 01/12] first cut at adding view frustum support to avatar mixer --- assignment-client/src/avatars/AvatarMixer.cpp | 23 +++++++ assignment-client/src/avatars/AvatarMixer.h | 1 + .../src/avatars/AvatarMixerClientData.cpp | 7 ++ .../src/avatars/AvatarMixerClientData.h | 9 +++ interface/src/Application.cpp | 9 +++ interface/src/Application.h | 1 + libraries/networking/src/udt/PacketHeaders.h | 3 +- libraries/shared/src/ViewFrustum.cpp | 67 +++++++++++++++++++ libraries/shared/src/ViewFrustum.h | 4 ++ 9 files changed, 123 insertions(+), 1 deletion(-) diff --git a/assignment-client/src/avatars/AvatarMixer.cpp b/assignment-client/src/avatars/AvatarMixer.cpp index a37c9f7dd5..36952d4fd6 100644 --- a/assignment-client/src/avatars/AvatarMixer.cpp +++ b/assignment-client/src/avatars/AvatarMixer.cpp @@ -44,6 +44,7 @@ AvatarMixer::AvatarMixer(ReceivedMessage& message) : connect(DependencyManager::get().data(), &NodeList::nodeKilled, this, &AvatarMixer::nodeKilled); auto& packetReceiver = DependencyManager::get()->getPacketReceiver(); + packetReceiver.registerListener(PacketType::ViewFrustum, this, "handleViewFrustumPacket"); packetReceiver.registerListener(PacketType::AvatarData, this, "handleAvatarDataPacket"); packetReceiver.registerListener(PacketType::AvatarIdentity, this, "handleAvatarIdentityPacket"); packetReceiver.registerListener(PacketType::KillAvatar, this, "handleKillAvatarPacket"); @@ -275,6 +276,16 @@ void AvatarMixer::broadcastAvatarData() { } // Not close enough to ignore nodeData->removeFromRadiusIgnoringSet(otherNode->getUUID()); + + + // Also check to see if the other node is in our view + glm::vec3 otherNodeBoxScale = (otherData->getPosition() - otherData->getGlobalBoundingBoxCorner()) * 2.0f; + AABox otherNodeBox(otherData->getGlobalBoundingBoxCorner(), otherNodeBoxScale); + if (!nodeData->otherAvatarInView(otherNodeBox)) { + qDebug() << "Avatar out of view!"; + return false; + } + return true; } }, @@ -448,6 +459,18 @@ void AvatarMixer::nodeKilled(SharedNodePointer killedNode) { } } +void AvatarMixer::handleViewFrustumPacket(QSharedPointer message, SharedNodePointer senderNode) { + auto nodeList = DependencyManager::get(); + nodeList->getOrCreateLinkedData(senderNode); + + if (senderNode->getLinkedData()) { + AvatarMixerClientData* nodeData = dynamic_cast(senderNode->getLinkedData()); + if (nodeData != nullptr) { + nodeData->readViewFrustumPacket(message->getMessage()); + } + } +} + void AvatarMixer::handleAvatarDataPacket(QSharedPointer message, SharedNodePointer senderNode) { auto nodeList = DependencyManager::get(); nodeList->updateNodeWithDataFromPacket(message, senderNode); diff --git a/assignment-client/src/avatars/AvatarMixer.h b/assignment-client/src/avatars/AvatarMixer.h index f537cc9244..ba74b43308 100644 --- a/assignment-client/src/avatars/AvatarMixer.h +++ b/assignment-client/src/avatars/AvatarMixer.h @@ -34,6 +34,7 @@ public slots: void sendStatsPacket() override; private slots: + void handleViewFrustumPacket(QSharedPointer message, SharedNodePointer senderNode); void handleAvatarDataPacket(QSharedPointer message, SharedNodePointer senderNode); void handleAvatarIdentityPacket(QSharedPointer message, SharedNodePointer senderNode); void handleKillAvatarPacket(QSharedPointer message); diff --git a/assignment-client/src/avatars/AvatarMixerClientData.cpp b/assignment-client/src/avatars/AvatarMixerClientData.cpp index de843c7cea..3773bd723f 100644 --- a/assignment-client/src/avatars/AvatarMixerClientData.cpp +++ b/assignment-client/src/avatars/AvatarMixerClientData.cpp @@ -57,6 +57,13 @@ void AvatarMixerClientData::ignoreOther(SharedNodePointer self, SharedNodePointe } } +void AvatarMixerClientData::readViewFrustumPacket(const QByteArray& message) { + _currentViewFrustumIsValid = true; + _currentViewFrustum.fromByteArray(message); + qDebug() << __FUNCTION__; + _currentViewFrustum.printDebugDetails(); +} + void AvatarMixerClientData::loadJSONStats(QJsonObject& jsonObject) const { jsonObject["display_name"] = _avatar->getDisplayName(); jsonObject["full_rate_distance"] = _fullRateDistance; diff --git a/assignment-client/src/avatars/AvatarMixerClientData.h b/assignment-client/src/avatars/AvatarMixerClientData.h index 20c1b6b647..85955804a9 100644 --- a/assignment-client/src/avatars/AvatarMixerClientData.h +++ b/assignment-client/src/avatars/AvatarMixerClientData.h @@ -27,6 +27,7 @@ #include #include #include +#include const QString OUTBOUND_AVATAR_DATA_STATS_KEY = "outbound_av_data_kbps"; const QString INBOUND_AVATAR_DATA_STATS_KEY = "inbound_av_data_kbps"; @@ -87,6 +88,12 @@ public: void removeFromRadiusIgnoringSet(const QUuid& other) { _radiusIgnoredOthers.erase(other); } void ignoreOther(SharedNodePointer self, SharedNodePointer other); + void readViewFrustumPacket(const QByteArray& message); + + bool otherAvatarInView(const AABox& otherAvatarBox) { + return !_currentViewFrustumIsValid || _currentViewFrustum.boxIntersectsKeyhole(otherAvatarBox); + } + private: AvatarSharedPointer _avatar { new AvatarData() }; @@ -108,6 +115,8 @@ private: SimpleMovingAverage _avgOtherAvatarDataRate; std::unordered_set _radiusIgnoredOthers; + ViewFrustum _currentViewFrustum; + bool _currentViewFrustumIsValid { false }; }; #endif // hifi_AvatarMixerClientData_h diff --git a/interface/src/Application.cpp b/interface/src/Application.cpp index 4ec29b636b..8c3e3fad73 100644 --- a/interface/src/Application.cpp +++ b/interface/src/Application.cpp @@ -4176,6 +4176,7 @@ void Application::update(float deltaTime) { if (DependencyManager::get()->shouldRenderEntities()) { queryOctree(NodeType::EntityServer, PacketType::EntityQuery, _entityServerJurisdictions); } + sendAvatarViewFrustum(); _lastQueriedViewFrustum = _viewFrustum; } } @@ -4215,6 +4216,14 @@ void Application::update(float deltaTime) { AnimDebugDraw::getInstance().update(); } +void Application::sendAvatarViewFrustum() { + QByteArray viewFrustumByteArray = _viewFrustum.toByteArray(); + auto avatarPacket = NLPacket::create(PacketType::ViewFrustum, viewFrustumByteArray.size()); + avatarPacket->write(viewFrustumByteArray); + + DependencyManager::get()->broadcastToNodes(std::move(avatarPacket), NodeSet() << NodeType::AvatarMixer); +} + int Application::sendNackPackets() { diff --git a/interface/src/Application.h b/interface/src/Application.h index 5ab94465cc..ea768b956b 100644 --- a/interface/src/Application.h +++ b/interface/src/Application.h @@ -415,6 +415,7 @@ private: void renderRearViewMirror(RenderArgs* renderArgs, const QRect& region, bool isZoomed); int sendNackPackets(); + void sendAvatarViewFrustum(); std::shared_ptr getMyAvatar() const; diff --git a/libraries/networking/src/udt/PacketHeaders.h b/libraries/networking/src/udt/PacketHeaders.h index dbeb1e63b0..bc5ff6fe98 100644 --- a/libraries/networking/src/udt/PacketHeaders.h +++ b/libraries/networking/src/udt/PacketHeaders.h @@ -101,7 +101,8 @@ public: NodeKickRequest, NodeMuteRequest, RadiusIgnoreRequest, - LAST_PACKET_TYPE = RadiusIgnoreRequest + ViewFrustum, + LAST_PACKET_TYPE = ViewFrustum }; }; diff --git a/libraries/shared/src/ViewFrustum.cpp b/libraries/shared/src/ViewFrustum.cpp index 0a1a41ace1..ead8f8634e 100644 --- a/libraries/shared/src/ViewFrustum.cpp +++ b/libraries/shared/src/ViewFrustum.cpp @@ -130,6 +130,73 @@ const char* ViewFrustum::debugPlaneName (int plane) const { return "Unknown"; } +void ViewFrustum::fromByteArray(const QByteArray& input) { + + // From the wire! + glm::vec3 cameraPosition; + glm::quat cameraOrientation; + float cameraCenterRadius; + float cameraFov; + float cameraAspectRatio; + float cameraNearClip; + float cameraFarClip; + + const unsigned char* startPosition = reinterpret_cast(input.constData()); + const unsigned char* sourceBuffer = startPosition; + + // camera details + memcpy(&cameraPosition, sourceBuffer, sizeof(cameraPosition)); + sourceBuffer += sizeof(cameraPosition); + sourceBuffer += unpackOrientationQuatFromBytes(sourceBuffer, cameraOrientation); + sourceBuffer += unpackFloatAngleFromTwoByte((uint16_t*)sourceBuffer, &cameraFov); + sourceBuffer += unpackFloatRatioFromTwoByte(sourceBuffer, cameraAspectRatio); + sourceBuffer += unpackClipValueFromTwoByte(sourceBuffer, cameraNearClip); + sourceBuffer += unpackClipValueFromTwoByte(sourceBuffer, cameraFarClip); + memcpy(&cameraCenterRadius, sourceBuffer, sizeof(cameraCenterRadius)); + sourceBuffer += sizeof(cameraCenterRadius); + + setPosition(cameraPosition); + setOrientation(cameraOrientation); + setCenterRadius(cameraCenterRadius); + + // Also make sure it's got the correct lens details from the camera + const float VIEW_FRUSTUM_FOV_OVERSEND = 60.0f; + float originalFOV = cameraFov; + float wideFOV = originalFOV + VIEW_FRUSTUM_FOV_OVERSEND; + + if (0.0f != cameraAspectRatio && + 0.0f != cameraNearClip && + 0.0f != cameraFarClip && + cameraNearClip != cameraFarClip) { + setProjection(glm::perspective( + glm::radians(wideFOV), // hack + cameraAspectRatio, + cameraNearClip, + cameraFarClip)); + } +} + + +QByteArray ViewFrustum::toByteArray() { + static const int LARGE_ENOUGH = 1024; + QByteArray viewFrustumDataByteArray(LARGE_ENOUGH, 0); + unsigned char* destinationBuffer = reinterpret_cast(viewFrustumDataByteArray.data()); + unsigned char* startPosition = destinationBuffer; + + // camera details + memcpy(destinationBuffer, &_position, sizeof(_position)); + destinationBuffer += sizeof(_position); + destinationBuffer += packOrientationQuatToBytes(destinationBuffer, _orientation); + destinationBuffer += packFloatAngleToTwoByte(destinationBuffer, _fieldOfView); + destinationBuffer += packFloatRatioToTwoByte(destinationBuffer, _aspectRatio); + destinationBuffer += packClipValueToTwoByte(destinationBuffer, _nearClip); + destinationBuffer += packClipValueToTwoByte(destinationBuffer, _farClip); + memcpy(destinationBuffer, &_centerSphereRadius, sizeof(_centerSphereRadius)); + destinationBuffer += sizeof(_centerSphereRadius); + + return viewFrustumDataByteArray.left(destinationBuffer - startPosition); +} + ViewFrustum::intersection ViewFrustum::calculateCubeFrustumIntersection(const AACube& cube) const { // only check against frustum ViewFrustum::intersection result = INSIDE; diff --git a/libraries/shared/src/ViewFrustum.h b/libraries/shared/src/ViewFrustum.h index 091aff7a92..9a6cb9ab68 100644 --- a/libraries/shared/src/ViewFrustum.h +++ b/libraries/shared/src/ViewFrustum.h @@ -139,6 +139,10 @@ public: const ::Plane* getPlanes() const { return _planes; } void invalidate(); // causes all reasonable intersection tests to fail + + QByteArray toByteArray(); + void fromByteArray(const QByteArray& input); + private: glm::mat4 _view; glm::mat4 _projection; From 7189f766a2e61fcf6925a65840a2ca8d1454c598 Mon Sep 17 00:00:00 2001 From: Brad Hefta-Gaub Date: Tue, 13 Dec 2016 20:56:41 -0800 Subject: [PATCH 02/12] more hacking --- assignment-client/src/avatars/AvatarMixer.cpp | 5 ++++- .../src/avatars/AvatarMixerClientData.cpp | 11 +++++++++-- assignment-client/src/avatars/AvatarMixerClientData.h | 5 ++--- libraries/shared/src/ViewFrustum.cpp | 2 ++ 4 files changed, 17 insertions(+), 6 deletions(-) diff --git a/assignment-client/src/avatars/AvatarMixer.cpp b/assignment-client/src/avatars/AvatarMixer.cpp index 36952d4fd6..5f031e537e 100644 --- a/assignment-client/src/avatars/AvatarMixer.cpp +++ b/assignment-client/src/avatars/AvatarMixer.cpp @@ -281,9 +281,12 @@ void AvatarMixer::broadcastAvatarData() { // Also check to see if the other node is in our view glm::vec3 otherNodeBoxScale = (otherData->getPosition() - otherData->getGlobalBoundingBoxCorner()) * 2.0f; AABox otherNodeBox(otherData->getGlobalBoundingBoxCorner(), otherNodeBoxScale); + if (!nodeData->otherAvatarInView(otherNodeBox)) { - qDebug() << "Avatar out of view!"; + //qDebug() << "Avatar out of view!"; return false; + } else { + //qDebug() << "Avatar in view!"; } return true; diff --git a/assignment-client/src/avatars/AvatarMixerClientData.cpp b/assignment-client/src/avatars/AvatarMixerClientData.cpp index 3773bd723f..63a1b108b8 100644 --- a/assignment-client/src/avatars/AvatarMixerClientData.cpp +++ b/assignment-client/src/avatars/AvatarMixerClientData.cpp @@ -60,10 +60,17 @@ void AvatarMixerClientData::ignoreOther(SharedNodePointer self, SharedNodePointe void AvatarMixerClientData::readViewFrustumPacket(const QByteArray& message) { _currentViewFrustumIsValid = true; _currentViewFrustum.fromByteArray(message); - qDebug() << __FUNCTION__; - _currentViewFrustum.printDebugDetails(); } +bool AvatarMixerClientData::otherAvatarInView(const glm::vec3& otherAvatar) { + return !_currentViewFrustumIsValid || _currentViewFrustum.pointIntersectsFrustum(otherAvatar); +} + +bool AvatarMixerClientData::otherAvatarInView(const AABox& otherAvatarBox) { + return !_currentViewFrustumIsValid || _currentViewFrustum.boxIntersectsKeyhole(otherAvatarBox); +} + + void AvatarMixerClientData::loadJSONStats(QJsonObject& jsonObject) const { jsonObject["display_name"] = _avatar->getDisplayName(); jsonObject["full_rate_distance"] = _fullRateDistance; diff --git a/assignment-client/src/avatars/AvatarMixerClientData.h b/assignment-client/src/avatars/AvatarMixerClientData.h index 85955804a9..b7b7b16ea1 100644 --- a/assignment-client/src/avatars/AvatarMixerClientData.h +++ b/assignment-client/src/avatars/AvatarMixerClientData.h @@ -90,9 +90,8 @@ public: void readViewFrustumPacket(const QByteArray& message); - bool otherAvatarInView(const AABox& otherAvatarBox) { - return !_currentViewFrustumIsValid || _currentViewFrustum.boxIntersectsKeyhole(otherAvatarBox); - } + bool otherAvatarInView(const AABox& otherAvatarBox); + bool otherAvatarInView(const glm::vec3& otherAvatar); private: AvatarSharedPointer _avatar { new AvatarData() }; diff --git a/libraries/shared/src/ViewFrustum.cpp b/libraries/shared/src/ViewFrustum.cpp index ead8f8634e..b4ed462639 100644 --- a/libraries/shared/src/ViewFrustum.cpp +++ b/libraries/shared/src/ViewFrustum.cpp @@ -173,6 +173,8 @@ void ViewFrustum::fromByteArray(const QByteArray& input) { cameraAspectRatio, cameraNearClip, cameraFarClip)); + + calculate(); } } From 229017928193aca520d4d592bd0d4cde8c5a4430 Mon Sep 17 00:00:00 2001 From: Brad Hefta-Gaub Date: Wed, 14 Dec 2016 11:47:45 -0800 Subject: [PATCH 03/12] add compression and minimal data flags to AvatarData --- interface/src/avatar/MyAvatar.cpp | 6 +- interface/src/avatar/MyAvatar.h | 2 +- libraries/avatars/src/AvatarData.cpp | 421 +++++++++++++++------------ libraries/avatars/src/AvatarData.h | 9 +- 4 files changed, 250 insertions(+), 188 deletions(-) diff --git a/interface/src/avatar/MyAvatar.cpp b/interface/src/avatar/MyAvatar.cpp index c56d255201..50517b706c 100644 --- a/interface/src/avatar/MyAvatar.cpp +++ b/interface/src/avatar/MyAvatar.cpp @@ -227,7 +227,7 @@ void MyAvatar::simulateAttachments(float deltaTime) { // don't update attachments here, do it in harvestResultsFromPhysicsSimulation() } -QByteArray MyAvatar::toByteArray(bool cullSmallChanges, bool sendAll) { +QByteArray MyAvatar::toByteArray(bool cullSmallChanges, bool sendAll, bool sendMinimum, bool compressed) { CameraMode mode = qApp->getCamera()->getMode(); _globalPosition = getPosition(); _globalBoundingBoxCorner.x = _characterController.getCapsuleRadius(); @@ -238,12 +238,12 @@ QByteArray MyAvatar::toByteArray(bool cullSmallChanges, bool sendAll) { // fake the avatar position that is sent up to the AvatarMixer glm::vec3 oldPosition = getPosition(); setPosition(getSkeletonPosition()); - QByteArray array = AvatarData::toByteArray(cullSmallChanges, sendAll); + QByteArray array = AvatarData::toByteArray(cullSmallChanges, sendAll, sendMinimum, compressed); // copy the correct position back setPosition(oldPosition); return array; } - return AvatarData::toByteArray(cullSmallChanges, sendAll); + return AvatarData::toByteArray(cullSmallChanges, sendAll, sendMinimum, compressed); } void MyAvatar::centerBody() { diff --git a/interface/src/avatar/MyAvatar.h b/interface/src/avatar/MyAvatar.h index f081ec533b..609e93f616 100644 --- a/interface/src/avatar/MyAvatar.h +++ b/interface/src/avatar/MyAvatar.h @@ -333,7 +333,7 @@ private: glm::vec3 getWorldBodyPosition() const; glm::quat getWorldBodyOrientation() const; - QByteArray toByteArray(bool cullSmallChanges, bool sendAll) override; + QByteArray toByteArray(bool cullSmallChanges, bool sendAll, bool sendMinimum, bool compressed) override; void simulate(float deltaTime); void updateFromTrackers(float deltaTime); virtual void render(RenderArgs* renderArgs, const glm::vec3& cameraPositio) override; diff --git a/libraries/avatars/src/AvatarData.cpp b/libraries/avatars/src/AvatarData.cpp index f81a1169af..ac378276fd 100644 --- a/libraries/avatars/src/AvatarData.cpp +++ b/libraries/avatars/src/AvatarData.cpp @@ -181,7 +181,7 @@ void AvatarData::setHandPosition(const glm::vec3& handPosition) { _handPosition = glm::inverse(getOrientation()) * (handPosition - getPosition()); } -QByteArray AvatarData::toByteArray(bool cullSmallChanges, bool sendAll) { +QByteArray AvatarData::toByteArray(bool cullSmallChanges, bool sendAll, bool sendMinimum, bool compressed) { // TODO: DRY this up to a shared method // that can pack any type given the number of bytes // and return the number of bytes to push the pointer @@ -199,208 +199,242 @@ QByteArray AvatarData::toByteArray(bool cullSmallChanges, bool sendAll) { unsigned char* destinationBuffer = reinterpret_cast(avatarDataByteArray.data()); unsigned char* startPosition = destinationBuffer; - auto header = reinterpret_cast(destinationBuffer); - header->position[0] = getLocalPosition().x; - header->position[1] = getLocalPosition().y; - header->position[2] = getLocalPosition().z; - header->globalPosition[0] = _globalPosition.x; - header->globalPosition[1] = _globalPosition.y; - header->globalPosition[2] = _globalPosition.z; - header->globalBoundingBoxCorner[0] = getPosition().x - _globalBoundingBoxCorner.x; - header->globalBoundingBoxCorner[1] = getPosition().y - _globalBoundingBoxCorner.y; - header->globalBoundingBoxCorner[2] = getPosition().z - _globalBoundingBoxCorner.z; - - glm::vec3 bodyEulerAngles = glm::degrees(safeEulerAngles(getLocalOrientation())); - packFloatAngleToTwoByte((uint8_t*)(header->localOrientation + 0), bodyEulerAngles.y); - packFloatAngleToTwoByte((uint8_t*)(header->localOrientation + 1), bodyEulerAngles.x); - packFloatAngleToTwoByte((uint8_t*)(header->localOrientation + 2), bodyEulerAngles.z); - packFloatRatioToTwoByte((uint8_t*)(&header->scale), getDomainLimitedScale()); - header->lookAtPosition[0] = _headData->_lookAtPosition.x; - header->lookAtPosition[1] = _headData->_lookAtPosition.y; - header->lookAtPosition[2] = _headData->_lookAtPosition.z; - header->audioLoudness = _headData->_audioLoudness; - - glm::mat4 sensorToWorldMatrix = getSensorToWorldMatrix(); - packOrientationQuatToSixBytes(header->sensorToWorldQuat, glmExtractRotation(sensorToWorldMatrix)); - glm::vec3 scale = extractScale(sensorToWorldMatrix); - packFloatScalarToSignedTwoByteFixed((uint8_t*)&header->sensorToWorldScale, scale.x, SENSOR_TO_WORLD_SCALE_RADIX); - header->sensorToWorldTrans[0] = sensorToWorldMatrix[3][0]; - header->sensorToWorldTrans[1] = sensorToWorldMatrix[3][1]; - header->sensorToWorldTrans[2] = sensorToWorldMatrix[3][2]; - - setSemiNibbleAt(header->flags, KEY_STATE_START_BIT, _keyState); - // hand state - bool isFingerPointing = _handState & IS_FINGER_POINTING_FLAG; - setSemiNibbleAt(header->flags, HAND_STATE_START_BIT, _handState & ~IS_FINGER_POINTING_FLAG); - if (isFingerPointing) { - setAtBit(header->flags, HAND_STATE_FINGER_POINTING_BIT); - } - // faceshift state - if (_headData->_isFaceTrackerConnected) { - setAtBit(header->flags, IS_FACESHIFT_CONNECTED); - } - // eye tracker state - if (_headData->_isEyeTrackerConnected) { - setAtBit(header->flags, IS_EYE_TRACKER_CONNECTED); - } - // referential state - QUuid parentID = getParentID(); - if (!parentID.isNull()) { - setAtBit(header->flags, HAS_REFERENTIAL); - } - destinationBuffer += sizeof(AvatarDataPacket::Header); - - if (!parentID.isNull()) { - auto parentInfo = reinterpret_cast(destinationBuffer); - QByteArray referentialAsBytes = parentID.toRfc4122(); - memcpy(parentInfo->parentUUID, referentialAsBytes.data(), referentialAsBytes.size()); - parentInfo->parentJointIndex = _parentJointIndex; - destinationBuffer += sizeof(AvatarDataPacket::ParentInfo); + // Leading flags, to indicate how much data is actually included in the packet... + UINT8 packetStateFlags = 0; + if (sendMinimum) { + setAtBit(packetStateFlags, AVATARDATA_FLAGS_MINIMUM); } - // If it is connected, pack up the data - if (_headData->_isFaceTrackerConnected) { - auto faceTrackerInfo = reinterpret_cast(destinationBuffer); - - faceTrackerInfo->leftEyeBlink = _headData->_leftEyeBlink; - faceTrackerInfo->rightEyeBlink = _headData->_rightEyeBlink; - faceTrackerInfo->averageLoudness = _headData->_averageLoudness; - faceTrackerInfo->browAudioLift = _headData->_browAudioLift; - faceTrackerInfo->numBlendshapeCoefficients = _headData->_blendshapeCoefficients.size(); - destinationBuffer += sizeof(AvatarDataPacket::FaceTrackerInfo); - - // followed by a variable number of float coefficients - memcpy(destinationBuffer, _headData->_blendshapeCoefficients.data(), _headData->_blendshapeCoefficients.size() * sizeof(float)); - destinationBuffer += _headData->_blendshapeCoefficients.size() * sizeof(float); + // do we need this, or do we just always compress it? + if (compressed) { + setAtBit(packetStateFlags, AVATARDATA_FLAGS_COMPRESSED); } + memcpy(destinationBuffer, &packetStateFlags, sizeof(packetStateFlags)); + destinationBuffer += sizeof(packetStateFlags); - QReadLocker readLock(&_jointDataLock); + if (!sendMinimum) { + auto header = reinterpret_cast(destinationBuffer); + header->position[0] = getLocalPosition().x; + header->position[1] = getLocalPosition().y; + header->position[2] = getLocalPosition().z; + header->globalPosition[0] = _globalPosition.x; + header->globalPosition[1] = _globalPosition.y; + header->globalPosition[2] = _globalPosition.z; + header->globalBoundingBoxCorner[0] = getPosition().x - _globalBoundingBoxCorner.x; + header->globalBoundingBoxCorner[1] = getPosition().y - _globalBoundingBoxCorner.y; + header->globalBoundingBoxCorner[2] = getPosition().z - _globalBoundingBoxCorner.z; - // joint rotation data - *destinationBuffer++ = _jointData.size(); - unsigned char* validityPosition = destinationBuffer; - unsigned char validity = 0; - int validityBit = 0; + glm::vec3 bodyEulerAngles = glm::degrees(safeEulerAngles(getLocalOrientation())); + packFloatAngleToTwoByte((uint8_t*)(header->localOrientation + 0), bodyEulerAngles.y); + packFloatAngleToTwoByte((uint8_t*)(header->localOrientation + 1), bodyEulerAngles.x); + packFloatAngleToTwoByte((uint8_t*)(header->localOrientation + 2), bodyEulerAngles.z); + packFloatRatioToTwoByte((uint8_t*)(&header->scale), getDomainLimitedScale()); + header->lookAtPosition[0] = _headData->_lookAtPosition.x; + header->lookAtPosition[1] = _headData->_lookAtPosition.y; + header->lookAtPosition[2] = _headData->_lookAtPosition.z; + header->audioLoudness = _headData->_audioLoudness; - #ifdef WANT_DEBUG - int rotationSentCount = 0; - unsigned char* beforeRotations = destinationBuffer; - #endif + glm::mat4 sensorToWorldMatrix = getSensorToWorldMatrix(); + packOrientationQuatToSixBytes(header->sensorToWorldQuat, glmExtractRotation(sensorToWorldMatrix)); + glm::vec3 scale = extractScale(sensorToWorldMatrix); + packFloatScalarToSignedTwoByteFixed((uint8_t*)&header->sensorToWorldScale, scale.x, SENSOR_TO_WORLD_SCALE_RADIX); + header->sensorToWorldTrans[0] = sensorToWorldMatrix[3][0]; + header->sensorToWorldTrans[1] = sensorToWorldMatrix[3][1]; + header->sensorToWorldTrans[2] = sensorToWorldMatrix[3][2]; - _lastSentJointData.resize(_jointData.size()); + setSemiNibbleAt(header->flags, KEY_STATE_START_BIT, _keyState); + // hand state + bool isFingerPointing = _handState & IS_FINGER_POINTING_FLAG; + setSemiNibbleAt(header->flags, HAND_STATE_START_BIT, _handState & ~IS_FINGER_POINTING_FLAG); + if (isFingerPointing) { + setAtBit(header->flags, HAND_STATE_FINGER_POINTING_BIT); + } + // faceshift state + if (_headData->_isFaceTrackerConnected) { + setAtBit(header->flags, IS_FACESHIFT_CONNECTED); + } + // eye tracker state + if (_headData->_isEyeTrackerConnected) { + setAtBit(header->flags, IS_EYE_TRACKER_CONNECTED); + } + // referential state + QUuid parentID = getParentID(); + if (!parentID.isNull()) { + setAtBit(header->flags, HAS_REFERENTIAL); + } + destinationBuffer += sizeof(AvatarDataPacket::Header); - for (int i=0; i < _jointData.size(); i++) { - const JointData& data = _jointData[i]; - if (sendAll || _lastSentJointData[i].rotation != data.rotation) { - if (sendAll || - !cullSmallChanges || - fabsf(glm::dot(data.rotation, _lastSentJointData[i].rotation)) <= AVATAR_MIN_ROTATION_DOT) { - if (data.rotationSet) { - validity |= (1 << validityBit); - #ifdef WANT_DEBUG - rotationSentCount++; - #endif + if (!parentID.isNull()) { + auto parentInfo = reinterpret_cast(destinationBuffer); + QByteArray referentialAsBytes = parentID.toRfc4122(); + memcpy(parentInfo->parentUUID, referentialAsBytes.data(), referentialAsBytes.size()); + parentInfo->parentJointIndex = _parentJointIndex; + destinationBuffer += sizeof(AvatarDataPacket::ParentInfo); + } + + // If it is connected, pack up the data + if (_headData->_isFaceTrackerConnected) { + auto faceTrackerInfo = reinterpret_cast(destinationBuffer); + + faceTrackerInfo->leftEyeBlink = _headData->_leftEyeBlink; + faceTrackerInfo->rightEyeBlink = _headData->_rightEyeBlink; + faceTrackerInfo->averageLoudness = _headData->_averageLoudness; + faceTrackerInfo->browAudioLift = _headData->_browAudioLift; + faceTrackerInfo->numBlendshapeCoefficients = _headData->_blendshapeCoefficients.size(); + destinationBuffer += sizeof(AvatarDataPacket::FaceTrackerInfo); + + // followed by a variable number of float coefficients + memcpy(destinationBuffer, _headData->_blendshapeCoefficients.data(), _headData->_blendshapeCoefficients.size() * sizeof(float)); + destinationBuffer += _headData->_blendshapeCoefficients.size() * sizeof(float); + } + + QReadLocker readLock(&_jointDataLock); + + // joint rotation data + *destinationBuffer++ = _jointData.size(); + unsigned char* validityPosition = destinationBuffer; + unsigned char validity = 0; + int validityBit = 0; + + #ifdef WANT_DEBUG + int rotationSentCount = 0; + unsigned char* beforeRotations = destinationBuffer; + #endif + + _lastSentJointData.resize(_jointData.size()); + + for (int i=0; i < _jointData.size(); i++) { + const JointData& data = _jointData[i]; + if (sendAll || _lastSentJointData[i].rotation != data.rotation) { + if (sendAll || + !cullSmallChanges || + fabsf(glm::dot(data.rotation, _lastSentJointData[i].rotation)) <= AVATAR_MIN_ROTATION_DOT) { + if (data.rotationSet) { + validity |= (1 << validityBit); + #ifdef WANT_DEBUG + rotationSentCount++; + #endif + } } } - } - if (++validityBit == BITS_IN_BYTE) { - *destinationBuffer++ = validity; - validityBit = validity = 0; - } - } - if (validityBit != 0) { - *destinationBuffer++ = validity; - } - - validityBit = 0; - validity = *validityPosition++; - for (int i = 0; i < _jointData.size(); i ++) { - const JointData& data = _jointData[i]; - if (validity & (1 << validityBit)) { - destinationBuffer += packOrientationQuatToSixBytes(destinationBuffer, data.rotation); - } - if (++validityBit == BITS_IN_BYTE) { - validityBit = 0; - validity = *validityPosition++; - } - } - - - // joint translation data - validityPosition = destinationBuffer; - validity = 0; - validityBit = 0; - - #ifdef WANT_DEBUG - int translationSentCount = 0; - unsigned char* beforeTranslations = destinationBuffer; - #endif - - float maxTranslationDimension = 0.0; - for (int i=0; i < _jointData.size(); i++) { - const JointData& data = _jointData[i]; - if (sendAll || _lastSentJointData[i].translation != data.translation) { - if (sendAll || - !cullSmallChanges || - glm::distance(data.translation, _lastSentJointData[i].translation) > AVATAR_MIN_TRANSLATION) { - if (data.translationSet) { - validity |= (1 << validityBit); - #ifdef WANT_DEBUG - translationSentCount++; - #endif - maxTranslationDimension = glm::max(fabsf(data.translation.x), maxTranslationDimension); - maxTranslationDimension = glm::max(fabsf(data.translation.y), maxTranslationDimension); - maxTranslationDimension = glm::max(fabsf(data.translation.z), maxTranslationDimension); - } + if (++validityBit == BITS_IN_BYTE) { + *destinationBuffer++ = validity; + validityBit = validity = 0; } } - if (++validityBit == BITS_IN_BYTE) { + if (validityBit != 0) { *destinationBuffer++ = validity; - validityBit = validity = 0; } - } - if (validityBit != 0) { - *destinationBuffer++ = validity; - } - - validityBit = 0; - validity = *validityPosition++; - for (int i = 0; i < _jointData.size(); i ++) { - const JointData& data = _jointData[i]; - if (validity & (1 << validityBit)) { - destinationBuffer += - packFloatVec3ToSignedTwoByteFixed(destinationBuffer, data.translation, TRANSLATION_COMPRESSION_RADIX); + validityBit = 0; + validity = *validityPosition++; + for (int i = 0; i < _jointData.size(); i ++) { + const JointData& data = _jointData[i]; + if (validity & (1 << validityBit)) { + destinationBuffer += packOrientationQuatToSixBytes(destinationBuffer, data.rotation); + } + if (++validityBit == BITS_IN_BYTE) { + validityBit = 0; + validity = *validityPosition++; + } } - if (++validityBit == BITS_IN_BYTE) { - validityBit = 0; - validity = *validityPosition++; + + + // joint translation data + validityPosition = destinationBuffer; + validity = 0; + validityBit = 0; + + #ifdef WANT_DEBUG + int translationSentCount = 0; + unsigned char* beforeTranslations = destinationBuffer; + #endif + + float maxTranslationDimension = 0.0; + for (int i=0; i < _jointData.size(); i++) { + const JointData& data = _jointData[i]; + if (sendAll || _lastSentJointData[i].translation != data.translation) { + if (sendAll || + !cullSmallChanges || + glm::distance(data.translation, _lastSentJointData[i].translation) > AVATAR_MIN_TRANSLATION) { + if (data.translationSet) { + validity |= (1 << validityBit); + #ifdef WANT_DEBUG + translationSentCount++; + #endif + maxTranslationDimension = glm::max(fabsf(data.translation.x), maxTranslationDimension); + maxTranslationDimension = glm::max(fabsf(data.translation.y), maxTranslationDimension); + maxTranslationDimension = glm::max(fabsf(data.translation.z), maxTranslationDimension); + } + } + } + if (++validityBit == BITS_IN_BYTE) { + *destinationBuffer++ = validity; + validityBit = validity = 0; + } } + + if (validityBit != 0) { + *destinationBuffer++ = validity; + } + + validityBit = 0; + validity = *validityPosition++; + for (int i = 0; i < _jointData.size(); i ++) { + const JointData& data = _jointData[i]; + if (validity & (1 << validityBit)) { + destinationBuffer += + packFloatVec3ToSignedTwoByteFixed(destinationBuffer, data.translation, TRANSLATION_COMPRESSION_RADIX); + } + if (++validityBit == BITS_IN_BYTE) { + validityBit = 0; + validity = *validityPosition++; + } + } + + // faux joints + Transform controllerLeftHandTransform = Transform(getControllerLeftHandMatrix()); + destinationBuffer += packOrientationQuatToSixBytes(destinationBuffer, controllerLeftHandTransform.getRotation()); + destinationBuffer += packFloatVec3ToSignedTwoByteFixed(destinationBuffer, controllerLeftHandTransform.getTranslation(), + TRANSLATION_COMPRESSION_RADIX); + Transform controllerRightHandTransform = Transform(getControllerRightHandMatrix()); + destinationBuffer += packOrientationQuatToSixBytes(destinationBuffer, controllerRightHandTransform.getRotation()); + destinationBuffer += packFloatVec3ToSignedTwoByteFixed(destinationBuffer, controllerRightHandTransform.getTranslation(), + TRANSLATION_COMPRESSION_RADIX); + + #ifdef WANT_DEBUG + if (sendAll) { + qDebug() << "AvatarData::toByteArray" << cullSmallChanges << sendAll + << "rotations:" << rotationSentCount << "translations:" << translationSentCount + << "largest:" << maxTranslationDimension + << "size:" + << (beforeRotations - startPosition) << "+" + << (beforeTranslations - beforeRotations) << "+" + << (destinationBuffer - beforeTranslations) << "=" + << (destinationBuffer - startPosition); + } + #endif } - // faux joints - Transform controllerLeftHandTransform = Transform(getControllerLeftHandMatrix()); - destinationBuffer += packOrientationQuatToSixBytes(destinationBuffer, controllerLeftHandTransform.getRotation()); - destinationBuffer += packFloatVec3ToSignedTwoByteFixed(destinationBuffer, controllerLeftHandTransform.getTranslation(), - TRANSLATION_COMPRESSION_RADIX); - Transform controllerRightHandTransform = Transform(getControllerRightHandMatrix()); - destinationBuffer += packOrientationQuatToSixBytes(destinationBuffer, controllerRightHandTransform.getRotation()); - destinationBuffer += packFloatVec3ToSignedTwoByteFixed(destinationBuffer, controllerRightHandTransform.getTranslation(), - TRANSLATION_COMPRESSION_RADIX); + // NOTE: first byte of array should always be uncompressed - #ifdef WANT_DEBUG - if (sendAll) { - qDebug() << "AvatarData::toByteArray" << cullSmallChanges << sendAll - << "rotations:" << rotationSentCount << "translations:" << translationSentCount - << "largest:" << maxTranslationDimension - << "size:" - << (beforeRotations - startPosition) << "+" - << (beforeTranslations - beforeRotations) << "+" - << (destinationBuffer - beforeTranslations) << "=" - << (destinationBuffer - startPosition); + if (compressed) { + static const int SKIP_PACKET_FLAGS = 1; + static const int COMPRESSION_LEVEL = 9; + QByteArray uncompressedPortion = avatarDataByteArray.mid(SKIP_PACKET_FLAGS, + (destinationBuffer - startPosition) - SKIP_PACKET_FLAGS); + + QByteArray compressedPortion = qCompress(uncompressedPortion, COMPRESSION_LEVEL); + QByteArray flagsAndCompressed; + flagsAndCompressed.append(packetStateFlags); + flagsAndCompressed.append(compressedPortion); + + //qDebug() << __FUNCTION__ << "compressing data was:" << (uncompressedPortion.size() + SKIP_PACKET_FLAGS) << "now:" << flagsAndCompressed.size(); + + return flagsAndCompressed; } - #endif + // entire buffer is uncompressed return avatarDataByteArray.left(destinationBuffer - startPosition); } @@ -474,9 +508,30 @@ int AvatarData::parseDataFromBuffer(const QByteArray& buffer) { _headData = new HeadData(this); } + UINT8 packetStateFlags = buffer.at(0); + bool minimumSent = oneAtBit(packetStateFlags, AVATARDATA_FLAGS_MINIMUM); + bool packetIsCompressed = oneAtBit(packetStateFlags, AVATARDATA_FLAGS_COMPRESSED); + + // if this is the minimum, then it only includes the flags + if (minimumSent) { + int numBytesRead = sizeof(packetStateFlags); + _averageBytesReceived.updateAverage(numBytesRead); + return numBytesRead; + } + + QByteArray uncompressBuffer; const unsigned char* startPosition = reinterpret_cast(buffer.data()); const unsigned char* endPosition = startPosition + buffer.size(); const unsigned char* sourceBuffer = startPosition; + + if (packetIsCompressed) { + uncompressBuffer = qUncompress(buffer.right(buffer.size() - sizeof(packetStateFlags))); + startPosition = reinterpret_cast(uncompressBuffer.data()); + endPosition = startPosition + uncompressBuffer.size(); + sourceBuffer = startPosition; + //qDebug() << __FUNCTION__ << "uncompressing compressed data was:" << buffer.size() << "now:" << uncompressBuffer.size(); + } + quint64 now = usecTimestampNow(); PACKET_READ_CHECK(Header, sizeof(AvatarDataPacket::Header)); @@ -679,8 +734,8 @@ int AvatarData::parseDataFromBuffer(const QByteArray& buffer) { #ifdef WANT_DEBUG if (numValidJointRotations > 15) { qDebug() << "RECEIVING -- rotations:" << numValidJointRotations - << "translations:" << numValidJointTranslations - << "size:" << (int)(sourceBuffer - startPosition); + << "translations:" << numValidJointTranslations + << "size:" << (int)(sourceBuffer - startPosition); } #endif diff --git a/libraries/avatars/src/AvatarData.h b/libraries/avatars/src/AvatarData.h index b34b912f14..49f9336aeb 100644 --- a/libraries/avatars/src/AvatarData.h +++ b/libraries/avatars/src/AvatarData.h @@ -106,6 +106,13 @@ const char LEFT_HAND_POINTING_FLAG = 1; const char RIGHT_HAND_POINTING_FLAG = 2; const char IS_FINGER_POINTING_FLAG = 4; +// AvatarData state flags - we store the details about the packet encoding in the first byte, +// before the "header" structure +const char AVATARDATA_FLAGS_MINIMUM = 0; +const char AVATARDATA_FLAGS_COMPRESSED = 1; + + + static const float MAX_AVATAR_SCALE = 1000.0f; static const float MIN_AVATAR_SCALE = .005f; @@ -201,7 +208,7 @@ public: glm::vec3 getHandPosition() const; void setHandPosition(const glm::vec3& handPosition); - virtual QByteArray toByteArray(bool cullSmallChanges, bool sendAll); + virtual QByteArray toByteArray(bool cullSmallChanges, bool sendAll, bool sendMinimum = false, bool compressed = true); virtual void doneEncoding(bool cullSmallChanges); /// \return true if an error should be logged From e57774f6306d0a1bd8e1b884e7a02a4677c73504 Mon Sep 17 00:00:00 2001 From: Brad Hefta-Gaub Date: Wed, 14 Dec 2016 12:19:50 -0800 Subject: [PATCH 04/12] fix unix/mac build --- libraries/avatars/src/AvatarData.cpp | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/libraries/avatars/src/AvatarData.cpp b/libraries/avatars/src/AvatarData.cpp index ac378276fd..34e9838284 100644 --- a/libraries/avatars/src/AvatarData.cpp +++ b/libraries/avatars/src/AvatarData.cpp @@ -200,7 +200,7 @@ QByteArray AvatarData::toByteArray(bool cullSmallChanges, bool sendAll, bool sen unsigned char* startPosition = destinationBuffer; // Leading flags, to indicate how much data is actually included in the packet... - UINT8 packetStateFlags = 0; + uint8_t packetStateFlags = 0; if (sendMinimum) { setAtBit(packetStateFlags, AVATARDATA_FLAGS_MINIMUM); } @@ -508,7 +508,7 @@ int AvatarData::parseDataFromBuffer(const QByteArray& buffer) { _headData = new HeadData(this); } - UINT8 packetStateFlags = buffer.at(0); + uint8_t packetStateFlags = buffer.at(0); bool minimumSent = oneAtBit(packetStateFlags, AVATARDATA_FLAGS_MINIMUM); bool packetIsCompressed = oneAtBit(packetStateFlags, AVATARDATA_FLAGS_COMPRESSED); From daccb708193111b41c173a56cecc10c3213de3fe Mon Sep 17 00:00:00 2001 From: Brad Hefta-Gaub Date: Wed, 14 Dec 2016 13:18:09 -0800 Subject: [PATCH 05/12] disable compression by default --- libraries/avatars/src/AvatarData.h | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/libraries/avatars/src/AvatarData.h b/libraries/avatars/src/AvatarData.h index 49f9336aeb..c985559913 100644 --- a/libraries/avatars/src/AvatarData.h +++ b/libraries/avatars/src/AvatarData.h @@ -208,7 +208,7 @@ public: glm::vec3 getHandPosition() const; void setHandPosition(const glm::vec3& handPosition); - virtual QByteArray toByteArray(bool cullSmallChanges, bool sendAll, bool sendMinimum = false, bool compressed = true); + virtual QByteArray toByteArray(bool cullSmallChanges, bool sendAll, bool sendMinimum = false, bool compressed = false); virtual void doneEncoding(bool cullSmallChanges); /// \return true if an error should be logged From 2443b2d6d97635bc45b76d45596192f68b197a50 Mon Sep 17 00:00:00 2001 From: Brad Hefta-Gaub Date: Wed, 14 Dec 2016 17:35:02 -0800 Subject: [PATCH 06/12] duh --- libraries/avatars/src/AvatarData.cpp | 8 +++++++- 1 file changed, 7 insertions(+), 1 deletion(-) diff --git a/libraries/avatars/src/AvatarData.cpp b/libraries/avatars/src/AvatarData.cpp index 34e9838284..f29d2d9df0 100644 --- a/libraries/avatars/src/AvatarData.cpp +++ b/libraries/avatars/src/AvatarData.cpp @@ -209,10 +209,13 @@ QByteArray AvatarData::toByteArray(bool cullSmallChanges, bool sendAll, bool sen if (compressed) { setAtBit(packetStateFlags, AVATARDATA_FLAGS_COMPRESSED); } + memcpy(destinationBuffer, &packetStateFlags, sizeof(packetStateFlags)); destinationBuffer += sizeof(packetStateFlags); if (!sendMinimum) { + //qDebug() << __FUNCTION__ << "not minimum... sending actual content!!"; + auto header = reinterpret_cast(destinationBuffer); header->position[0] = getLocalPosition().x; header->position[1] = getLocalPosition().y; @@ -514,15 +517,18 @@ int AvatarData::parseDataFromBuffer(const QByteArray& buffer) { // if this is the minimum, then it only includes the flags if (minimumSent) { + //qDebug() << __FUNCTION__ << "minimum... not expecting actual content!!"; + int numBytesRead = sizeof(packetStateFlags); _averageBytesReceived.updateAverage(numBytesRead); return numBytesRead; } + //qDebug() << __FUNCTION__ << "NOT minimum... expecting actual content!!"; QByteArray uncompressBuffer; const unsigned char* startPosition = reinterpret_cast(buffer.data()); const unsigned char* endPosition = startPosition + buffer.size(); - const unsigned char* sourceBuffer = startPosition; + const unsigned char* sourceBuffer = startPosition + sizeof(packetStateFlags); // skip the flags!! if (packetIsCompressed) { uncompressBuffer = qUncompress(buffer.right(buffer.size() - sizeof(packetStateFlags))); From c8fb467579104955fce9bcce5cad78a4112413c7 Mon Sep 17 00:00:00 2001 From: Brad Hefta-Gaub Date: Wed, 14 Dec 2016 18:17:11 -0800 Subject: [PATCH 07/12] send minimum data for out of view avatars --- assignment-client/src/avatars/AvatarMixer.cpp | 21 ++++-------- libraries/avatars/src/AvatarData.cpp | 32 +++++++++++++------ 2 files changed, 30 insertions(+), 23 deletions(-) diff --git a/assignment-client/src/avatars/AvatarMixer.cpp b/assignment-client/src/avatars/AvatarMixer.cpp index 5f031e537e..1fe71cf59e 100644 --- a/assignment-client/src/avatars/AvatarMixer.cpp +++ b/assignment-client/src/avatars/AvatarMixer.cpp @@ -276,19 +276,6 @@ void AvatarMixer::broadcastAvatarData() { } // Not close enough to ignore nodeData->removeFromRadiusIgnoringSet(otherNode->getUUID()); - - - // Also check to see if the other node is in our view - glm::vec3 otherNodeBoxScale = (otherData->getPosition() - otherData->getGlobalBoundingBoxCorner()) * 2.0f; - AABox otherNodeBox(otherData->getGlobalBoundingBoxCorner(), otherNodeBoxScale); - - if (!nodeData->otherAvatarInView(otherNodeBox)) { - //qDebug() << "Avatar out of view!"; - return false; - } else { - //qDebug() << "Avatar in view!"; - } - return true; } }, @@ -368,9 +355,15 @@ void AvatarMixer::broadcastAvatarData() { // start a new segment in the PacketList for this avatar avatarPacketList->startSegment(); + // determine if avatar is in view, to determine how much data to include... + glm::vec3 otherNodeBoxScale = (otherNodeData->getPosition() - otherNodeData->getGlobalBoundingBoxCorner()) * 2.0f; + AABox otherNodeBox(otherNodeData->getGlobalBoundingBoxCorner(), otherNodeBoxScale); + + bool sendMinimumForOutOfView = !nodeData->otherAvatarInView(otherNodeBox); + numAvatarDataBytes += avatarPacketList->write(otherNode->getUUID().toRfc4122()); numAvatarDataBytes += - avatarPacketList->write(otherAvatar.toByteArray(false, distribution(generator) < AVATAR_SEND_FULL_UPDATE_RATIO)); + avatarPacketList->write(otherAvatar.toByteArray(false, distribution(generator) < AVATAR_SEND_FULL_UPDATE_RATIO), sendMinimumForOutOfView); avatarPacketList->endSegment(); }); diff --git a/libraries/avatars/src/AvatarData.cpp b/libraries/avatars/src/AvatarData.cpp index f29d2d9df0..00db9ae943 100644 --- a/libraries/avatars/src/AvatarData.cpp +++ b/libraries/avatars/src/AvatarData.cpp @@ -213,7 +213,12 @@ QByteArray AvatarData::toByteArray(bool cullSmallChanges, bool sendAll, bool sen memcpy(destinationBuffer, &packetStateFlags, sizeof(packetStateFlags)); destinationBuffer += sizeof(packetStateFlags); - if (!sendMinimum) { + if (sendMinimum) { + memcpy(destinationBuffer, &_globalPosition, sizeof(_globalPosition)); + destinationBuffer += sizeof(_globalPosition); + qDebug() << __FUNCTION__ << "minimum... included global position!!"; + } + else { //qDebug() << __FUNCTION__ << "not minimum... sending actual content!!"; auto header = reinterpret_cast(destinationBuffer); @@ -515,14 +520,6 @@ int AvatarData::parseDataFromBuffer(const QByteArray& buffer) { bool minimumSent = oneAtBit(packetStateFlags, AVATARDATA_FLAGS_MINIMUM); bool packetIsCompressed = oneAtBit(packetStateFlags, AVATARDATA_FLAGS_COMPRESSED); - // if this is the minimum, then it only includes the flags - if (minimumSent) { - //qDebug() << __FUNCTION__ << "minimum... not expecting actual content!!"; - - int numBytesRead = sizeof(packetStateFlags); - _averageBytesReceived.updateAverage(numBytesRead); - return numBytesRead; - } //qDebug() << __FUNCTION__ << "NOT minimum... expecting actual content!!"; QByteArray uncompressBuffer; @@ -538,6 +535,20 @@ int AvatarData::parseDataFromBuffer(const QByteArray& buffer) { //qDebug() << __FUNCTION__ << "uncompressing compressed data was:" << buffer.size() << "now:" << uncompressBuffer.size(); } + // if this is the minimum, then it only includes the flags + if (minimumSent) { + //qDebug() << __FUNCTION__ << "minimum... not expecting actual content!!"; + + memcpy(&_globalPosition, sourceBuffer, sizeof(_globalPosition)); + sourceBuffer += sizeof(_globalPosition); + int numBytesRead = (sourceBuffer - startPosition) + sizeof(packetStateFlags); + _averageBytesReceived.updateAverage(numBytesRead); + + qDebug() << __FUNCTION__ << "minimum... included global position!! numBytesRead:" << numBytesRead; + + return numBytesRead; + } + quint64 now = usecTimestampNow(); PACKET_READ_CHECK(Header, sizeof(AvatarDataPacket::Header)); @@ -750,6 +761,9 @@ int AvatarData::parseDataFromBuffer(const QByteArray& buffer) { sourceBuffer = unpackFauxJoint(sourceBuffer, _controllerRightHandMatrixCache); int numBytesRead = sourceBuffer - startPosition; + if (packetIsCompressed) { + numBytesRead += sizeof(packetStateFlags); + } _averageBytesReceived.updateAverage(numBytesRead); return numBytesRead; } From 439434b300d5d488dd128001c026f520824536a0 Mon Sep 17 00:00:00 2001 From: Brad Hefta-Gaub Date: Wed, 14 Dec 2016 20:24:52 -0800 Subject: [PATCH 08/12] fixes --- assignment-client/src/avatars/AvatarMixer.cpp | 2 +- libraries/avatars/src/AvatarData.cpp | 7 +++---- 2 files changed, 4 insertions(+), 5 deletions(-) diff --git a/assignment-client/src/avatars/AvatarMixer.cpp b/assignment-client/src/avatars/AvatarMixer.cpp index 1fe71cf59e..b394673702 100644 --- a/assignment-client/src/avatars/AvatarMixer.cpp +++ b/assignment-client/src/avatars/AvatarMixer.cpp @@ -363,7 +363,7 @@ void AvatarMixer::broadcastAvatarData() { numAvatarDataBytes += avatarPacketList->write(otherNode->getUUID().toRfc4122()); numAvatarDataBytes += - avatarPacketList->write(otherAvatar.toByteArray(false, distribution(generator) < AVATAR_SEND_FULL_UPDATE_RATIO), sendMinimumForOutOfView); + avatarPacketList->write(otherAvatar.toByteArray(false, distribution(generator) < AVATAR_SEND_FULL_UPDATE_RATIO, sendMinimumForOutOfView)); avatarPacketList->endSegment(); }); diff --git a/libraries/avatars/src/AvatarData.cpp b/libraries/avatars/src/AvatarData.cpp index 00db9ae943..3f61c5f8aa 100644 --- a/libraries/avatars/src/AvatarData.cpp +++ b/libraries/avatars/src/AvatarData.cpp @@ -217,8 +217,7 @@ QByteArray AvatarData::toByteArray(bool cullSmallChanges, bool sendAll, bool sen memcpy(destinationBuffer, &_globalPosition, sizeof(_globalPosition)); destinationBuffer += sizeof(_globalPosition); qDebug() << __FUNCTION__ << "minimum... included global position!!"; - } - else { + } else { //qDebug() << __FUNCTION__ << "not minimum... sending actual content!!"; auto header = reinterpret_cast(destinationBuffer); @@ -541,10 +540,10 @@ int AvatarData::parseDataFromBuffer(const QByteArray& buffer) { memcpy(&_globalPosition, sourceBuffer, sizeof(_globalPosition)); sourceBuffer += sizeof(_globalPosition); - int numBytesRead = (sourceBuffer - startPosition) + sizeof(packetStateFlags); + int numBytesRead = (sourceBuffer - startPosition); _averageBytesReceived.updateAverage(numBytesRead); - qDebug() << __FUNCTION__ << "minimum... included global position!! numBytesRead:" << numBytesRead; + //qDebug() << __FUNCTION__ << "minimum... included global position!! numBytesRead:" << numBytesRead; return numBytesRead; } From ed19b4b592914c061d22f2ad4e42bd277fd9d24a Mon Sep 17 00:00:00 2001 From: Brad Hefta-Gaub Date: Thu, 15 Dec 2016 14:13:38 -0800 Subject: [PATCH 09/12] remove compression at the per avatar data buffer for now, since it gave no benefit --- assignment-client/src/avatars/AvatarMixer.cpp | 1 - interface/src/avatar/MyAvatar.cpp | 6 +-- interface/src/avatar/MyAvatar.h | 2 +- libraries/avatars/src/AvatarData.cpp | 46 +------------------ libraries/avatars/src/AvatarData.h | 4 +- 5 files changed, 6 insertions(+), 53 deletions(-) diff --git a/assignment-client/src/avatars/AvatarMixer.cpp b/assignment-client/src/avatars/AvatarMixer.cpp index b394673702..90a8f65c9c 100644 --- a/assignment-client/src/avatars/AvatarMixer.cpp +++ b/assignment-client/src/avatars/AvatarMixer.cpp @@ -358,7 +358,6 @@ void AvatarMixer::broadcastAvatarData() { // determine if avatar is in view, to determine how much data to include... glm::vec3 otherNodeBoxScale = (otherNodeData->getPosition() - otherNodeData->getGlobalBoundingBoxCorner()) * 2.0f; AABox otherNodeBox(otherNodeData->getGlobalBoundingBoxCorner(), otherNodeBoxScale); - bool sendMinimumForOutOfView = !nodeData->otherAvatarInView(otherNodeBox); numAvatarDataBytes += avatarPacketList->write(otherNode->getUUID().toRfc4122()); diff --git a/interface/src/avatar/MyAvatar.cpp b/interface/src/avatar/MyAvatar.cpp index 50517b706c..1b1bd9f30d 100644 --- a/interface/src/avatar/MyAvatar.cpp +++ b/interface/src/avatar/MyAvatar.cpp @@ -227,7 +227,7 @@ void MyAvatar::simulateAttachments(float deltaTime) { // don't update attachments here, do it in harvestResultsFromPhysicsSimulation() } -QByteArray MyAvatar::toByteArray(bool cullSmallChanges, bool sendAll, bool sendMinimum, bool compressed) { +QByteArray MyAvatar::toByteArray(bool cullSmallChanges, bool sendAll, bool sendMinimum) { CameraMode mode = qApp->getCamera()->getMode(); _globalPosition = getPosition(); _globalBoundingBoxCorner.x = _characterController.getCapsuleRadius(); @@ -238,12 +238,12 @@ QByteArray MyAvatar::toByteArray(bool cullSmallChanges, bool sendAll, bool sendM // fake the avatar position that is sent up to the AvatarMixer glm::vec3 oldPosition = getPosition(); setPosition(getSkeletonPosition()); - QByteArray array = AvatarData::toByteArray(cullSmallChanges, sendAll, sendMinimum, compressed); + QByteArray array = AvatarData::toByteArray(cullSmallChanges, sendAll, sendMinimum); // copy the correct position back setPosition(oldPosition); return array; } - return AvatarData::toByteArray(cullSmallChanges, sendAll, sendMinimum, compressed); + return AvatarData::toByteArray(cullSmallChanges, sendAll, sendMinimum); } void MyAvatar::centerBody() { diff --git a/interface/src/avatar/MyAvatar.h b/interface/src/avatar/MyAvatar.h index 609e93f616..14770fa14f 100644 --- a/interface/src/avatar/MyAvatar.h +++ b/interface/src/avatar/MyAvatar.h @@ -333,7 +333,7 @@ private: glm::vec3 getWorldBodyPosition() const; glm::quat getWorldBodyOrientation() const; - QByteArray toByteArray(bool cullSmallChanges, bool sendAll, bool sendMinimum, bool compressed) override; + QByteArray toByteArray(bool cullSmallChanges, bool sendAll, bool sendMinimum) override; void simulate(float deltaTime); void updateFromTrackers(float deltaTime); virtual void render(RenderArgs* renderArgs, const glm::vec3& cameraPositio) override; diff --git a/libraries/avatars/src/AvatarData.cpp b/libraries/avatars/src/AvatarData.cpp index 3f61c5f8aa..135cf48fa3 100644 --- a/libraries/avatars/src/AvatarData.cpp +++ b/libraries/avatars/src/AvatarData.cpp @@ -181,7 +181,7 @@ void AvatarData::setHandPosition(const glm::vec3& handPosition) { _handPosition = glm::inverse(getOrientation()) * (handPosition - getPosition()); } -QByteArray AvatarData::toByteArray(bool cullSmallChanges, bool sendAll, bool sendMinimum, bool compressed) { +QByteArray AvatarData::toByteArray(bool cullSmallChanges, bool sendAll, bool sendMinimum) { // TODO: DRY this up to a shared method // that can pack any type given the number of bytes // and return the number of bytes to push the pointer @@ -205,11 +205,6 @@ QByteArray AvatarData::toByteArray(bool cullSmallChanges, bool sendAll, bool sen setAtBit(packetStateFlags, AVATARDATA_FLAGS_MINIMUM); } - // do we need this, or do we just always compress it? - if (compressed) { - setAtBit(packetStateFlags, AVATARDATA_FLAGS_COMPRESSED); - } - memcpy(destinationBuffer, &packetStateFlags, sizeof(packetStateFlags)); destinationBuffer += sizeof(packetStateFlags); @@ -423,25 +418,6 @@ QByteArray AvatarData::toByteArray(bool cullSmallChanges, bool sendAll, bool sen #endif } - // NOTE: first byte of array should always be uncompressed - - if (compressed) { - static const int SKIP_PACKET_FLAGS = 1; - static const int COMPRESSION_LEVEL = 9; - QByteArray uncompressedPortion = avatarDataByteArray.mid(SKIP_PACKET_FLAGS, - (destinationBuffer - startPosition) - SKIP_PACKET_FLAGS); - - QByteArray compressedPortion = qCompress(uncompressedPortion, COMPRESSION_LEVEL); - QByteArray flagsAndCompressed; - flagsAndCompressed.append(packetStateFlags); - flagsAndCompressed.append(compressedPortion); - - //qDebug() << __FUNCTION__ << "compressing data was:" << (uncompressedPortion.size() + SKIP_PACKET_FLAGS) << "now:" << flagsAndCompressed.size(); - - return flagsAndCompressed; - } - - // entire buffer is uncompressed return avatarDataByteArray.left(destinationBuffer - startPosition); } @@ -517,34 +493,17 @@ int AvatarData::parseDataFromBuffer(const QByteArray& buffer) { uint8_t packetStateFlags = buffer.at(0); bool minimumSent = oneAtBit(packetStateFlags, AVATARDATA_FLAGS_MINIMUM); - bool packetIsCompressed = oneAtBit(packetStateFlags, AVATARDATA_FLAGS_COMPRESSED); - //qDebug() << __FUNCTION__ << "NOT minimum... expecting actual content!!"; - - QByteArray uncompressBuffer; const unsigned char* startPosition = reinterpret_cast(buffer.data()); const unsigned char* endPosition = startPosition + buffer.size(); const unsigned char* sourceBuffer = startPosition + sizeof(packetStateFlags); // skip the flags!! - if (packetIsCompressed) { - uncompressBuffer = qUncompress(buffer.right(buffer.size() - sizeof(packetStateFlags))); - startPosition = reinterpret_cast(uncompressBuffer.data()); - endPosition = startPosition + uncompressBuffer.size(); - sourceBuffer = startPosition; - //qDebug() << __FUNCTION__ << "uncompressing compressed data was:" << buffer.size() << "now:" << uncompressBuffer.size(); - } - // if this is the minimum, then it only includes the flags if (minimumSent) { - //qDebug() << __FUNCTION__ << "minimum... not expecting actual content!!"; - memcpy(&_globalPosition, sourceBuffer, sizeof(_globalPosition)); sourceBuffer += sizeof(_globalPosition); int numBytesRead = (sourceBuffer - startPosition); _averageBytesReceived.updateAverage(numBytesRead); - - //qDebug() << __FUNCTION__ << "minimum... included global position!! numBytesRead:" << numBytesRead; - return numBytesRead; } @@ -760,9 +719,6 @@ int AvatarData::parseDataFromBuffer(const QByteArray& buffer) { sourceBuffer = unpackFauxJoint(sourceBuffer, _controllerRightHandMatrixCache); int numBytesRead = sourceBuffer - startPosition; - if (packetIsCompressed) { - numBytesRead += sizeof(packetStateFlags); - } _averageBytesReceived.updateAverage(numBytesRead); return numBytesRead; } diff --git a/libraries/avatars/src/AvatarData.h b/libraries/avatars/src/AvatarData.h index c985559913..dd23d5c86d 100644 --- a/libraries/avatars/src/AvatarData.h +++ b/libraries/avatars/src/AvatarData.h @@ -109,8 +109,6 @@ const char IS_FINGER_POINTING_FLAG = 4; // AvatarData state flags - we store the details about the packet encoding in the first byte, // before the "header" structure const char AVATARDATA_FLAGS_MINIMUM = 0; -const char AVATARDATA_FLAGS_COMPRESSED = 1; - static const float MAX_AVATAR_SCALE = 1000.0f; @@ -208,7 +206,7 @@ public: glm::vec3 getHandPosition() const; void setHandPosition(const glm::vec3& handPosition); - virtual QByteArray toByteArray(bool cullSmallChanges, bool sendAll, bool sendMinimum = false, bool compressed = false); + virtual QByteArray toByteArray(bool cullSmallChanges, bool sendAll, bool sendMinimum = false); virtual void doneEncoding(bool cullSmallChanges); /// \return true if an error should be logged From 6d7d3551d9cfc6cddb805e5576ed1ce0c6eb948f Mon Sep 17 00:00:00 2001 From: Brad Hefta-Gaub Date: Thu, 15 Dec 2016 18:32:11 -0800 Subject: [PATCH 10/12] added some stats to the mixer fixed debug logging, improved timer precision --- assignment-client/src/avatars/AvatarMixer.cpp | 26 +++++++++++++++++++ assignment-client/src/avatars/AvatarMixer.h | 6 +++++ .../src/avatars/AvatarMixerClientData.cpp | 2 ++ .../src/avatars/AvatarMixerClientData.h | 7 +++++ libraries/avatars/src/AvatarData.cpp | 3 --- 5 files changed, 41 insertions(+), 3 deletions(-) diff --git a/assignment-client/src/avatars/AvatarMixer.cpp b/assignment-client/src/avatars/AvatarMixer.cpp index 90a8f65c9c..53f7695095 100644 --- a/assignment-client/src/avatars/AvatarMixer.cpp +++ b/assignment-client/src/avatars/AvatarMixer.cpp @@ -72,6 +72,8 @@ const float IDENTITY_SEND_PROBABILITY = 1.0f / 187.0f; // 1) use the view frustum to cull those avatars that are out of view. Since avatar data doesn't need to be present // if the avatar is not in view or in the keyhole. void AvatarMixer::broadcastAvatarData() { + _broadcastRate.increment(); + int idleTime = AVATAR_DATA_SEND_INTERVAL_MSECS; if (_lastFrameTimestamp.time_since_epoch().count() > 0) { @@ -160,6 +162,7 @@ void AvatarMixer::broadcastAvatarData() { return; } ++_sumListeners; + nodeData->resetInViewStats(); AvatarData& avatar = nodeData->getAvatar(); glm::vec3 myPosition = avatar.getClientGlobalPosition(); @@ -360,6 +363,12 @@ void AvatarMixer::broadcastAvatarData() { AABox otherNodeBox(otherNodeData->getGlobalBoundingBoxCorner(), otherNodeBoxScale); bool sendMinimumForOutOfView = !nodeData->otherAvatarInView(otherNodeBox); + if (sendMinimumForOutOfView) { + nodeData->incrementAvatarOutOfView(); + } else { + nodeData->incrementAvatarInView(); + } + numAvatarDataBytes += avatarPacketList->write(otherNode->getUUID().toRfc4122()); numAvatarDataBytes += avatarPacketList->write(otherAvatar.toByteArray(false, distribution(generator) < AVATAR_SEND_FULL_UPDATE_RATIO, sendMinimumForOutOfView)); @@ -391,6 +400,9 @@ void AvatarMixer::broadcastAvatarData() { // We're done encoding this version of the otherAvatars. Update their "lastSent" joint-states so // that we can notice differences, next time around. + // + // FIXME - this seems suspicious, the code seems to consider all avatars, but not all avatars will + // have had their joints sent, so actually we should consider the time since they actually were sent???? nodeList->eachMatchingNode( [&](const SharedNodePointer& otherNode)->bool { if (!otherNode->getLinkedData()) { @@ -415,6 +427,18 @@ void AvatarMixer::broadcastAvatarData() { }); _lastFrameTimestamp = p_high_resolution_clock::now(); + +#ifdef WANT_DEBUG + auto sinceLastDebug = p_high_resolution_clock::now() - _lastDebugMessage; + auto sinceLastDebugUsecs = std::chrono::duration_cast(sinceLastDebug).count(); + quint64 DEBUG_INTERVAL = USECS_PER_SECOND * 5; + + if (sinceLastDebugUsecs > DEBUG_INTERVAL) { + qDebug() << "broadcast rate:" << _broadcastRate.rate() << "hz"; + _lastDebugMessage = p_high_resolution_clock::now(); + } +#endif + } void AvatarMixer::nodeKilled(SharedNodePointer killedNode) { @@ -511,6 +535,7 @@ void AvatarMixer::sendStatsPacket() { statsObject["trailing_sleep_percentage"] = _trailingSleepRatio * 100; statsObject["performance_throttling_ratio"] = _performanceThrottlingRatio; + statsObject["broadcast_loop_rate"] = _broadcastRate.rate(); QJsonObject avatarsObject; @@ -563,6 +588,7 @@ void AvatarMixer::run() { // setup the timer that will be fired on the broadcast thread _broadcastTimer = new QTimer; + _broadcastTimer->setTimerType(Qt::PreciseTimer); _broadcastTimer->setInterval(AVATAR_DATA_SEND_INTERVAL_MSECS); _broadcastTimer->moveToThread(&_broadcastThread); diff --git a/assignment-client/src/avatars/AvatarMixer.h b/assignment-client/src/avatars/AvatarMixer.h index ba74b43308..9ac121d71c 100644 --- a/assignment-client/src/avatars/AvatarMixer.h +++ b/assignment-client/src/avatars/AvatarMixer.h @@ -15,6 +15,7 @@ #ifndef hifi_AvatarMixer_h #define hifi_AvatarMixer_h +#include #include #include @@ -65,6 +66,11 @@ private: float _domainMaximumScale { MAX_AVATAR_SCALE }; QTimer* _broadcastTimer = nullptr; + + RateCounter<> _broadcastRate; + p_high_resolution_clock::time_point _lastDebugMessage; + + }; #endif // hifi_AvatarMixer_h diff --git a/assignment-client/src/avatars/AvatarMixerClientData.cpp b/assignment-client/src/avatars/AvatarMixerClientData.cpp index 63a1b108b8..162ef2c462 100644 --- a/assignment-client/src/avatars/AvatarMixerClientData.cpp +++ b/assignment-client/src/avatars/AvatarMixerClientData.cpp @@ -84,4 +84,6 @@ void AvatarMixerClientData::loadJSONStats(QJsonObject& jsonObject) const { jsonObject[INBOUND_AVATAR_DATA_STATS_KEY] = _avatar->getAverageBytesReceivedPerSecond() / (float) BYTES_PER_KILOBIT; jsonObject["av_data_receive_rate"] = _avatar->getReceiveRate(); + jsonObject["recent_other_av_in_view"] = _recentOtherAvatarsInView; + jsonObject["recent_other_av_out_of_view"] = _recentOtherAvatarsOutOfView; } diff --git a/assignment-client/src/avatars/AvatarMixerClientData.h b/assignment-client/src/avatars/AvatarMixerClientData.h index b7b7b16ea1..e6a91e28e2 100644 --- a/assignment-client/src/avatars/AvatarMixerClientData.h +++ b/assignment-client/src/avatars/AvatarMixerClientData.h @@ -93,6 +93,10 @@ public: bool otherAvatarInView(const AABox& otherAvatarBox); bool otherAvatarInView(const glm::vec3& otherAvatar); + void resetInViewStats() { _recentOtherAvatarsInView = _recentOtherAvatarsOutOfView = 0; } + void incrementAvatarInView() { _recentOtherAvatarsInView++; } + void incrementAvatarOutOfView() { _recentOtherAvatarsOutOfView++; } + private: AvatarSharedPointer _avatar { new AvatarData() }; @@ -116,6 +120,9 @@ private: std::unordered_set _radiusIgnoredOthers; ViewFrustum _currentViewFrustum; bool _currentViewFrustumIsValid { false }; + + int _recentOtherAvatarsInView { 0 }; + int _recentOtherAvatarsOutOfView { 0 }; }; #endif // hifi_AvatarMixerClientData_h diff --git a/libraries/avatars/src/AvatarData.cpp b/libraries/avatars/src/AvatarData.cpp index 135cf48fa3..ec8d589c23 100644 --- a/libraries/avatars/src/AvatarData.cpp +++ b/libraries/avatars/src/AvatarData.cpp @@ -211,10 +211,7 @@ QByteArray AvatarData::toByteArray(bool cullSmallChanges, bool sendAll, bool sen if (sendMinimum) { memcpy(destinationBuffer, &_globalPosition, sizeof(_globalPosition)); destinationBuffer += sizeof(_globalPosition); - qDebug() << __FUNCTION__ << "minimum... included global position!!"; } else { - //qDebug() << __FUNCTION__ << "not minimum... sending actual content!!"; - auto header = reinterpret_cast(destinationBuffer); header->position[0] = getLocalPosition().x; header->position[1] = getLocalPosition().y; From 5e65e3991cc5c7fc17e81137a40ff7a35cc0d280 Mon Sep 17 00:00:00 2001 From: Brad Hefta-Gaub Date: Tue, 20 Dec 2016 21:03:10 -0800 Subject: [PATCH 11/12] CR feedback --- assignment-client/src/Agent.cpp | 3 ++- assignment-client/src/avatars/AvatarMixer.cpp | 10 ++++++---- .../src/avatars/AvatarMixerClientData.cpp | 4 ---- .../src/avatars/AvatarMixerClientData.h | 1 - interface/src/avatar/MyAvatar.cpp | 6 +++--- interface/src/avatar/MyAvatar.h | 2 +- libraries/avatars/src/AvatarData.cpp | 12 ++++++++---- libraries/avatars/src/AvatarData.h | 9 ++++++++- 8 files changed, 28 insertions(+), 19 deletions(-) diff --git a/assignment-client/src/Agent.cpp b/assignment-client/src/Agent.cpp index dc6b2427f2..e79085244f 100644 --- a/assignment-client/src/Agent.cpp +++ b/assignment-client/src/Agent.cpp @@ -499,7 +499,8 @@ void Agent::processAgentAvatar() { if (!_scriptEngine->isFinished() && _isAvatar) { auto scriptedAvatar = DependencyManager::get(); - QByteArray avatarByteArray = scriptedAvatar->toByteArray(true, randFloat() < AVATAR_SEND_FULL_UPDATE_RATIO); + QByteArray avatarByteArray = scriptedAvatar->toByteArray((randFloat() < AVATAR_SEND_FULL_UPDATE_RATIO) + ? AvatarData::SendAllData : AvatarData::CullSmallData); scriptedAvatar->doneEncoding(true); static AvatarDataSequenceNumber sequenceNumber = 0; diff --git a/assignment-client/src/avatars/AvatarMixer.cpp b/assignment-client/src/avatars/AvatarMixer.cpp index 540526524e..d8d0b10fea 100644 --- a/assignment-client/src/avatars/AvatarMixer.cpp +++ b/assignment-client/src/avatars/AvatarMixer.cpp @@ -386,17 +386,19 @@ void AvatarMixer::broadcastAvatarData() { // determine if avatar is in view, to determine how much data to include... glm::vec3 otherNodeBoxScale = (otherNodeData->getPosition() - otherNodeData->getGlobalBoundingBoxCorner()) * 2.0f; AABox otherNodeBox(otherNodeData->getGlobalBoundingBoxCorner(), otherNodeBoxScale); - bool sendMinimumForOutOfView = !nodeData->otherAvatarInView(otherNodeBox); - if (sendMinimumForOutOfView) { + AvatarData::AvatarDataDetail detail; + if (!nodeData->otherAvatarInView(otherNodeBox)) { + detail = AvatarData::MinimumData; nodeData->incrementAvatarOutOfView(); } else { + detail = distribution(generator) < AVATAR_SEND_FULL_UPDATE_RATIO + ? AvatarData::SendAllData : AvatarData::IncludeSmallData; nodeData->incrementAvatarInView(); } numAvatarDataBytes += avatarPacketList->write(otherNode->getUUID().toRfc4122()); - numAvatarDataBytes += - avatarPacketList->write(otherAvatar.toByteArray(false, distribution(generator) < AVATAR_SEND_FULL_UPDATE_RATIO, sendMinimumForOutOfView)); + numAvatarDataBytes += avatarPacketList->write(otherAvatar.toByteArray(detail)); avatarPacketList->endSegment(); }); diff --git a/assignment-client/src/avatars/AvatarMixerClientData.cpp b/assignment-client/src/avatars/AvatarMixerClientData.cpp index 162ef2c462..e6e2fc3849 100644 --- a/assignment-client/src/avatars/AvatarMixerClientData.cpp +++ b/assignment-client/src/avatars/AvatarMixerClientData.cpp @@ -62,10 +62,6 @@ void AvatarMixerClientData::readViewFrustumPacket(const QByteArray& message) { _currentViewFrustum.fromByteArray(message); } -bool AvatarMixerClientData::otherAvatarInView(const glm::vec3& otherAvatar) { - return !_currentViewFrustumIsValid || _currentViewFrustum.pointIntersectsFrustum(otherAvatar); -} - bool AvatarMixerClientData::otherAvatarInView(const AABox& otherAvatarBox) { return !_currentViewFrustumIsValid || _currentViewFrustum.boxIntersectsKeyhole(otherAvatarBox); } diff --git a/assignment-client/src/avatars/AvatarMixerClientData.h b/assignment-client/src/avatars/AvatarMixerClientData.h index 6b916750b4..b894932ba7 100644 --- a/assignment-client/src/avatars/AvatarMixerClientData.h +++ b/assignment-client/src/avatars/AvatarMixerClientData.h @@ -95,7 +95,6 @@ public: void readViewFrustumPacket(const QByteArray& message); bool otherAvatarInView(const AABox& otherAvatarBox); - bool otherAvatarInView(const glm::vec3& otherAvatar); void resetInViewStats() { _recentOtherAvatarsInView = _recentOtherAvatarsOutOfView = 0; } void incrementAvatarInView() { _recentOtherAvatarsInView++; } diff --git a/interface/src/avatar/MyAvatar.cpp b/interface/src/avatar/MyAvatar.cpp index 9547d769fa..eebcee8e4c 100644 --- a/interface/src/avatar/MyAvatar.cpp +++ b/interface/src/avatar/MyAvatar.cpp @@ -226,7 +226,7 @@ void MyAvatar::simulateAttachments(float deltaTime) { // don't update attachments here, do it in harvestResultsFromPhysicsSimulation() } -QByteArray MyAvatar::toByteArray(bool cullSmallChanges, bool sendAll, bool sendMinimum) { +QByteArray MyAvatar::toByteArray(AvatarDataDetail dataDetail) { CameraMode mode = qApp->getCamera()->getMode(); _globalPosition = getPosition(); _globalBoundingBoxCorner.x = _characterController.getCapsuleRadius(); @@ -237,12 +237,12 @@ QByteArray MyAvatar::toByteArray(bool cullSmallChanges, bool sendAll, bool sendM // fake the avatar position that is sent up to the AvatarMixer glm::vec3 oldPosition = getPosition(); setPosition(getSkeletonPosition()); - QByteArray array = AvatarData::toByteArray(cullSmallChanges, sendAll, sendMinimum); + QByteArray array = AvatarData::toByteArray(dataDetail); // copy the correct position back setPosition(oldPosition); return array; } - return AvatarData::toByteArray(cullSmallChanges, sendAll, sendMinimum); + return AvatarData::toByteArray(dataDetail); } void MyAvatar::centerBody() { diff --git a/interface/src/avatar/MyAvatar.h b/interface/src/avatar/MyAvatar.h index 14770fa14f..0e5ce0fe7b 100644 --- a/interface/src/avatar/MyAvatar.h +++ b/interface/src/avatar/MyAvatar.h @@ -333,7 +333,7 @@ private: glm::vec3 getWorldBodyPosition() const; glm::quat getWorldBodyOrientation() const; - QByteArray toByteArray(bool cullSmallChanges, bool sendAll, bool sendMinimum) override; + QByteArray toByteArray(AvatarDataDetail dataDetail) override; void simulate(float deltaTime); void updateFromTrackers(float deltaTime); virtual void render(RenderArgs* renderArgs, const glm::vec3& cameraPositio) override; diff --git a/libraries/avatars/src/AvatarData.cpp b/libraries/avatars/src/AvatarData.cpp index 73df7a2f9e..d77f70fbe9 100644 --- a/libraries/avatars/src/AvatarData.cpp +++ b/libraries/avatars/src/AvatarData.cpp @@ -181,7 +181,11 @@ void AvatarData::setHandPosition(const glm::vec3& handPosition) { _handPosition = glm::inverse(getOrientation()) * (handPosition - getPosition()); } -QByteArray AvatarData::toByteArray(bool cullSmallChanges, bool sendAll, bool sendMinimum) { + +QByteArray AvatarData::toByteArray(AvatarDataDetail dataDetail) { + bool cullSmallChanges = (dataDetail == CullSmallData); + bool sendAll = (dataDetail == SendAllData); + bool sendMinimum = (dataDetail == MinimumData); // TODO: DRY this up to a shared method // that can pack any type given the number of bytes // and return the number of bytes to push the pointer @@ -1208,9 +1212,9 @@ void AvatarData::sendAvatarDataPacket() { // about 2% of the time, we send a full update (meaning, we transmit all the joint data), even if nothing has changed. // this is to guard against a joint moving once, the packet getting lost, and the joint never moving again. - bool sendFullUpdate = randFloat() < AVATAR_SEND_FULL_UPDATE_RATIO; - QByteArray avatarByteArray = toByteArray(true, sendFullUpdate); - doneEncoding(true); + QByteArray avatarByteArray = toByteArray((randFloat() < AVATAR_SEND_FULL_UPDATE_RATIO) ? SendAllData : CullSmallData); + + doneEncoding(true); // FIXME - doneEncoding() takes a bool for culling small changes, that's janky! static AvatarDataSequenceNumber sequenceNumber = 0; diff --git a/libraries/avatars/src/AvatarData.h b/libraries/avatars/src/AvatarData.h index 1ad6d32810..db06d52092 100644 --- a/libraries/avatars/src/AvatarData.h +++ b/libraries/avatars/src/AvatarData.h @@ -209,7 +209,14 @@ public: glm::vec3 getHandPosition() const; void setHandPosition(const glm::vec3& handPosition); - virtual QByteArray toByteArray(bool cullSmallChanges, bool sendAll, bool sendMinimum = false); + typedef enum { + MinimumData, + CullSmallData, + IncludeSmallData, + SendAllData + } AvatarDataDetail; + + virtual QByteArray toByteArray(AvatarDataDetail dataDetail); virtual void doneEncoding(bool cullSmallChanges); /// \return true if an error should be logged From 2ba4fee44bd2afbd14812863ec672319683e032c Mon Sep 17 00:00:00 2001 From: Brad Hefta-Gaub Date: Wed, 21 Dec 2016 08:26:20 -0800 Subject: [PATCH 12/12] CR feedback --- assignment-client/src/avatars/AvatarMixerClientData.cpp | 4 +--- assignment-client/src/avatars/AvatarMixerClientData.h | 3 +-- 2 files changed, 2 insertions(+), 5 deletions(-) diff --git a/assignment-client/src/avatars/AvatarMixerClientData.cpp b/assignment-client/src/avatars/AvatarMixerClientData.cpp index e6e2fc3849..c65703b8e6 100644 --- a/assignment-client/src/avatars/AvatarMixerClientData.cpp +++ b/assignment-client/src/avatars/AvatarMixerClientData.cpp @@ -58,15 +58,13 @@ void AvatarMixerClientData::ignoreOther(SharedNodePointer self, SharedNodePointe } void AvatarMixerClientData::readViewFrustumPacket(const QByteArray& message) { - _currentViewFrustumIsValid = true; _currentViewFrustum.fromByteArray(message); } bool AvatarMixerClientData::otherAvatarInView(const AABox& otherAvatarBox) { - return !_currentViewFrustumIsValid || _currentViewFrustum.boxIntersectsKeyhole(otherAvatarBox); + return _currentViewFrustum.boxIntersectsKeyhole(otherAvatarBox); } - void AvatarMixerClientData::loadJSONStats(QJsonObject& jsonObject) const { jsonObject["display_name"] = _avatar->getDisplayName(); jsonObject["full_rate_distance"] = _fullRateDistance; diff --git a/assignment-client/src/avatars/AvatarMixerClientData.h b/assignment-client/src/avatars/AvatarMixerClientData.h index b894932ba7..78a30d8206 100644 --- a/assignment-client/src/avatars/AvatarMixerClientData.h +++ b/assignment-client/src/avatars/AvatarMixerClientData.h @@ -35,7 +35,7 @@ const QString INBOUND_AVATAR_DATA_STATS_KEY = "inbound_av_data_kbps"; class AvatarMixerClientData : public NodeData { Q_OBJECT public: - AvatarMixerClientData(const QUuid& nodeID = QUuid()) : NodeData(nodeID) {} + AvatarMixerClientData(const QUuid& nodeID = QUuid()) : NodeData(nodeID) { _currentViewFrustum.invalidate(); } virtual ~AvatarMixerClientData() {} using HRCTime = p_high_resolution_clock::time_point; @@ -125,7 +125,6 @@ private: SimpleMovingAverage _avgOtherAvatarDataRate; std::unordered_set _radiusIgnoredOthers; ViewFrustum _currentViewFrustum; - bool _currentViewFrustumIsValid { false }; int _recentOtherAvatarsInView { 0 }; int _recentOtherAvatarsOutOfView { 0 };