From d0e8612a65c7d2cf0ab8cc88c4859e21a56a611e Mon Sep 17 00:00:00 2001 From: Atlante45 Date: Mon, 12 Jun 2017 15:25:22 -0700 Subject: [PATCH 01/46] First pass at AvatarMixer replication --- assignment-client/src/avatars/AvatarMixer.cpp | 75 +++++++++++++++++++ assignment-client/src/avatars/AvatarMixer.h | 3 + .../src/avatars/AvatarMixerClientData.cpp | 4 + libraries/networking/src/LimitedNodeList.cpp | 4 + .../networking/src/udt/PacketHeaders.cpp | 5 +- libraries/networking/src/udt/PacketHeaders.h | 5 +- 6 files changed, 93 insertions(+), 3 deletions(-) diff --git a/assignment-client/src/avatars/AvatarMixer.cpp b/assignment-client/src/avatars/AvatarMixer.cpp index 870149f1bc..f499a787cb 100644 --- a/assignment-client/src/avatars/AvatarMixer.cpp +++ b/assignment-client/src/avatars/AvatarMixer.cpp @@ -54,15 +54,79 @@ AvatarMixer::AvatarMixer(ReceivedMessage& message) : packetReceiver.registerListener(PacketType::RadiusIgnoreRequest, this, "handleRadiusIgnoreRequestPacket"); packetReceiver.registerListener(PacketType::RequestsDomainListData, this, "handleRequestsDomainListDataPacket"); + packetReceiver.registerListenerForTypes({ + PacketType::ReplicatedAvatarIdentity, + PacketType::ReplicatedAvatarData, + PacketType::ReplicatedKillAvatar + }, this, "handleReplicatedPackets"); + auto nodeList = DependencyManager::get(); connect(nodeList.data(), &NodeList::packetVersionMismatch, this, &AvatarMixer::handlePacketVersionMismatch); } +void AvatarMixer::handleReplicatedPackets(QSharedPointer message) { + auto nodeList = DependencyManager::get(); + auto nodeID = QUuid::fromRfc4122(message->readWithoutCopy(NUM_BYTES_RFC4122_UUID)); + + auto replicatedNode = nodeList->nodeWithUUID(nodeID); + if (!replicatedNode) { + QMetaObject::invokeMethod(nodeList.data(), "addOrUpdateNode", Qt::BlockingQueuedConnection, + Q_RETURN_ARG(SharedNodePointer, replicatedNode), + Q_ARG(QUuid, nodeID), Q_ARG(NodeType_t, NodeType::Agent), + Q_ARG(HifiSockAddr, message->getSenderSockAddr()), + Q_ARG(HifiSockAddr, message->getSenderSockAddr())); + } + + replicatedNode->setLastHeardMicrostamp(usecTimestampNow()); + replicatedNode->setIsUpstream(true); + + switch (message->getType()) { + case PacketType::AvatarData: + queueIncomingPacket(message, replicatedNode); + break; + case PacketType::AvatarIdentity: + handleAvatarIdentityPacket(message, replicatedNode); + break; + case PacketType::KillAvatar: + handleKillAvatarPacket(message); + break + } + +} + +void AvatarMixer::replicatePacket(ReceivedMessage& message) { + PacketType replicatedType; + if (message.getType() == PacketType::AvatarData) { + replicatedType = PacketType::ReplicatedAvatarData; + } else if (message.getType() == PacketType::AvatarIdentity) { + replicatedType = PacketType::ReplicatedAvatarIdentity; + } else if (message.getType() == PacketType::KillAvatar) { + replicatedType = PacketType::ReplicatedKillAvatar; + } else { + Q_UNREACHABLE(); + return; + } + + auto nodeList = DependencyManager::get(); + nodeList->eachMatchingNode([&](const SharedNodePointer& node) { + return node->getType() == NodeType::ReplicantAvatarMixer; + }, [&](const SharedNodePointer& node) { + // construct an NLPacket to send to the replicant that has the contents of the received packet + auto packet = NLPacket::create(replicatedType, message.getSize() + NUM_BYTES_RFC4122_UUID); + packet->write(message.getSourceID().toRfc4122()); + packet->write(message.getMessage()); + + nodeList->sendPacket(std::move(packet), *node); + }); +} + void AvatarMixer::queueIncomingPacket(QSharedPointer message, SharedNodePointer node) { auto start = usecTimestampNow(); getOrCreateClientData(node)->queuePacket(message, node); auto end = usecTimestampNow(); _queueIncomingPacketElapsedTime += (end - start); + + replicatePacket(*message); } @@ -397,6 +461,13 @@ void AvatarMixer::handleAvatarIdentityPacket(QSharedPointer mes if (nodeData != nullptr) { AvatarData& avatar = nodeData->getAvatar(); + auto data = message->getMessage(); + + if (message->getType() == PacketType::ReplicatedAvatarData) { + data = data.mid(NUM_BYTES_RFC4122_UUID); + } + + // parse the identity packet and update the change timestamp if appropriate AvatarData::Identity identity; AvatarData::parseAvatarIdentityPacket(message->getMessage(), identity); @@ -414,6 +485,8 @@ void AvatarMixer::handleAvatarIdentityPacket(QSharedPointer mes } auto end = usecTimestampNow(); _handleAvatarIdentityPacketElapsedTime += (end - start); + + replicatePacket(*message); } void AvatarMixer::handleKillAvatarPacket(QSharedPointer message) { @@ -421,6 +494,8 @@ void AvatarMixer::handleKillAvatarPacket(QSharedPointer message DependencyManager::get()->processKillNode(*message); auto end = usecTimestampNow(); _handleKillAvatarPacketElapsedTime += (end - start); + + replicatePacket(*message); } void AvatarMixer::handleNodeIgnoreRequestPacket(QSharedPointer message, SharedNodePointer senderNode) { diff --git a/assignment-client/src/avatars/AvatarMixer.h b/assignment-client/src/avatars/AvatarMixer.h index 1925ec1ebd..ecbed4dca7 100644 --- a/assignment-client/src/avatars/AvatarMixer.h +++ b/assignment-client/src/avatars/AvatarMixer.h @@ -46,6 +46,7 @@ private slots: void handleNodeIgnoreRequestPacket(QSharedPointer message, SharedNodePointer senderNode); void handleRadiusIgnoreRequestPacket(QSharedPointer packet, SharedNodePointer sendingNode); void handleRequestsDomainListDataPacket(QSharedPointer message, SharedNodePointer senderNode); + void handleReplicatedPackets(QSharedPointer message); void domainSettingsRequestComplete(); void handlePacketVersionMismatch(PacketType type, const HifiSockAddr& senderSockAddr, const QUuid& senderUUID); void start(); @@ -61,6 +62,8 @@ private: void manageDisplayName(const SharedNodePointer& node); + void replicatePacket(ReceivedMessage& message); + p_high_resolution_clock::time_point _lastFrameTimestamp; // FIXME - new throttling - use these values somehow diff --git a/assignment-client/src/avatars/AvatarMixerClientData.cpp b/assignment-client/src/avatars/AvatarMixerClientData.cpp index 4d80bc7d17..41717adeb5 100644 --- a/assignment-client/src/avatars/AvatarMixerClientData.cpp +++ b/assignment-client/src/avatars/AvatarMixerClientData.cpp @@ -62,6 +62,10 @@ int AvatarMixerClientData::parseData(ReceivedMessage& message) { // pull the sequence number from the data first uint16_t sequenceNumber; + if (message.getType() == PacketType::ReplicatedAvatarData) { + message.seek(NUM_BYTES_RFC4122_UUID); + } + message.readPrimitive(&sequenceNumber); if (sequenceNumber < _lastReceivedSequenceNumber && _lastReceivedSequenceNumber != UINT16_MAX) { diff --git a/libraries/networking/src/LimitedNodeList.cpp b/libraries/networking/src/LimitedNodeList.cpp index 0494efc7b4..f28a972720 100644 --- a/libraries/networking/src/LimitedNodeList.cpp +++ b/libraries/networking/src/LimitedNodeList.cpp @@ -549,6 +549,10 @@ bool LimitedNodeList::killNodeWithUUID(const QUuid& nodeUUID) { } void LimitedNodeList::processKillNode(ReceivedMessage& message) { + if (message.getType() == PacketType::ReplicatedAvatarData) { + message.seek(NUM_BYTES_RFC4122_UUID); + } + // read the node id QUuid nodeUUID = QUuid::fromRfc4122(message.readWithoutCopy(NUM_BYTES_RFC4122_UUID)); diff --git a/libraries/networking/src/udt/PacketHeaders.cpp b/libraries/networking/src/udt/PacketHeaders.cpp index f2f0062861..9f213d29ff 100644 --- a/libraries/networking/src/udt/PacketHeaders.cpp +++ b/libraries/networking/src/udt/PacketHeaders.cpp @@ -41,7 +41,8 @@ const QSet NON_SOURCED_PACKETS = QSet() << PacketType::ICEServerHeartbeatDenied << PacketType::AssignmentClientStatus << PacketType::StopNode << PacketType::DomainServerRemovedNode << PacketType::UsernameFromIDReply << PacketType::OctreeFileReplacement << PacketType::ReplicatedMicrophoneAudioNoEcho << PacketType::ReplicatedMicrophoneAudioWithEcho - << PacketType::ReplicatedInjectAudio << PacketType::ReplicatedSilentAudioFrame; + << PacketType::ReplicatedInjectAudio << PacketType::ReplicatedSilentAudioFrame + << PacketType::ReplicatedAvatarIdentity << PacketType::ReplicatedAvatarData << PacketType::ReplicatedKillAvatar; PacketVersion versionForPacketType(PacketType packetType) { switch (packetType) { @@ -121,7 +122,7 @@ static void ensureProtocolVersionsSignature() { std::call_once(once, [&] { QByteArray buffer; QDataStream stream(&buffer, QIODevice::WriteOnly); - uint8_t numberOfProtocols = static_cast(PacketType::LAST_PACKET_TYPE) + 1; + uint8_t numberOfProtocols = static_cast(PacketType::NUM_PACKET_TYPE); stream << numberOfProtocols; for (uint8_t packetType = 0; packetType < numberOfProtocols; packetType++) { uint8_t packetTypeVersion = static_cast(versionForPacketType(static_cast(packetType))); diff --git a/libraries/networking/src/udt/PacketHeaders.h b/libraries/networking/src/udt/PacketHeaders.h index 448ae812ff..137108ef75 100644 --- a/libraries/networking/src/udt/PacketHeaders.h +++ b/libraries/networking/src/udt/PacketHeaders.h @@ -119,8 +119,11 @@ public: ReplicatedMicrophoneAudioWithEcho, ReplicatedInjectAudio, ReplicatedSilentAudioFrame, - LAST_PACKET_TYPE = ReplicatedSilentAudioFrame, + ReplicatedAvatarIdentity, + ReplicatedAvatarData, + ReplicatedKillAvatar, + NUM_PACKET_TYPE }; }; From a0d107c72c961eb1c5d275cc391afdb08da55237 Mon Sep 17 00:00:00 2001 From: Atlante45 Date: Mon, 12 Jun 2017 15:45:12 -0700 Subject: [PATCH 02/46] Only replicate replicated nodes --- assignment-client/src/avatars/AvatarMixer.cpp | 69 +++++++++++-------- assignment-client/src/avatars/AvatarMixer.h | 4 +- 2 files changed, 42 insertions(+), 31 deletions(-) diff --git a/assignment-client/src/avatars/AvatarMixer.cpp b/assignment-client/src/avatars/AvatarMixer.cpp index f499a787cb..2b074f71e6 100644 --- a/assignment-client/src/avatars/AvatarMixer.cpp +++ b/assignment-client/src/avatars/AvatarMixer.cpp @@ -88,36 +88,47 @@ void AvatarMixer::handleReplicatedPackets(QSharedPointer messag handleAvatarIdentityPacket(message, replicatedNode); break; case PacketType::KillAvatar: - handleKillAvatarPacket(message); - break + handleKillAvatarPacket(message, replicatedNode); + break; + default: + // Do nothing + break; } } -void AvatarMixer::replicatePacket(ReceivedMessage& message) { - PacketType replicatedType; - if (message.getType() == PacketType::AvatarData) { - replicatedType = PacketType::ReplicatedAvatarData; - } else if (message.getType() == PacketType::AvatarIdentity) { - replicatedType = PacketType::ReplicatedAvatarIdentity; - } else if (message.getType() == PacketType::KillAvatar) { - replicatedType = PacketType::ReplicatedKillAvatar; - } else { - Q_UNREACHABLE(); - return; +void AvatarMixer::optionallyReplicatePacket(ReceivedMessage& message, const Node& node) { + // first, make sure that this is a packet from a node we are supposed to replicate + if (node.isReplicated()) { + // now make sure it's a packet type that we want to replicate + PacketType replicatedType; + switch (message.getType()) { + case PacketType::AvatarData: + replicatedType = PacketType::ReplicatedAvatarData; + break; + case PacketType::AvatarIdentity: + replicatedType = PacketType::ReplicatedAvatarIdentity; + break; + case PacketType::KillAvatar: + replicatedType = PacketType::ReplicatedKillAvatar; + break; + default: + Q_UNREACHABLE(); + return; + } + + auto nodeList = DependencyManager::get(); + nodeList->eachMatchingNode([&](const SharedNodePointer& node) { + return node->getType() == NodeType::ReplicantAvatarMixer; + }, [&](const SharedNodePointer& node) { + // construct an NLPacket to send to the replicant that has the contents of the received packet + auto packet = NLPacket::create(replicatedType, message.getSize() + NUM_BYTES_RFC4122_UUID); + packet->write(message.getSourceID().toRfc4122()); + packet->write(message.getMessage()); + + nodeList->sendPacket(std::move(packet), *node); + }); } - - auto nodeList = DependencyManager::get(); - nodeList->eachMatchingNode([&](const SharedNodePointer& node) { - return node->getType() == NodeType::ReplicantAvatarMixer; - }, [&](const SharedNodePointer& node) { - // construct an NLPacket to send to the replicant that has the contents of the received packet - auto packet = NLPacket::create(replicatedType, message.getSize() + NUM_BYTES_RFC4122_UUID); - packet->write(message.getSourceID().toRfc4122()); - packet->write(message.getMessage()); - - nodeList->sendPacket(std::move(packet), *node); - }); } void AvatarMixer::queueIncomingPacket(QSharedPointer message, SharedNodePointer node) { @@ -126,7 +137,7 @@ void AvatarMixer::queueIncomingPacket(QSharedPointer message, S auto end = usecTimestampNow(); _queueIncomingPacketElapsedTime += (end - start); - replicatePacket(*message); + optionallyReplicatePacket(*message, *node); } @@ -486,16 +497,16 @@ void AvatarMixer::handleAvatarIdentityPacket(QSharedPointer mes auto end = usecTimestampNow(); _handleAvatarIdentityPacketElapsedTime += (end - start); - replicatePacket(*message); + optionallyReplicatePacket(*message, *senderNode); } -void AvatarMixer::handleKillAvatarPacket(QSharedPointer message) { +void AvatarMixer::handleKillAvatarPacket(QSharedPointer message, SharedNodePointer node) { auto start = usecTimestampNow(); DependencyManager::get()->processKillNode(*message); auto end = usecTimestampNow(); _handleKillAvatarPacketElapsedTime += (end - start); - replicatePacket(*message); + optionallyReplicatePacket(*message, *node); } void AvatarMixer::handleNodeIgnoreRequestPacket(QSharedPointer message, SharedNodePointer senderNode) { diff --git a/assignment-client/src/avatars/AvatarMixer.h b/assignment-client/src/avatars/AvatarMixer.h index ecbed4dca7..17a6db3b08 100644 --- a/assignment-client/src/avatars/AvatarMixer.h +++ b/assignment-client/src/avatars/AvatarMixer.h @@ -42,7 +42,7 @@ private slots: void handleAdjustAvatarSorting(QSharedPointer message, SharedNodePointer senderNode); void handleViewFrustumPacket(QSharedPointer message, SharedNodePointer senderNode); void handleAvatarIdentityPacket(QSharedPointer message, SharedNodePointer senderNode); - void handleKillAvatarPacket(QSharedPointer message); + void handleKillAvatarPacket(QSharedPointer message, SharedNodePointer senderNode); void handleNodeIgnoreRequestPacket(QSharedPointer message, SharedNodePointer senderNode); void handleRadiusIgnoreRequestPacket(QSharedPointer packet, SharedNodePointer sendingNode); void handleRequestsDomainListDataPacket(QSharedPointer message, SharedNodePointer senderNode); @@ -62,7 +62,7 @@ private: void manageDisplayName(const SharedNodePointer& node); - void replicatePacket(ReceivedMessage& message); + void optionallyReplicatePacket(ReceivedMessage& message, const Node& node); p_high_resolution_clock::time_point _lastFrameTimestamp; From b4ce9fb4fcb732980fcce9d1d806aed1b8ba33ca Mon Sep 17 00:00:00 2001 From: Atlante45 Date: Mon, 12 Jun 2017 16:12:13 -0700 Subject: [PATCH 03/46] Remove blocking call + reduce packet construction --- assignment-client/src/avatars/AvatarMixer.cpp | 24 +++++++++---------- 1 file changed, 11 insertions(+), 13 deletions(-) diff --git a/assignment-client/src/avatars/AvatarMixer.cpp b/assignment-client/src/avatars/AvatarMixer.cpp index 2b074f71e6..75b3d23563 100644 --- a/assignment-client/src/avatars/AvatarMixer.cpp +++ b/assignment-client/src/avatars/AvatarMixer.cpp @@ -68,14 +68,8 @@ void AvatarMixer::handleReplicatedPackets(QSharedPointer messag auto nodeList = DependencyManager::get(); auto nodeID = QUuid::fromRfc4122(message->readWithoutCopy(NUM_BYTES_RFC4122_UUID)); - auto replicatedNode = nodeList->nodeWithUUID(nodeID); - if (!replicatedNode) { - QMetaObject::invokeMethod(nodeList.data(), "addOrUpdateNode", Qt::BlockingQueuedConnection, - Q_RETURN_ARG(SharedNodePointer, replicatedNode), - Q_ARG(QUuid, nodeID), Q_ARG(NodeType_t, NodeType::Agent), - Q_ARG(HifiSockAddr, message->getSenderSockAddr()), - Q_ARG(HifiSockAddr, message->getSenderSockAddr())); - } + auto replicatedNode = nodeList->addOrUpdateNode(nodeID, NodeType::Agent, + message->getSenderSockAddr(), message->getSenderSockAddr()) replicatedNode->setLastHeardMicrostamp(usecTimestampNow()); replicatedNode->setIsUpstream(true); @@ -117,16 +111,20 @@ void AvatarMixer::optionallyReplicatePacket(ReceivedMessage& message, const Node return; } + std::unique_ptr packet; + auto nodeList = DependencyManager::get(); nodeList->eachMatchingNode([&](const SharedNodePointer& node) { return node->getType() == NodeType::ReplicantAvatarMixer; }, [&](const SharedNodePointer& node) { - // construct an NLPacket to send to the replicant that has the contents of the received packet - auto packet = NLPacket::create(replicatedType, message.getSize() + NUM_BYTES_RFC4122_UUID); - packet->write(message.getSourceID().toRfc4122()); - packet->write(message.getMessage()); + if (!packet) { + // construct an NLPacket to send to the replicant that has the contents of the received packet + packet = NLPacket::create(replicatedType, message.getSize() + NUM_BYTES_RFC4122_UUID); + packet->write(message.getSourceID().toRfc4122()); + packet->write(message.getMessage()); + } - nodeList->sendPacket(std::move(packet), *node); + nodeList->sendUnreliablePacket(*packet, *node); }); } } From 9aebf68664ca4d1598e2c0df815f4bb3357497a2 Mon Sep 17 00:00:00 2001 From: Atlante45 Date: Mon, 12 Jun 2017 16:32:54 -0700 Subject: [PATCH 04/46] Parse replicant from domain settings --- assignment-client/src/avatars/AvatarMixer.cpp | 7 +++++-- 1 file changed, 5 insertions(+), 2 deletions(-) diff --git a/assignment-client/src/avatars/AvatarMixer.cpp b/assignment-client/src/avatars/AvatarMixer.cpp index 75b3d23563..aae447be60 100644 --- a/assignment-client/src/avatars/AvatarMixer.cpp +++ b/assignment-client/src/avatars/AvatarMixer.cpp @@ -69,7 +69,8 @@ void AvatarMixer::handleReplicatedPackets(QSharedPointer messag auto nodeID = QUuid::fromRfc4122(message->readWithoutCopy(NUM_BYTES_RFC4122_UUID)); auto replicatedNode = nodeList->addOrUpdateNode(nodeID, NodeType::Agent, - message->getSenderSockAddr(), message->getSenderSockAddr()) + message->getSenderSockAddr(), message->getSenderSockAddr(), + DEFAULT_AGENT_PERMISSIONS, true); replicatedNode->setLastHeardMicrostamp(usecTimestampNow()); replicatedNode->setIsUpstream(true); @@ -115,7 +116,7 @@ void AvatarMixer::optionallyReplicatePacket(ReceivedMessage& message, const Node auto nodeList = DependencyManager::get(); nodeList->eachMatchingNode([&](const SharedNodePointer& node) { - return node->getType() == NodeType::ReplicantAvatarMixer; + return node->getType() == NodeType::DownstreamAvatarMixer; }, [&](const SharedNodePointer& node) { if (!packet) { // construct an NLPacket to send to the replicant that has the contents of the received packet @@ -848,4 +849,6 @@ void AvatarMixer::parseDomainServerSettings(const QJsonObject& domainSettings) { qCDebug(avatars) << "This domain requires a minimum avatar scale of" << _domainMinimumScale << "and a maximum avatar scale of" << _domainMaximumScale; + + parseDownstreamServers(domainSettings, NodeType::AvatarMixer); } From 5e3479560354cbfb374bcbdf94516af8a5a7bb68 Mon Sep 17 00:00:00 2001 From: Atlante45 Date: Mon, 12 Jun 2017 18:38:23 -0700 Subject: [PATCH 05/46] Fix packet types filtering --- assignment-client/src/avatars/AvatarMixer.cpp | 9 ++++----- assignment-client/src/avatars/AvatarMixerClientData.cpp | 1 + 2 files changed, 5 insertions(+), 5 deletions(-) diff --git a/assignment-client/src/avatars/AvatarMixer.cpp b/assignment-client/src/avatars/AvatarMixer.cpp index aae447be60..ab51e00e74 100644 --- a/assignment-client/src/avatars/AvatarMixer.cpp +++ b/assignment-client/src/avatars/AvatarMixer.cpp @@ -76,13 +76,13 @@ void AvatarMixer::handleReplicatedPackets(QSharedPointer messag replicatedNode->setIsUpstream(true); switch (message->getType()) { - case PacketType::AvatarData: + case PacketType::ReplicatedAvatarData: queueIncomingPacket(message, replicatedNode); break; - case PacketType::AvatarIdentity: + case PacketType::ReplicatedAvatarIdentity: handleAvatarIdentityPacket(message, replicatedNode); break; - case PacketType::KillAvatar: + case PacketType::ReplicatedKillAvatar: handleKillAvatarPacket(message, replicatedNode); break; default: @@ -108,7 +108,6 @@ void AvatarMixer::optionallyReplicatePacket(ReceivedMessage& message, const Node replicatedType = PacketType::ReplicatedKillAvatar; break; default: - Q_UNREACHABLE(); return; } @@ -125,7 +124,7 @@ void AvatarMixer::optionallyReplicatePacket(ReceivedMessage& message, const Node packet->write(message.getMessage()); } - nodeList->sendUnreliablePacket(*packet, *node); + nodeList->sendUnreliablePacket(*packet, node->getPublicSocket()); }); } } diff --git a/assignment-client/src/avatars/AvatarMixerClientData.cpp b/assignment-client/src/avatars/AvatarMixerClientData.cpp index 41717adeb5..3c9edcd4a5 100644 --- a/assignment-client/src/avatars/AvatarMixerClientData.cpp +++ b/assignment-client/src/avatars/AvatarMixerClientData.cpp @@ -45,6 +45,7 @@ int AvatarMixerClientData::processPackets() { switch (packet->getType()) { case PacketType::AvatarData: + case PacketType::ReplicatedAvatarData: parseData(*packet); break; default: From 44a63ca27ebac8663b0009738e1f89b84e90f8a1 Mon Sep 17 00:00:00 2001 From: Stephen Birarda Date: Tue, 13 Jun 2017 10:44:28 -0700 Subject: [PATCH 06/46] handle node killed and remove double IDs in kill packets --- assignment-client/src/avatars/AvatarMixer.cpp | 48 ++++++++++++++++--- libraries/networking/src/LimitedNodeList.cpp | 2 +- 2 files changed, 42 insertions(+), 8 deletions(-) diff --git a/assignment-client/src/avatars/AvatarMixer.cpp b/assignment-client/src/avatars/AvatarMixer.cpp index ab51e00e74..17cdf8b7d6 100644 --- a/assignment-client/src/avatars/AvatarMixer.cpp +++ b/assignment-client/src/avatars/AvatarMixer.cpp @@ -119,8 +119,18 @@ void AvatarMixer::optionallyReplicatePacket(ReceivedMessage& message, const Node }, [&](const SharedNodePointer& node) { if (!packet) { // construct an NLPacket to send to the replicant that has the contents of the received packet - packet = NLPacket::create(replicatedType, message.getSize() + NUM_BYTES_RFC4122_UUID); - packet->write(message.getSourceID().toRfc4122()); + auto packetSize = message.getSize(); + + if (replicatedType != PacketType::ReplicatedKillAvatar) { + packetSize += NUM_BYTES_RFC4122_UUID; + } + + packet = NLPacket::create(replicatedType, packetSize); + + if (replicatedType != PacketType::ReplicatedKillAvatar) { + packet->write(message.getSourceID().toRfc4122()); + } + packet->write(message.getMessage()); } @@ -352,13 +362,38 @@ void AvatarMixer::nodeKilled(SharedNodePointer killedNode) { } } + std::unique_ptr killPacket; + std::unique_ptr replicatedKillPacket; + // this was an avatar we were sending to other people // send a kill packet for it to our other nodes - auto killPacket = NLPacket::create(PacketType::KillAvatar, NUM_BYTES_RFC4122_UUID + sizeof(KillAvatarReason)); - killPacket->write(killedNode->getUUID().toRfc4122()); - killPacket->writePrimitive(KillAvatarReason::AvatarDisconnected); + nodeList->eachMatchingNode([&](const SharedNodePointer& node) { + // we relay avatar kill packets to agents that are not upstream + // and downstream avatar mixers, if the node that was just killed was being replicated + return (node->getType() == NodeType::Agent && !node->isUpstream()) + || (killedNode->isReplicated() && node->getType() == NodeType::DownstreamAvatarMixer); + }, [&](const SharedNodePointer& node) { + if (node->getType() == NodeType::Agent) { + if (!killPacket) { + killPacket = NLPacket::create(PacketType::KillAvatar, NUM_BYTES_RFC4122_UUID + sizeof(KillAvatarReason)); + killPacket->write(killedNode->getUUID().toRfc4122()); + killPacket->writePrimitive(KillAvatarReason::AvatarDisconnected); + } + + nodeList->sendUnreliablePacket(*killPacket, *node); + } else { + // send a replicated kill packet to the downstream avatar mixer + if (!replicatedKillPacket) { + replicatedKillPacket = NLPacket::create(PacketType::ReplicatedKillAvatar, + NUM_BYTES_RFC4122_UUID + sizeof(KillAvatarReason)); + replicatedKillPacket->write(killedNode->getUUID().toRfc4122()); + replicatedKillPacket->writePrimitive(KillAvatarReason::AvatarDisconnected); + } + + nodeList->sendUnreliablePacket(*replicatedKillPacket, node->getPublicSocket()); + } + }); - nodeList->broadcastToNodes(std::move(killPacket), NodeSet() << NodeType::Agent); // we also want to remove sequence number data for this avatar on our other avatars // so invoke the appropriate method on the AvatarMixerClientData for other avatars @@ -756,7 +791,6 @@ void AvatarMixer::run() { connect(&domainHandler, &DomainHandler::settingsReceiveFail, this, &AvatarMixer::domainSettingsRequestFailed); ThreadedAssignment::commonInit(AVATAR_MIXER_LOGGING_NAME, NodeType::AvatarMixer); - } AvatarMixerClientData* AvatarMixer::getOrCreateClientData(SharedNodePointer node) { diff --git a/libraries/networking/src/LimitedNodeList.cpp b/libraries/networking/src/LimitedNodeList.cpp index f28a972720..b39c86e6e3 100644 --- a/libraries/networking/src/LimitedNodeList.cpp +++ b/libraries/networking/src/LimitedNodeList.cpp @@ -550,7 +550,7 @@ bool LimitedNodeList::killNodeWithUUID(const QUuid& nodeUUID) { void LimitedNodeList::processKillNode(ReceivedMessage& message) { if (message.getType() == PacketType::ReplicatedAvatarData) { - message.seek(NUM_BYTES_RFC4122_UUID); + message.seek(0); } // read the node id From 8ce6590f0b807f24bac9f920879499866f47348c Mon Sep 17 00:00:00 2001 From: Stephen Birarda Date: Tue, 13 Jun 2017 10:56:47 -0700 Subject: [PATCH 07/46] remove ReplicatedAvatarData and immediate replication of identity --- assignment-client/src/avatars/AvatarMixer.cpp | 48 +++---------------- libraries/networking/src/LimitedNodeList.cpp | 4 -- .../networking/src/udt/PacketHeaders.cpp | 2 +- libraries/networking/src/udt/PacketHeaders.h | 1 - 4 files changed, 8 insertions(+), 47 deletions(-) diff --git a/assignment-client/src/avatars/AvatarMixer.cpp b/assignment-client/src/avatars/AvatarMixer.cpp index 17cdf8b7d6..34fcc3c0ed 100644 --- a/assignment-client/src/avatars/AvatarMixer.cpp +++ b/assignment-client/src/avatars/AvatarMixer.cpp @@ -56,7 +56,6 @@ AvatarMixer::AvatarMixer(ReceivedMessage& message) : packetReceiver.registerListenerForTypes({ PacketType::ReplicatedAvatarIdentity, - PacketType::ReplicatedAvatarData, PacketType::ReplicatedKillAvatar }, this, "handleReplicatedPackets"); @@ -76,13 +75,13 @@ void AvatarMixer::handleReplicatedPackets(QSharedPointer messag replicatedNode->setIsUpstream(true); switch (message->getType()) { - case PacketType::ReplicatedAvatarData: - queueIncomingPacket(message, replicatedNode); - break; case PacketType::ReplicatedAvatarIdentity: handleAvatarIdentityPacket(message, replicatedNode); break; case PacketType::ReplicatedKillAvatar: + // seek back in the message so the kill packet handler can start by reading the source ID + message->seek(0); + handleKillAvatarPacket(message, replicatedNode); break; default: @@ -94,22 +93,9 @@ void AvatarMixer::handleReplicatedPackets(QSharedPointer messag void AvatarMixer::optionallyReplicatePacket(ReceivedMessage& message, const Node& node) { // first, make sure that this is a packet from a node we are supposed to replicate - if (node.isReplicated()) { - // now make sure it's a packet type that we want to replicate - PacketType replicatedType; - switch (message.getType()) { - case PacketType::AvatarData: - replicatedType = PacketType::ReplicatedAvatarData; - break; - case PacketType::AvatarIdentity: - replicatedType = PacketType::ReplicatedAvatarIdentity; - break; - case PacketType::KillAvatar: - replicatedType = PacketType::ReplicatedKillAvatar; - break; - default: - return; - } + if (node.isReplicated() + && (message.getType() == PacketType::KillAvatar || message.getType() == PacketType::ReplicatedKillAvatar)) { + PacketType replicatedType = PacketType::ReplicatedKillAvatar; std::unique_ptr packet; @@ -119,18 +105,7 @@ void AvatarMixer::optionallyReplicatePacket(ReceivedMessage& message, const Node }, [&](const SharedNodePointer& node) { if (!packet) { // construct an NLPacket to send to the replicant that has the contents of the received packet - auto packetSize = message.getSize(); - - if (replicatedType != PacketType::ReplicatedKillAvatar) { - packetSize += NUM_BYTES_RFC4122_UUID; - } - - packet = NLPacket::create(replicatedType, packetSize); - - if (replicatedType != PacketType::ReplicatedKillAvatar) { - packet->write(message.getSourceID().toRfc4122()); - } - + packet = NLPacket::create(replicatedType, message.getSize()); packet->write(message.getMessage()); } @@ -144,8 +119,6 @@ void AvatarMixer::queueIncomingPacket(QSharedPointer message, S getOrCreateClientData(node)->queuePacket(message, node); auto end = usecTimestampNow(); _queueIncomingPacketElapsedTime += (end - start); - - optionallyReplicatePacket(*message, *node); } @@ -505,13 +478,6 @@ void AvatarMixer::handleAvatarIdentityPacket(QSharedPointer mes if (nodeData != nullptr) { AvatarData& avatar = nodeData->getAvatar(); - auto data = message->getMessage(); - - if (message->getType() == PacketType::ReplicatedAvatarData) { - data = data.mid(NUM_BYTES_RFC4122_UUID); - } - - // parse the identity packet and update the change timestamp if appropriate AvatarData::Identity identity; AvatarData::parseAvatarIdentityPacket(message->getMessage(), identity); diff --git a/libraries/networking/src/LimitedNodeList.cpp b/libraries/networking/src/LimitedNodeList.cpp index b39c86e6e3..0494efc7b4 100644 --- a/libraries/networking/src/LimitedNodeList.cpp +++ b/libraries/networking/src/LimitedNodeList.cpp @@ -549,10 +549,6 @@ bool LimitedNodeList::killNodeWithUUID(const QUuid& nodeUUID) { } void LimitedNodeList::processKillNode(ReceivedMessage& message) { - if (message.getType() == PacketType::ReplicatedAvatarData) { - message.seek(0); - } - // read the node id QUuid nodeUUID = QUuid::fromRfc4122(message.readWithoutCopy(NUM_BYTES_RFC4122_UUID)); diff --git a/libraries/networking/src/udt/PacketHeaders.cpp b/libraries/networking/src/udt/PacketHeaders.cpp index 9f213d29ff..ad30a184ca 100644 --- a/libraries/networking/src/udt/PacketHeaders.cpp +++ b/libraries/networking/src/udt/PacketHeaders.cpp @@ -42,7 +42,7 @@ const QSet NON_SOURCED_PACKETS = QSet() << PacketType::DomainServerRemovedNode << PacketType::UsernameFromIDReply << PacketType::OctreeFileReplacement << PacketType::ReplicatedMicrophoneAudioNoEcho << PacketType::ReplicatedMicrophoneAudioWithEcho << PacketType::ReplicatedInjectAudio << PacketType::ReplicatedSilentAudioFrame - << PacketType::ReplicatedAvatarIdentity << PacketType::ReplicatedAvatarData << PacketType::ReplicatedKillAvatar; + << PacketType::ReplicatedAvatarIdentity << PacketType::ReplicatedKillAvatar; PacketVersion versionForPacketType(PacketType packetType) { switch (packetType) { diff --git a/libraries/networking/src/udt/PacketHeaders.h b/libraries/networking/src/udt/PacketHeaders.h index 137108ef75..b7d55c3266 100644 --- a/libraries/networking/src/udt/PacketHeaders.h +++ b/libraries/networking/src/udt/PacketHeaders.h @@ -120,7 +120,6 @@ public: ReplicatedInjectAudio, ReplicatedSilentAudioFrame, ReplicatedAvatarIdentity, - ReplicatedAvatarData, ReplicatedKillAvatar, NUM_PACKET_TYPE From a8ea8724d39e42e143c4944aba71c1af43138c1f Mon Sep 17 00:00:00 2001 From: Stephen Birarda Date: Tue, 13 Jun 2017 12:39:47 -0700 Subject: [PATCH 08/46] split broadcast of avatar data for agent and downstream mixer --- .../src/avatars/AvatarMixerSlave.cpp | 631 +++++++++--------- .../src/avatars/AvatarMixerSlave.h | 3 + 2 files changed, 325 insertions(+), 309 deletions(-) diff --git a/assignment-client/src/avatars/AvatarMixerSlave.cpp b/assignment-client/src/avatars/AvatarMixerSlave.cpp index 2ad8bb58ed..7732058f2d 100644 --- a/assignment-client/src/avatars/AvatarMixerSlave.cpp +++ b/assignment-client/src/avatars/AvatarMixerSlave.cpp @@ -81,6 +81,20 @@ static const int AVATAR_MIXER_BROADCAST_FRAMES_PER_SECOND = 45; void AvatarMixerSlave::broadcastAvatarData(const SharedNodePointer& node) { quint64 start = usecTimestampNow(); + if (node->getLinkedData()) { + if (node->getType() == NodeType::Agent && node->getActiveSocket() && !node->isUpstream()) { + broadcastAvatarDataToAgent(node); + } else if (node->getType() == NodeType::DownstreamAvatarMixer) { + broadcastAvatarDataToDownstreamMixer(node); + } + } + + quint64 end = usecTimestampNow(); + _stats.jobElapsedTime += (end - start); +} + +void AvatarMixerSlave::broadcastAvatarDataToAgent(const SharedNodePointer& node) { + auto nodeList = DependencyManager::get(); // setup for distributed random floating point values @@ -88,331 +102,330 @@ void AvatarMixerSlave::broadcastAvatarData(const SharedNodePointer& node) { std::mt19937 generator(randomDevice()); std::uniform_real_distribution distribution; - if (node->getLinkedData() && (node->getType() == NodeType::Agent) && node->getActiveSocket()) { - _stats.nodesBroadcastedTo++; + _stats.nodesBroadcastedTo++; - AvatarMixerClientData* nodeData = reinterpret_cast(node->getLinkedData()); + AvatarMixerClientData* nodeData = reinterpret_cast(node->getLinkedData()); - nodeData->resetInViewStats(); + nodeData->resetInViewStats(); - const AvatarData& avatar = nodeData->getAvatar(); - glm::vec3 myPosition = avatar.getClientGlobalPosition(); + const AvatarData& avatar = nodeData->getAvatar(); + glm::vec3 myPosition = avatar.getClientGlobalPosition(); - // reset the internal state for correct random number distribution - distribution.reset(); + // reset the internal state for correct random number distribution + distribution.reset(); - // reset the number of sent avatars - nodeData->resetNumAvatarsSentLastFrame(); + // reset the number of sent avatars + nodeData->resetNumAvatarsSentLastFrame(); - // keep a counter of the number of considered avatars - int numOtherAvatars = 0; + // keep a counter of the number of considered avatars + int numOtherAvatars = 0; - // keep track of outbound data rate specifically for avatar data - int numAvatarDataBytes = 0; - int identityBytesSent = 0; + // keep track of outbound data rate specifically for avatar data + int numAvatarDataBytes = 0; + int identityBytesSent = 0; - // max number of avatarBytes per frame - auto maxAvatarBytesPerFrame = (_maxKbpsPerNode * BYTES_PER_KILOBIT) / AVATAR_MIXER_BROADCAST_FRAMES_PER_SECOND; + // max number of avatarBytes per frame + auto maxAvatarBytesPerFrame = (_maxKbpsPerNode * BYTES_PER_KILOBIT) / AVATAR_MIXER_BROADCAST_FRAMES_PER_SECOND; - // FIXME - find a way to not send the sessionID for every avatar - int minimumBytesPerAvatar = AvatarDataPacket::AVATAR_HAS_FLAGS_SIZE + NUM_BYTES_RFC4122_UUID; + // FIXME - find a way to not send the sessionID for every avatar + int minimumBytesPerAvatar = AvatarDataPacket::AVATAR_HAS_FLAGS_SIZE + NUM_BYTES_RFC4122_UUID; - int overBudgetAvatars = 0; + int overBudgetAvatars = 0; - // keep track of the number of other avatars held back in this frame - int numAvatarsHeldBack = 0; + // keep track of the number of other avatars held back in this frame + int numAvatarsHeldBack = 0; - // keep track of the number of other avatar frames skipped - int numAvatarsWithSkippedFrames = 0; + // keep track of the number of other avatar frames skipped + int numAvatarsWithSkippedFrames = 0; - // When this is true, the AvatarMixer will send Avatar data to a client - // about avatars they've ignored or that are out of view - bool PALIsOpen = nodeData->getRequestsDomainListData(); + // When this is true, the AvatarMixer will send Avatar data to a client + // about avatars they've ignored or that are out of view + bool PALIsOpen = nodeData->getRequestsDomainListData(); - // When this is true, the AvatarMixer will send Avatar data to a client about avatars that have ignored them - bool getsAnyIgnored = PALIsOpen && node->getCanKick(); + // When this is true, the AvatarMixer will send Avatar data to a client about avatars that have ignored them + bool getsAnyIgnored = PALIsOpen && node->getCanKick(); - if (PALIsOpen) { - // Increase minimumBytesPerAvatar if the PAL is open - minimumBytesPerAvatar += sizeof(AvatarDataPacket::AvatarGlobalPosition) + - sizeof(AvatarDataPacket::AudioLoudness); - } - - // setup a PacketList for the avatarPackets - auto avatarPacketList = NLPacketList::create(PacketType::BulkAvatarData); - - // Define the minimum bubble size - static const glm::vec3 minBubbleSize = glm::vec3(0.3f, 1.3f, 0.3f); - // Define the scale of the box for the current node - glm::vec3 nodeBoxScale = (nodeData->getPosition() - nodeData->getGlobalBoundingBoxCorner()) * 2.0f; - // Set up the bounding box for the current node - AABox nodeBox(nodeData->getGlobalBoundingBoxCorner(), nodeBoxScale); - // Clamp the size of the bounding box to a minimum scale - if (glm::any(glm::lessThan(nodeBoxScale, minBubbleSize))) { - nodeBox.setScaleStayCentered(minBubbleSize); - } - // Quadruple the scale of both bounding boxes - nodeBox.embiggen(4.0f); - - - // setup list of AvatarData as well as maps to map betweeen the AvatarData and the original nodes - // for calling the AvatarData::sortAvatars() function and getting our sorted list of client nodes - QList avatarList; - std::unordered_map avatarDataToNodes; - - std::for_each(_begin, _end, [&](const SharedNodePointer& otherNode) { - const AvatarMixerClientData* otherNodeData = reinterpret_cast(otherNode->getLinkedData()); - - // theoretically it's possible for a Node to be in the NodeList (and therefore end up here), - // but not have yet sent data that's linked to the node. Check for that case and don't - // consider those nodes. - if (otherNodeData) { - AvatarSharedPointer otherAvatar = otherNodeData->getAvatarSharedPointer(); - avatarList << otherAvatar; - avatarDataToNodes[otherAvatar] = otherNode; - } - }); - - AvatarSharedPointer thisAvatar = nodeData->getAvatarSharedPointer(); - ViewFrustum cameraView = nodeData->getViewFrustom(); - std::priority_queue sortedAvatars; - AvatarData::sortAvatars(avatarList, cameraView, sortedAvatars, - - [&](AvatarSharedPointer avatar)->uint64_t{ - auto avatarNode = avatarDataToNodes[avatar]; - assert(avatarNode); // we can't have gotten here without the avatarData being a valid key in the map - return nodeData->getLastBroadcastTime(avatarNode->getUUID()); - }, - - [&](AvatarSharedPointer avatar)->float{ - glm::vec3 nodeBoxHalfScale = (avatar->getPosition() - avatar->getGlobalBoundingBoxCorner()); - return glm::max(nodeBoxHalfScale.x, glm::max(nodeBoxHalfScale.y, nodeBoxHalfScale.z)); - }, - - [&](AvatarSharedPointer avatar)->bool{ - if (avatar == thisAvatar) { - return true; // ignore ourselves... - } - - bool shouldIgnore = false; - - // We will also ignore other nodes for a couple of different reasons: - // 1) ignore bubbles and ignore specific node - // 2) the node hasn't really updated it's frame data recently, this can - // happen if for example the avatar is connected on a desktop and sending - // updates at ~30hz. So every 3 frames we skip a frame. - auto avatarNode = avatarDataToNodes[avatar]; - - assert(avatarNode); // we can't have gotten here without the avatarData being a valid key in the map - - const AvatarMixerClientData* avatarNodeData = reinterpret_cast(avatarNode->getLinkedData()); - assert(avatarNodeData); // we can't have gotten here without avatarNode having valid data - quint64 startIgnoreCalculation = usecTimestampNow(); - - // make sure we have data for this avatar, that it isn't the same node, - // and isn't an avatar that the viewing node has ignored - // or that has ignored the viewing node - if (!avatarNode->getLinkedData() - || avatarNode->getUUID() == node->getUUID() - || (node->isIgnoringNodeWithID(avatarNode->getUUID()) && !PALIsOpen) - || (avatarNode->isIgnoringNodeWithID(node->getUUID()) && !getsAnyIgnored)) { - shouldIgnore = true; - } else { - - // Check to see if the space bubble is enabled - // Don't bother with these checks if the other avatar has their bubble enabled and we're gettingAnyIgnored - if (node->isIgnoreRadiusEnabled() || (avatarNode->isIgnoreRadiusEnabled() && !getsAnyIgnored)) { - - // Define the scale of the box for the current other node - glm::vec3 otherNodeBoxScale = (avatarNodeData->getPosition() - avatarNodeData->getGlobalBoundingBoxCorner()) * 2.0f; - // Set up the bounding box for the current other node - AABox otherNodeBox(avatarNodeData->getGlobalBoundingBoxCorner(), otherNodeBoxScale); - // Clamp the size of the bounding box to a minimum scale - if (glm::any(glm::lessThan(otherNodeBoxScale, minBubbleSize))) { - otherNodeBox.setScaleStayCentered(minBubbleSize); - } - // Quadruple the scale of both bounding boxes - otherNodeBox.embiggen(4.0f); - - // Perform the collision check between the two bounding boxes - if (nodeBox.touches(otherNodeBox)) { - nodeData->ignoreOther(node, avatarNode); - shouldIgnore = !getsAnyIgnored; - } - } - // Not close enough to ignore - if (!shouldIgnore) { - nodeData->removeFromRadiusIgnoringSet(node, avatarNode->getUUID()); - } - } - quint64 endIgnoreCalculation = usecTimestampNow(); - _stats.ignoreCalculationElapsedTime += (endIgnoreCalculation - startIgnoreCalculation); - - if (!shouldIgnore) { - AvatarDataSequenceNumber lastSeqToReceiver = nodeData->getLastBroadcastSequenceNumber(avatarNode->getUUID()); - AvatarDataSequenceNumber lastSeqFromSender = avatarNodeData->getLastReceivedSequenceNumber(); - - // FIXME - This code does appear to be working. But it seems brittle. - // It supports determining if the frame of data for this "other" - // avatar has already been sent to the reciever. This has been - // verified to work on a desktop display that renders at 60hz and - // therefore sends to mixer at 30hz. Each second you'd expect to - // have 15 (45hz-30hz) duplicate frames. In this case, the stat - // avg_other_av_skips_per_second does report 15. - // - // make sure we haven't already sent this data from this sender to this receiver - // or that somehow we haven't sent - if (lastSeqToReceiver == lastSeqFromSender && lastSeqToReceiver != 0) { - ++numAvatarsHeldBack; - shouldIgnore = true; - } else if (lastSeqFromSender - lastSeqToReceiver > 1) { - // this is a skip - we still send the packet but capture the presence of the skip so we see it happening - ++numAvatarsWithSkippedFrames; - } - } - return shouldIgnore; - }); - - // loop through our sorted avatars and allocate our bandwidth to them accordingly - int avatarRank = 0; - - // this is overly conservative, because it includes some avatars we might not consider - int remainingAvatars = (int)sortedAvatars.size(); - - while (!sortedAvatars.empty()) { - AvatarPriority sortData = sortedAvatars.top(); - sortedAvatars.pop(); - const auto& avatarData = sortData.avatar; - avatarRank++; - remainingAvatars--; - - auto otherNode = avatarDataToNodes[avatarData]; - assert(otherNode); // we can't have gotten here without the avatarData being a valid key in the map - - // NOTE: Here's where we determine if we are over budget and drop to bare minimum data - int minimRemainingAvatarBytes = minimumBytesPerAvatar * remainingAvatars; - bool overBudget = (identityBytesSent + numAvatarDataBytes + minimRemainingAvatarBytes) > maxAvatarBytesPerFrame; - - quint64 startAvatarDataPacking = usecTimestampNow(); - - ++numOtherAvatars; - - const AvatarMixerClientData* otherNodeData = reinterpret_cast(otherNode->getLinkedData()); - - // If the time that the mixer sent AVATAR DATA about Avatar B to Avatar A is BEFORE OR EQUAL TO - // the time that Avatar B flagged an IDENTITY DATA change, send IDENTITY DATA about Avatar B to Avatar A. - if (nodeData->getLastBroadcastTime(otherNode->getUUID()) <= otherNodeData->getIdentityChangeTimestamp()) { - identityBytesSent += sendIdentityPacket(otherNodeData, node); - } - - const AvatarData* otherAvatar = otherNodeData->getConstAvatarData(); - glm::vec3 otherPosition = otherAvatar->getClientGlobalPosition(); - - // determine if avatar is in view, to determine how much data to include... - glm::vec3 otherNodeBoxScale = (otherPosition - otherNodeData->getGlobalBoundingBoxCorner()) * 2.0f; - AABox otherNodeBox(otherNodeData->getGlobalBoundingBoxCorner(), otherNodeBoxScale); - bool isInView = nodeData->otherAvatarInView(otherNodeBox); - - // start a new segment in the PacketList for this avatar - avatarPacketList->startSegment(); - - AvatarData::AvatarDataDetail detail; - - if (overBudget) { - overBudgetAvatars++; - _stats.overBudgetAvatars++; - detail = PALIsOpen ? AvatarData::PALMinimum : AvatarData::NoData; - } else if (!isInView) { - detail = PALIsOpen ? AvatarData::PALMinimum : AvatarData::MinimumData; - nodeData->incrementAvatarOutOfView(); - } else { - detail = distribution(generator) < AVATAR_SEND_FULL_UPDATE_RATIO - ? AvatarData::SendAllData : AvatarData::CullSmallData; - nodeData->incrementAvatarInView(); - } - - bool includeThisAvatar = true; - auto lastEncodeForOther = nodeData->getLastOtherAvatarEncodeTime(otherNode->getUUID()); - QVector& lastSentJointsForOther = nodeData->getLastOtherAvatarSentJoints(otherNode->getUUID()); - bool distanceAdjust = true; - glm::vec3 viewerPosition = myPosition; - AvatarDataPacket::HasFlags hasFlagsOut; // the result of the toByteArray - bool dropFaceTracking = false; - - quint64 start = usecTimestampNow(); - QByteArray bytes = otherAvatar->toByteArray(detail, lastEncodeForOther, lastSentJointsForOther, - hasFlagsOut, dropFaceTracking, distanceAdjust, viewerPosition, &lastSentJointsForOther); - quint64 end = usecTimestampNow(); - _stats.toByteArrayElapsedTime += (end - start); - - static const int MAX_ALLOWED_AVATAR_DATA = (1400 - NUM_BYTES_RFC4122_UUID); - if (bytes.size() > MAX_ALLOWED_AVATAR_DATA) { - qCWarning(avatars) << "otherAvatar.toByteArray() resulted in very large buffer:" << bytes.size() << "... attempt to drop facial data"; - - dropFaceTracking = true; // first try dropping the facial data - bytes = otherAvatar->toByteArray(detail, lastEncodeForOther, lastSentJointsForOther, - hasFlagsOut, dropFaceTracking, distanceAdjust, viewerPosition, &lastSentJointsForOther); - - if (bytes.size() > MAX_ALLOWED_AVATAR_DATA) { - qCWarning(avatars) << "otherAvatar.toByteArray() without facial data resulted in very large buffer:" << bytes.size() << "... reduce to MinimumData"; - bytes = otherAvatar->toByteArray(AvatarData::MinimumData, lastEncodeForOther, lastSentJointsForOther, - hasFlagsOut, dropFaceTracking, distanceAdjust, viewerPosition, &lastSentJointsForOther); - } - - if (bytes.size() > MAX_ALLOWED_AVATAR_DATA) { - qCWarning(avatars) << "otherAvatar.toByteArray() MinimumData resulted in very large buffer:" << bytes.size() << "... FAIL!!"; - includeThisAvatar = false; - } - } - - if (includeThisAvatar) { - numAvatarDataBytes += avatarPacketList->write(otherNode->getUUID().toRfc4122()); - numAvatarDataBytes += avatarPacketList->write(bytes); - - if (detail != AvatarData::NoData) { - _stats.numOthersIncluded++; - - // increment the number of avatars sent to this reciever - nodeData->incrementNumAvatarsSentLastFrame(); - - // set the last sent sequence number for this sender on the receiver - nodeData->setLastBroadcastSequenceNumber(otherNode->getUUID(), - otherNodeData->getLastReceivedSequenceNumber()); - - // remember the last time we sent details about this other node to the receiver - nodeData->setLastBroadcastTime(otherNode->getUUID(), start); - } - } - - avatarPacketList->endSegment(); - - quint64 endAvatarDataPacking = usecTimestampNow(); - _stats.avatarDataPackingElapsedTime += (endAvatarDataPacking - startAvatarDataPacking); - }; - - quint64 startPacketSending = usecTimestampNow(); - - // close the current packet so that we're always sending something - avatarPacketList->closeCurrentPacket(true); - - _stats.numPacketsSent += (int)avatarPacketList->getNumPackets(); - _stats.numBytesSent += numAvatarDataBytes; - - // send the avatar data PacketList - nodeList->sendPacketList(std::move(avatarPacketList), *node); - - // record the bytes sent for other avatar data in the AvatarMixerClientData - nodeData->recordSentAvatarData(numAvatarDataBytes); - - // record the number of avatars held back this frame - nodeData->recordNumOtherAvatarStarves(numAvatarsHeldBack); - nodeData->recordNumOtherAvatarSkips(numAvatarsWithSkippedFrames); - - quint64 endPacketSending = usecTimestampNow(); - _stats.packetSendingElapsedTime += (endPacketSending - startPacketSending); + if (PALIsOpen) { + // Increase minimumBytesPerAvatar if the PAL is open + minimumBytesPerAvatar += sizeof(AvatarDataPacket::AvatarGlobalPosition) + + sizeof(AvatarDataPacket::AudioLoudness); } - quint64 end = usecTimestampNow(); - _stats.jobElapsedTime += (end - start); + // setup a PacketList for the avatarPackets + auto avatarPacketList = NLPacketList::create(PacketType::BulkAvatarData); + + // Define the minimum bubble size + static const glm::vec3 minBubbleSize = glm::vec3(0.3f, 1.3f, 0.3f); + // Define the scale of the box for the current node + glm::vec3 nodeBoxScale = (nodeData->getPosition() - nodeData->getGlobalBoundingBoxCorner()) * 2.0f; + // Set up the bounding box for the current node + AABox nodeBox(nodeData->getGlobalBoundingBoxCorner(), nodeBoxScale); + // Clamp the size of the bounding box to a minimum scale + if (glm::any(glm::lessThan(nodeBoxScale, minBubbleSize))) { + nodeBox.setScaleStayCentered(minBubbleSize); + } + // Quadruple the scale of both bounding boxes + nodeBox.embiggen(4.0f); + + + // setup list of AvatarData as well as maps to map betweeen the AvatarData and the original nodes + // for calling the AvatarData::sortAvatars() function and getting our sorted list of client nodes + QList avatarList; + std::unordered_map avatarDataToNodes; + + std::for_each(_begin, _end, [&](const SharedNodePointer& otherNode) { + const AvatarMixerClientData* otherNodeData = reinterpret_cast(otherNode->getLinkedData()); + + // theoretically it's possible for a Node to be in the NodeList (and therefore end up here), + // but not have yet sent data that's linked to the node. Check for that case and don't + // consider those nodes. + if (otherNodeData) { + AvatarSharedPointer otherAvatar = otherNodeData->getAvatarSharedPointer(); + avatarList << otherAvatar; + avatarDataToNodes[otherAvatar] = otherNode; + } + }); + + AvatarSharedPointer thisAvatar = nodeData->getAvatarSharedPointer(); + ViewFrustum cameraView = nodeData->getViewFrustom(); + std::priority_queue sortedAvatars; + AvatarData::sortAvatars(avatarList, cameraView, sortedAvatars, + + [&](AvatarSharedPointer avatar)->uint64_t{ + auto avatarNode = avatarDataToNodes[avatar]; + assert(avatarNode); // we can't have gotten here without the avatarData being a valid key in the map + return nodeData->getLastBroadcastTime(avatarNode->getUUID()); + }, + + [&](AvatarSharedPointer avatar)->float{ + glm::vec3 nodeBoxHalfScale = (avatar->getPosition() - avatar->getGlobalBoundingBoxCorner()); + return glm::max(nodeBoxHalfScale.x, glm::max(nodeBoxHalfScale.y, nodeBoxHalfScale.z)); + }, + + [&](AvatarSharedPointer avatar)->bool{ + if (avatar == thisAvatar) { + return true; // ignore ourselves... + } + + bool shouldIgnore = false; + + // We will also ignore other nodes for a couple of different reasons: + // 1) ignore bubbles and ignore specific node + // 2) the node hasn't really updated it's frame data recently, this can + // happen if for example the avatar is connected on a desktop and sending + // updates at ~30hz. So every 3 frames we skip a frame. + auto avatarNode = avatarDataToNodes[avatar]; + + assert(avatarNode); // we can't have gotten here without the avatarData being a valid key in the map + + const AvatarMixerClientData* avatarNodeData = reinterpret_cast(avatarNode->getLinkedData()); + assert(avatarNodeData); // we can't have gotten here without avatarNode having valid data + quint64 startIgnoreCalculation = usecTimestampNow(); + + // make sure we have data for this avatar, that it isn't the same node, + // and isn't an avatar that the viewing node has ignored + // or that has ignored the viewing node + if (!avatarNode->getLinkedData() + || avatarNode->getUUID() == node->getUUID() + || (node->isIgnoringNodeWithID(avatarNode->getUUID()) && !PALIsOpen) + || (avatarNode->isIgnoringNodeWithID(node->getUUID()) && !getsAnyIgnored)) { + shouldIgnore = true; + } else { + + // Check to see if the space bubble is enabled + // Don't bother with these checks if the other avatar has their bubble enabled and we're gettingAnyIgnored + if (node->isIgnoreRadiusEnabled() || (avatarNode->isIgnoreRadiusEnabled() && !getsAnyIgnored)) { + + // Define the scale of the box for the current other node + glm::vec3 otherNodeBoxScale = (avatarNodeData->getPosition() - avatarNodeData->getGlobalBoundingBoxCorner()) * 2.0f; + // Set up the bounding box for the current other node + AABox otherNodeBox(avatarNodeData->getGlobalBoundingBoxCorner(), otherNodeBoxScale); + // Clamp the size of the bounding box to a minimum scale + if (glm::any(glm::lessThan(otherNodeBoxScale, minBubbleSize))) { + otherNodeBox.setScaleStayCentered(minBubbleSize); + } + // Quadruple the scale of both bounding boxes + otherNodeBox.embiggen(4.0f); + + // Perform the collision check between the two bounding boxes + if (nodeBox.touches(otherNodeBox)) { + nodeData->ignoreOther(node, avatarNode); + shouldIgnore = !getsAnyIgnored; + } + } + // Not close enough to ignore + if (!shouldIgnore) { + nodeData->removeFromRadiusIgnoringSet(node, avatarNode->getUUID()); + } + } + quint64 endIgnoreCalculation = usecTimestampNow(); + _stats.ignoreCalculationElapsedTime += (endIgnoreCalculation - startIgnoreCalculation); + + if (!shouldIgnore) { + AvatarDataSequenceNumber lastSeqToReceiver = nodeData->getLastBroadcastSequenceNumber(avatarNode->getUUID()); + AvatarDataSequenceNumber lastSeqFromSender = avatarNodeData->getLastReceivedSequenceNumber(); + + // FIXME - This code does appear to be working. But it seems brittle. + // It supports determining if the frame of data for this "other" + // avatar has already been sent to the reciever. This has been + // verified to work on a desktop display that renders at 60hz and + // therefore sends to mixer at 30hz. Each second you'd expect to + // have 15 (45hz-30hz) duplicate frames. In this case, the stat + // avg_other_av_skips_per_second does report 15. + // + // make sure we haven't already sent this data from this sender to this receiver + // or that somehow we haven't sent + if (lastSeqToReceiver == lastSeqFromSender && lastSeqToReceiver != 0) { + ++numAvatarsHeldBack; + shouldIgnore = true; + } else if (lastSeqFromSender - lastSeqToReceiver > 1) { + // this is a skip - we still send the packet but capture the presence of the skip so we see it happening + ++numAvatarsWithSkippedFrames; + } + } + return shouldIgnore; + }); + + // loop through our sorted avatars and allocate our bandwidth to them accordingly + int avatarRank = 0; + + // this is overly conservative, because it includes some avatars we might not consider + int remainingAvatars = (int)sortedAvatars.size(); + + while (!sortedAvatars.empty()) { + AvatarPriority sortData = sortedAvatars.top(); + sortedAvatars.pop(); + const auto& avatarData = sortData.avatar; + avatarRank++; + remainingAvatars--; + + auto otherNode = avatarDataToNodes[avatarData]; + assert(otherNode); // we can't have gotten here without the avatarData being a valid key in the map + + // NOTE: Here's where we determine if we are over budget and drop to bare minimum data + int minimRemainingAvatarBytes = minimumBytesPerAvatar * remainingAvatars; + bool overBudget = (identityBytesSent + numAvatarDataBytes + minimRemainingAvatarBytes) > maxAvatarBytesPerFrame; + + quint64 startAvatarDataPacking = usecTimestampNow(); + + ++numOtherAvatars; + + const AvatarMixerClientData* otherNodeData = reinterpret_cast(otherNode->getLinkedData()); + + // If the time that the mixer sent AVATAR DATA about Avatar B to Avatar A is BEFORE OR EQUAL TO + // the time that Avatar B flagged an IDENTITY DATA change, send IDENTITY DATA about Avatar B to Avatar A. + if (nodeData->getLastBroadcastTime(otherNode->getUUID()) <= otherNodeData->getIdentityChangeTimestamp()) { + identityBytesSent += sendIdentityPacket(otherNodeData, node); + } + + const AvatarData* otherAvatar = otherNodeData->getConstAvatarData(); + glm::vec3 otherPosition = otherAvatar->getClientGlobalPosition(); + + // determine if avatar is in view, to determine how much data to include... + glm::vec3 otherNodeBoxScale = (otherPosition - otherNodeData->getGlobalBoundingBoxCorner()) * 2.0f; + AABox otherNodeBox(otherNodeData->getGlobalBoundingBoxCorner(), otherNodeBoxScale); + bool isInView = nodeData->otherAvatarInView(otherNodeBox); + + // start a new segment in the PacketList for this avatar + avatarPacketList->startSegment(); + + AvatarData::AvatarDataDetail detail; + + if (overBudget) { + overBudgetAvatars++; + _stats.overBudgetAvatars++; + detail = PALIsOpen ? AvatarData::PALMinimum : AvatarData::NoData; + } else if (!isInView) { + detail = PALIsOpen ? AvatarData::PALMinimum : AvatarData::MinimumData; + nodeData->incrementAvatarOutOfView(); + } else { + detail = distribution(generator) < AVATAR_SEND_FULL_UPDATE_RATIO + ? AvatarData::SendAllData : AvatarData::CullSmallData; + nodeData->incrementAvatarInView(); + } + + bool includeThisAvatar = true; + auto lastEncodeForOther = nodeData->getLastOtherAvatarEncodeTime(otherNode->getUUID()); + QVector& lastSentJointsForOther = nodeData->getLastOtherAvatarSentJoints(otherNode->getUUID()); + bool distanceAdjust = true; + glm::vec3 viewerPosition = myPosition; + AvatarDataPacket::HasFlags hasFlagsOut; // the result of the toByteArray + bool dropFaceTracking = false; + + quint64 start = usecTimestampNow(); + QByteArray bytes = otherAvatar->toByteArray(detail, lastEncodeForOther, lastSentJointsForOther, + hasFlagsOut, dropFaceTracking, distanceAdjust, viewerPosition, &lastSentJointsForOther); + quint64 end = usecTimestampNow(); + _stats.toByteArrayElapsedTime += (end - start); + + static const int MAX_ALLOWED_AVATAR_DATA = (1400 - NUM_BYTES_RFC4122_UUID); + if (bytes.size() > MAX_ALLOWED_AVATAR_DATA) { + qCWarning(avatars) << "otherAvatar.toByteArray() resulted in very large buffer:" << bytes.size() << "... attempt to drop facial data"; + + dropFaceTracking = true; // first try dropping the facial data + bytes = otherAvatar->toByteArray(detail, lastEncodeForOther, lastSentJointsForOther, + hasFlagsOut, dropFaceTracking, distanceAdjust, viewerPosition, &lastSentJointsForOther); + + if (bytes.size() > MAX_ALLOWED_AVATAR_DATA) { + qCWarning(avatars) << "otherAvatar.toByteArray() without facial data resulted in very large buffer:" << bytes.size() << "... reduce to MinimumData"; + bytes = otherAvatar->toByteArray(AvatarData::MinimumData, lastEncodeForOther, lastSentJointsForOther, + hasFlagsOut, dropFaceTracking, distanceAdjust, viewerPosition, &lastSentJointsForOther); + } + + if (bytes.size() > MAX_ALLOWED_AVATAR_DATA) { + qCWarning(avatars) << "otherAvatar.toByteArray() MinimumData resulted in very large buffer:" << bytes.size() << "... FAIL!!"; + includeThisAvatar = false; + } + } + + if (includeThisAvatar) { + numAvatarDataBytes += avatarPacketList->write(otherNode->getUUID().toRfc4122()); + numAvatarDataBytes += avatarPacketList->write(bytes); + + if (detail != AvatarData::NoData) { + _stats.numOthersIncluded++; + + // increment the number of avatars sent to this reciever + nodeData->incrementNumAvatarsSentLastFrame(); + + // set the last sent sequence number for this sender on the receiver + nodeData->setLastBroadcastSequenceNumber(otherNode->getUUID(), + otherNodeData->getLastReceivedSequenceNumber()); + + // remember the last time we sent details about this other node to the receiver + nodeData->setLastBroadcastTime(otherNode->getUUID(), start); + } + } + + avatarPacketList->endSegment(); + + quint64 endAvatarDataPacking = usecTimestampNow(); + _stats.avatarDataPackingElapsedTime += (endAvatarDataPacking - startAvatarDataPacking); + }; + + quint64 startPacketSending = usecTimestampNow(); + + // close the current packet so that we're always sending something + avatarPacketList->closeCurrentPacket(true); + + _stats.numPacketsSent += (int)avatarPacketList->getNumPackets(); + _stats.numBytesSent += numAvatarDataBytes; + + // send the avatar data PacketList + nodeList->sendPacketList(std::move(avatarPacketList), *node); + + // record the bytes sent for other avatar data in the AvatarMixerClientData + nodeData->recordSentAvatarData(numAvatarDataBytes); + + // record the number of avatars held back this frame + nodeData->recordNumOtherAvatarStarves(numAvatarsHeldBack); + nodeData->recordNumOtherAvatarSkips(numAvatarsWithSkippedFrames); + + quint64 endPacketSending = usecTimestampNow(); + _stats.packetSendingElapsedTime += (endPacketSending - startPacketSending); +} + +void AvatarMixerSlave::broadcastAvatarDataToDownstreamMixer(const SharedNodePointer& node) { + } diff --git a/assignment-client/src/avatars/AvatarMixerSlave.h b/assignment-client/src/avatars/AvatarMixerSlave.h index 04141d9d72..c11ac3291c 100644 --- a/assignment-client/src/avatars/AvatarMixerSlave.h +++ b/assignment-client/src/avatars/AvatarMixerSlave.h @@ -93,6 +93,9 @@ public: private: int sendIdentityPacket(const AvatarMixerClientData* nodeData, const SharedNodePointer& destinationNode); + void broadcastAvatarDataToAgent(const SharedNodePointer& node); + void broadcastAvatarDataToDownstreamMixer(const SharedNodePointer& node); + // frame state ConstIter _begin; ConstIter _end; From 98abb237832821810eb0d7759d4f3ecdb679a207 Mon Sep 17 00:00:00 2001 From: Stephen Birarda Date: Tue, 13 Jun 2017 12:41:39 -0700 Subject: [PATCH 09/46] remove ReplicatedAvatarData handling --- assignment-client/src/avatars/AvatarMixerClientData.cpp | 5 ----- 1 file changed, 5 deletions(-) diff --git a/assignment-client/src/avatars/AvatarMixerClientData.cpp b/assignment-client/src/avatars/AvatarMixerClientData.cpp index 3c9edcd4a5..4d80bc7d17 100644 --- a/assignment-client/src/avatars/AvatarMixerClientData.cpp +++ b/assignment-client/src/avatars/AvatarMixerClientData.cpp @@ -45,7 +45,6 @@ int AvatarMixerClientData::processPackets() { switch (packet->getType()) { case PacketType::AvatarData: - case PacketType::ReplicatedAvatarData: parseData(*packet); break; default: @@ -63,10 +62,6 @@ int AvatarMixerClientData::parseData(ReceivedMessage& message) { // pull the sequence number from the data first uint16_t sequenceNumber; - if (message.getType() == PacketType::ReplicatedAvatarData) { - message.seek(NUM_BYTES_RFC4122_UUID); - } - message.readPrimitive(&sequenceNumber); if (sequenceNumber < _lastReceivedSequenceNumber && _lastReceivedSequenceNumber != UINT16_MAX) { From dc94f83591cc135b9281708dc08c03ce04938e71 Mon Sep 17 00:00:00 2001 From: Stephen Birarda Date: Tue, 13 Jun 2017 13:42:19 -0700 Subject: [PATCH 10/46] build out bulk avatar data replication --- .../src/avatars/AvatarMixerSlave.cpp | 454 +++++++++++------- libraries/networking/src/NLPacketList.h | 2 + .../networking/src/udt/PacketHeaders.cpp | 2 +- libraries/networking/src/udt/PacketHeaders.h | 2 +- libraries/networking/src/udt/PacketList.cpp | 4 +- libraries/networking/src/udt/PacketList.h | 5 +- 6 files changed, 278 insertions(+), 191 deletions(-) diff --git a/assignment-client/src/avatars/AvatarMixerSlave.cpp b/assignment-client/src/avatars/AvatarMixerSlave.cpp index 7732058f2d..802ea442c3 100644 --- a/assignment-client/src/avatars/AvatarMixerSlave.cpp +++ b/assignment-client/src/avatars/AvatarMixerSlave.cpp @@ -81,12 +81,10 @@ static const int AVATAR_MIXER_BROADCAST_FRAMES_PER_SECOND = 45; void AvatarMixerSlave::broadcastAvatarData(const SharedNodePointer& node) { quint64 start = usecTimestampNow(); - if (node->getLinkedData()) { - if (node->getType() == NodeType::Agent && node->getActiveSocket() && !node->isUpstream()) { - broadcastAvatarDataToAgent(node); - } else if (node->getType() == NodeType::DownstreamAvatarMixer) { - broadcastAvatarDataToDownstreamMixer(node); - } + if (node->getType() == NodeType::Agent && node->getLinkedData() && node->getActiveSocket() && !node->isUpstream()) { + broadcastAvatarDataToAgent(node); + } else if (node->getType() == NodeType::DownstreamAvatarMixer) { + broadcastAvatarDataToDownstreamMixer(node); } quint64 end = usecTimestampNow(); @@ -174,12 +172,11 @@ void AvatarMixerSlave::broadcastAvatarDataToAgent(const SharedNodePointer& node) std::unordered_map avatarDataToNodes; std::for_each(_begin, _end, [&](const SharedNodePointer& otherNode) { - const AvatarMixerClientData* otherNodeData = reinterpret_cast(otherNode->getLinkedData()); + // make sure this is an agent that we have avatar data for before considering it for inclusion + if (otherNode->getType() == NodeType::Agent + && otherNode->getLinkedData()) { + const AvatarMixerClientData* otherNodeData = reinterpret_cast(otherNode->getLinkedData()); - // theoretically it's possible for a Node to be in the NodeList (and therefore end up here), - // but not have yet sent data that's linked to the node. Check for that case and don't - // consider those nodes. - if (otherNodeData) { AvatarSharedPointer otherAvatar = otherNodeData->getAvatarSharedPointer(); avatarList << otherAvatar; avatarDataToNodes[otherAvatar] = otherNode; @@ -190,217 +187,216 @@ void AvatarMixerSlave::broadcastAvatarDataToAgent(const SharedNodePointer& node) ViewFrustum cameraView = nodeData->getViewFrustom(); std::priority_queue sortedAvatars; AvatarData::sortAvatars(avatarList, cameraView, sortedAvatars, + [&](AvatarSharedPointer avatar)->uint64_t{ + auto avatarNode = avatarDataToNodes[avatar]; + assert(avatarNode); // we can't have gotten here without the avatarData being a valid key in the map + return nodeData->getLastBroadcastTime(avatarNode->getUUID()); + }, - [&](AvatarSharedPointer avatar)->uint64_t{ - auto avatarNode = avatarDataToNodes[avatar]; - assert(avatarNode); // we can't have gotten here without the avatarData being a valid key in the map - return nodeData->getLastBroadcastTime(avatarNode->getUUID()); - }, + [&](AvatarSharedPointer avatar)->float{ + glm::vec3 nodeBoxHalfScale = (avatar->getPosition() - avatar->getGlobalBoundingBoxCorner()); + return glm::max(nodeBoxHalfScale.x, glm::max(nodeBoxHalfScale.y, nodeBoxHalfScale.z)); + }, - [&](AvatarSharedPointer avatar)->float{ - glm::vec3 nodeBoxHalfScale = (avatar->getPosition() - avatar->getGlobalBoundingBoxCorner()); - return glm::max(nodeBoxHalfScale.x, glm::max(nodeBoxHalfScale.y, nodeBoxHalfScale.z)); - }, + [&](AvatarSharedPointer avatar)->bool{ + if (avatar == thisAvatar) { + return true; // ignore ourselves... + } - [&](AvatarSharedPointer avatar)->bool{ - if (avatar == thisAvatar) { - return true; // ignore ourselves... - } + bool shouldIgnore = false; - bool shouldIgnore = false; + // We will also ignore other nodes for a couple of different reasons: + // 1) ignore bubbles and ignore specific node + // 2) the node hasn't really updated it's frame data recently, this can + // happen if for example the avatar is connected on a desktop and sending + // updates at ~30hz. So every 3 frames we skip a frame. + auto avatarNode = avatarDataToNodes[avatar]; - // We will also ignore other nodes for a couple of different reasons: - // 1) ignore bubbles and ignore specific node - // 2) the node hasn't really updated it's frame data recently, this can - // happen if for example the avatar is connected on a desktop and sending - // updates at ~30hz. So every 3 frames we skip a frame. - auto avatarNode = avatarDataToNodes[avatar]; + assert(avatarNode); // we can't have gotten here without the avatarData being a valid key in the map - assert(avatarNode); // we can't have gotten here without the avatarData being a valid key in the map + const AvatarMixerClientData* avatarNodeData = reinterpret_cast(avatarNode->getLinkedData()); + assert(avatarNodeData); // we can't have gotten here without avatarNode having valid data + quint64 startIgnoreCalculation = usecTimestampNow(); - const AvatarMixerClientData* avatarNodeData = reinterpret_cast(avatarNode->getLinkedData()); - assert(avatarNodeData); // we can't have gotten here without avatarNode having valid data - quint64 startIgnoreCalculation = usecTimestampNow(); + // make sure we have data for this avatar, that it isn't the same node, + // and isn't an avatar that the viewing node has ignored + // or that has ignored the viewing node + if (!avatarNode->getLinkedData() + || avatarNode->getUUID() == node->getUUID() + || (node->isIgnoringNodeWithID(avatarNode->getUUID()) && !PALIsOpen) + || (avatarNode->isIgnoringNodeWithID(node->getUUID()) && !getsAnyIgnored)) { + shouldIgnore = true; + } else { - // make sure we have data for this avatar, that it isn't the same node, - // and isn't an avatar that the viewing node has ignored - // or that has ignored the viewing node - if (!avatarNode->getLinkedData() - || avatarNode->getUUID() == node->getUUID() - || (node->isIgnoringNodeWithID(avatarNode->getUUID()) && !PALIsOpen) - || (avatarNode->isIgnoringNodeWithID(node->getUUID()) && !getsAnyIgnored)) { - shouldIgnore = true; - } else { + // Check to see if the space bubble is enabled + // Don't bother with these checks if the other avatar has their bubble enabled and we're gettingAnyIgnored + if (node->isIgnoreRadiusEnabled() || (avatarNode->isIgnoreRadiusEnabled() && !getsAnyIgnored)) { - // Check to see if the space bubble is enabled - // Don't bother with these checks if the other avatar has their bubble enabled and we're gettingAnyIgnored - if (node->isIgnoreRadiusEnabled() || (avatarNode->isIgnoreRadiusEnabled() && !getsAnyIgnored)) { + // Define the scale of the box for the current other node + glm::vec3 otherNodeBoxScale = (avatarNodeData->getPosition() - avatarNodeData->getGlobalBoundingBoxCorner()) * 2.0f; + // Set up the bounding box for the current other node + AABox otherNodeBox(avatarNodeData->getGlobalBoundingBoxCorner(), otherNodeBoxScale); + // Clamp the size of the bounding box to a minimum scale + if (glm::any(glm::lessThan(otherNodeBoxScale, minBubbleSize))) { + otherNodeBox.setScaleStayCentered(minBubbleSize); + } + // Quadruple the scale of both bounding boxes + otherNodeBox.embiggen(4.0f); - // Define the scale of the box for the current other node - glm::vec3 otherNodeBoxScale = (avatarNodeData->getPosition() - avatarNodeData->getGlobalBoundingBoxCorner()) * 2.0f; - // Set up the bounding box for the current other node - AABox otherNodeBox(avatarNodeData->getGlobalBoundingBoxCorner(), otherNodeBoxScale); - // Clamp the size of the bounding box to a minimum scale - if (glm::any(glm::lessThan(otherNodeBoxScale, minBubbleSize))) { - otherNodeBox.setScaleStayCentered(minBubbleSize); - } - // Quadruple the scale of both bounding boxes - otherNodeBox.embiggen(4.0f); + // Perform the collision check between the two bounding boxes + if (nodeBox.touches(otherNodeBox)) { + nodeData->ignoreOther(node, avatarNode); + shouldIgnore = !getsAnyIgnored; + } + } + // Not close enough to ignore + if (!shouldIgnore) { + nodeData->removeFromRadiusIgnoringSet(node, avatarNode->getUUID()); + } + } + quint64 endIgnoreCalculation = usecTimestampNow(); + _stats.ignoreCalculationElapsedTime += (endIgnoreCalculation - startIgnoreCalculation); - // Perform the collision check between the two bounding boxes - if (nodeBox.touches(otherNodeBox)) { - nodeData->ignoreOther(node, avatarNode); - shouldIgnore = !getsAnyIgnored; - } - } - // Not close enough to ignore - if (!shouldIgnore) { - nodeData->removeFromRadiusIgnoringSet(node, avatarNode->getUUID()); - } - } - quint64 endIgnoreCalculation = usecTimestampNow(); - _stats.ignoreCalculationElapsedTime += (endIgnoreCalculation - startIgnoreCalculation); + if (!shouldIgnore) { + AvatarDataSequenceNumber lastSeqToReceiver = nodeData->getLastBroadcastSequenceNumber(avatarNode->getUUID()); + AvatarDataSequenceNumber lastSeqFromSender = avatarNodeData->getLastReceivedSequenceNumber(); - if (!shouldIgnore) { - AvatarDataSequenceNumber lastSeqToReceiver = nodeData->getLastBroadcastSequenceNumber(avatarNode->getUUID()); - AvatarDataSequenceNumber lastSeqFromSender = avatarNodeData->getLastReceivedSequenceNumber(); + // FIXME - This code does appear to be working. But it seems brittle. + // It supports determining if the frame of data for this "other" + // avatar has already been sent to the reciever. This has been + // verified to work on a desktop display that renders at 60hz and + // therefore sends to mixer at 30hz. Each second you'd expect to + // have 15 (45hz-30hz) duplicate frames. In this case, the stat + // avg_other_av_skips_per_second does report 15. + // + // make sure we haven't already sent this data from this sender to this receiver + // or that somehow we haven't sent + if (lastSeqToReceiver == lastSeqFromSender && lastSeqToReceiver != 0) { + ++numAvatarsHeldBack; + shouldIgnore = true; + } else if (lastSeqFromSender - lastSeqToReceiver > 1) { + // this is a skip - we still send the packet but capture the presence of the skip so we see it happening + ++numAvatarsWithSkippedFrames; + } + } + return shouldIgnore; + }); - // FIXME - This code does appear to be working. But it seems brittle. - // It supports determining if the frame of data for this "other" - // avatar has already been sent to the reciever. This has been - // verified to work on a desktop display that renders at 60hz and - // therefore sends to mixer at 30hz. Each second you'd expect to - // have 15 (45hz-30hz) duplicate frames. In this case, the stat - // avg_other_av_skips_per_second does report 15. - // - // make sure we haven't already sent this data from this sender to this receiver - // or that somehow we haven't sent - if (lastSeqToReceiver == lastSeqFromSender && lastSeqToReceiver != 0) { - ++numAvatarsHeldBack; - shouldIgnore = true; - } else if (lastSeqFromSender - lastSeqToReceiver > 1) { - // this is a skip - we still send the packet but capture the presence of the skip so we see it happening - ++numAvatarsWithSkippedFrames; - } - } - return shouldIgnore; - }); + // loop through our sorted avatars and allocate our bandwidth to them accordingly + int avatarRank = 0; - // loop through our sorted avatars and allocate our bandwidth to them accordingly - int avatarRank = 0; + // this is overly conservative, because it includes some avatars we might not consider + int remainingAvatars = (int)sortedAvatars.size(); - // this is overly conservative, because it includes some avatars we might not consider - int remainingAvatars = (int)sortedAvatars.size(); + while (!sortedAvatars.empty()) { + AvatarPriority sortData = sortedAvatars.top(); + sortedAvatars.pop(); + const auto& avatarData = sortData.avatar; + avatarRank++; + remainingAvatars--; - while (!sortedAvatars.empty()) { - AvatarPriority sortData = sortedAvatars.top(); - sortedAvatars.pop(); - const auto& avatarData = sortData.avatar; - avatarRank++; - remainingAvatars--; + auto otherNode = avatarDataToNodes[avatarData]; + assert(otherNode); // we can't have gotten here without the avatarData being a valid key in the map - auto otherNode = avatarDataToNodes[avatarData]; - assert(otherNode); // we can't have gotten here without the avatarData being a valid key in the map + // NOTE: Here's where we determine if we are over budget and drop to bare minimum data + int minimRemainingAvatarBytes = minimumBytesPerAvatar * remainingAvatars; + bool overBudget = (identityBytesSent + numAvatarDataBytes + minimRemainingAvatarBytes) > maxAvatarBytesPerFrame; - // NOTE: Here's where we determine if we are over budget and drop to bare minimum data - int minimRemainingAvatarBytes = minimumBytesPerAvatar * remainingAvatars; - bool overBudget = (identityBytesSent + numAvatarDataBytes + minimRemainingAvatarBytes) > maxAvatarBytesPerFrame; + quint64 startAvatarDataPacking = usecTimestampNow(); - quint64 startAvatarDataPacking = usecTimestampNow(); + ++numOtherAvatars; - ++numOtherAvatars; + const AvatarMixerClientData* otherNodeData = reinterpret_cast(otherNode->getLinkedData()); - const AvatarMixerClientData* otherNodeData = reinterpret_cast(otherNode->getLinkedData()); + // If the time that the mixer sent AVATAR DATA about Avatar B to Avatar A is BEFORE OR EQUAL TO + // the time that Avatar B flagged an IDENTITY DATA change, send IDENTITY DATA about Avatar B to Avatar A. + if (nodeData->getLastBroadcastTime(otherNode->getUUID()) <= otherNodeData->getIdentityChangeTimestamp()) { + identityBytesSent += sendIdentityPacket(otherNodeData, node); + } - // If the time that the mixer sent AVATAR DATA about Avatar B to Avatar A is BEFORE OR EQUAL TO - // the time that Avatar B flagged an IDENTITY DATA change, send IDENTITY DATA about Avatar B to Avatar A. - if (nodeData->getLastBroadcastTime(otherNode->getUUID()) <= otherNodeData->getIdentityChangeTimestamp()) { - identityBytesSent += sendIdentityPacket(otherNodeData, node); - } + const AvatarData* otherAvatar = otherNodeData->getConstAvatarData(); + glm::vec3 otherPosition = otherAvatar->getClientGlobalPosition(); - const AvatarData* otherAvatar = otherNodeData->getConstAvatarData(); - glm::vec3 otherPosition = otherAvatar->getClientGlobalPosition(); + // determine if avatar is in view, to determine how much data to include... + glm::vec3 otherNodeBoxScale = (otherPosition - otherNodeData->getGlobalBoundingBoxCorner()) * 2.0f; + AABox otherNodeBox(otherNodeData->getGlobalBoundingBoxCorner(), otherNodeBoxScale); + bool isInView = nodeData->otherAvatarInView(otherNodeBox); - // determine if avatar is in view, to determine how much data to include... - glm::vec3 otherNodeBoxScale = (otherPosition - otherNodeData->getGlobalBoundingBoxCorner()) * 2.0f; - AABox otherNodeBox(otherNodeData->getGlobalBoundingBoxCorner(), otherNodeBoxScale); - bool isInView = nodeData->otherAvatarInView(otherNodeBox); + // start a new segment in the PacketList for this avatar + avatarPacketList->startSegment(); - // start a new segment in the PacketList for this avatar - avatarPacketList->startSegment(); + AvatarData::AvatarDataDetail detail; - AvatarData::AvatarDataDetail detail; + if (overBudget) { + overBudgetAvatars++; + _stats.overBudgetAvatars++; + detail = PALIsOpen ? AvatarData::PALMinimum : AvatarData::NoData; + } else if (!isInView) { + detail = PALIsOpen ? AvatarData::PALMinimum : AvatarData::MinimumData; + nodeData->incrementAvatarOutOfView(); + } else { + detail = distribution(generator) < AVATAR_SEND_FULL_UPDATE_RATIO + ? AvatarData::SendAllData : AvatarData::CullSmallData; + nodeData->incrementAvatarInView(); + } - if (overBudget) { - overBudgetAvatars++; - _stats.overBudgetAvatars++; - detail = PALIsOpen ? AvatarData::PALMinimum : AvatarData::NoData; - } else if (!isInView) { - detail = PALIsOpen ? AvatarData::PALMinimum : AvatarData::MinimumData; - nodeData->incrementAvatarOutOfView(); - } else { - detail = distribution(generator) < AVATAR_SEND_FULL_UPDATE_RATIO - ? AvatarData::SendAllData : AvatarData::CullSmallData; - nodeData->incrementAvatarInView(); - } + bool includeThisAvatar = true; + auto lastEncodeForOther = nodeData->getLastOtherAvatarEncodeTime(otherNode->getUUID()); + QVector& lastSentJointsForOther = nodeData->getLastOtherAvatarSentJoints(otherNode->getUUID()); + bool distanceAdjust = true; + glm::vec3 viewerPosition = myPosition; + AvatarDataPacket::HasFlags hasFlagsOut; // the result of the toByteArray + bool dropFaceTracking = false; - bool includeThisAvatar = true; - auto lastEncodeForOther = nodeData->getLastOtherAvatarEncodeTime(otherNode->getUUID()); - QVector& lastSentJointsForOther = nodeData->getLastOtherAvatarSentJoints(otherNode->getUUID()); - bool distanceAdjust = true; - glm::vec3 viewerPosition = myPosition; - AvatarDataPacket::HasFlags hasFlagsOut; // the result of the toByteArray - bool dropFaceTracking = false; - - quint64 start = usecTimestampNow(); - QByteArray bytes = otherAvatar->toByteArray(detail, lastEncodeForOther, lastSentJointsForOther, - hasFlagsOut, dropFaceTracking, distanceAdjust, viewerPosition, &lastSentJointsForOther); - quint64 end = usecTimestampNow(); - _stats.toByteArrayElapsedTime += (end - start); - - static const int MAX_ALLOWED_AVATAR_DATA = (1400 - NUM_BYTES_RFC4122_UUID); - if (bytes.size() > MAX_ALLOWED_AVATAR_DATA) { - qCWarning(avatars) << "otherAvatar.toByteArray() resulted in very large buffer:" << bytes.size() << "... attempt to drop facial data"; - - dropFaceTracking = true; // first try dropping the facial data - bytes = otherAvatar->toByteArray(detail, lastEncodeForOther, lastSentJointsForOther, - hasFlagsOut, dropFaceTracking, distanceAdjust, viewerPosition, &lastSentJointsForOther); + quint64 start = usecTimestampNow(); + QByteArray bytes = otherAvatar->toByteArray(detail, lastEncodeForOther, lastSentJointsForOther, + hasFlagsOut, dropFaceTracking, distanceAdjust, viewerPosition, &lastSentJointsForOther); + quint64 end = usecTimestampNow(); + _stats.toByteArrayElapsedTime += (end - start); + static const int MAX_ALLOWED_AVATAR_DATA = (1400 - NUM_BYTES_RFC4122_UUID); if (bytes.size() > MAX_ALLOWED_AVATAR_DATA) { - qCWarning(avatars) << "otherAvatar.toByteArray() without facial data resulted in very large buffer:" << bytes.size() << "... reduce to MinimumData"; - bytes = otherAvatar->toByteArray(AvatarData::MinimumData, lastEncodeForOther, lastSentJointsForOther, + qCWarning(avatars) << "otherAvatar.toByteArray() resulted in very large buffer:" << bytes.size() << "... attempt to drop facial data"; + + dropFaceTracking = true; // first try dropping the facial data + bytes = otherAvatar->toByteArray(detail, lastEncodeForOther, lastSentJointsForOther, hasFlagsOut, dropFaceTracking, distanceAdjust, viewerPosition, &lastSentJointsForOther); + + if (bytes.size() > MAX_ALLOWED_AVATAR_DATA) { + qCWarning(avatars) << "otherAvatar.toByteArray() without facial data resulted in very large buffer:" << bytes.size() << "... reduce to MinimumData"; + bytes = otherAvatar->toByteArray(AvatarData::MinimumData, lastEncodeForOther, lastSentJointsForOther, + hasFlagsOut, dropFaceTracking, distanceAdjust, viewerPosition, &lastSentJointsForOther); + } + + if (bytes.size() > MAX_ALLOWED_AVATAR_DATA) { + qCWarning(avatars) << "otherAvatar.toByteArray() MinimumData resulted in very large buffer:" << bytes.size() << "... FAIL!!"; + includeThisAvatar = false; + } } - if (bytes.size() > MAX_ALLOWED_AVATAR_DATA) { - qCWarning(avatars) << "otherAvatar.toByteArray() MinimumData resulted in very large buffer:" << bytes.size() << "... FAIL!!"; - includeThisAvatar = false; + if (includeThisAvatar) { + numAvatarDataBytes += avatarPacketList->write(otherNode->getUUID().toRfc4122()); + numAvatarDataBytes += avatarPacketList->write(bytes); + + if (detail != AvatarData::NoData) { + _stats.numOthersIncluded++; + + // increment the number of avatars sent to this reciever + nodeData->incrementNumAvatarsSentLastFrame(); + + // set the last sent sequence number for this sender on the receiver + nodeData->setLastBroadcastSequenceNumber(otherNode->getUUID(), + otherNodeData->getLastReceivedSequenceNumber()); + + // remember the last time we sent details about this other node to the receiver + nodeData->setLastBroadcastTime(otherNode->getUUID(), start); + } } - } - if (includeThisAvatar) { - numAvatarDataBytes += avatarPacketList->write(otherNode->getUUID().toRfc4122()); - numAvatarDataBytes += avatarPacketList->write(bytes); + avatarPacketList->endSegment(); - if (detail != AvatarData::NoData) { - _stats.numOthersIncluded++; - - // increment the number of avatars sent to this reciever - nodeData->incrementNumAvatarsSentLastFrame(); - - // set the last sent sequence number for this sender on the receiver - nodeData->setLastBroadcastSequenceNumber(otherNode->getUUID(), - otherNodeData->getLastReceivedSequenceNumber()); - - // remember the last time we sent details about this other node to the receiver - nodeData->setLastBroadcastTime(otherNode->getUUID(), start); - } - } - - avatarPacketList->endSegment(); - - quint64 endAvatarDataPacking = usecTimestampNow(); - _stats.avatarDataPackingElapsedTime += (endAvatarDataPacking - startAvatarDataPacking); + quint64 endAvatarDataPacking = usecTimestampNow(); + _stats.avatarDataPackingElapsedTime += (endAvatarDataPacking - startAvatarDataPacking); }; quint64 startPacketSending = usecTimestampNow(); @@ -426,6 +422,92 @@ void AvatarMixerSlave::broadcastAvatarDataToAgent(const SharedNodePointer& node) } void AvatarMixerSlave::broadcastAvatarDataToDownstreamMixer(const SharedNodePointer& node) { + _stats.nodesBroadcastedTo++; + // setup a PacketList for the replicated bulk avatar data + auto avatarPacketList = NLPacketList::create(PacketType::ReplicatedBulkAvatarData); + + int numAvatarDataBytes = 0; + + std::for_each(_begin, _end, [&](const SharedNodePointer& agentNode) { + // collect agents that we have avatar data for + if (agentNode->getType() == NodeType::Agent && agentNode->getLinkedData()) { + const AvatarMixerClientData* agentNodeData = reinterpret_cast(agentNode->getLinkedData()); + + AvatarSharedPointer otherAvatar = agentNodeData->getAvatarSharedPointer(); + + quint64 startAvatarDataPacking = usecTimestampNow(); + + // we cannot send a downstream avatar mixer any updates that expect them to have previous state for this avatar + // since we have no idea if they're online and receiving our packets + + // so we always send a full update for this avatar + quint64 start = usecTimestampNow(); + AvatarDataPacket::HasFlags flagsOut; + QByteArray avatarByteArray = otherAvatar->toByteArray(AvatarData::SendAllData, 0, {}, + flagsOut, false, false, + glm::vec3(0), nullptr); + quint64 end = usecTimestampNow(); + _stats.toByteArrayElapsedTime += (end - start); + + // figure out how large our avatar byte array can be to fit in the packet list + // given that we need it and the avatar UUID and the size of the byte array (16 bit) + // to fit in a segment of the packet list + auto maxAvatarByteArraySize = avatarPacketList->getMaxSegmentSize(); + maxAvatarByteArraySize -= NUM_BYTES_RFC4122_UUID; + maxAvatarByteArraySize -= sizeof(quint16); + + if (avatarByteArray.size() > maxAvatarByteArraySize) { + qCWarning(avatars) << "Replicated avatar data too large for" << otherAvatar->getSessionUUID() + << "-" << avatarByteArray.size() << "bytes"; + + avatarByteArray = otherAvatar->toByteArray(AvatarData::SendAllData, 0, {}, + flagsOut, true, false, + glm::vec3(0), nullptr); + + if (avatarByteArray.size() > maxAvatarByteArraySize) { + qCWarning(avatars) << "Replicated avatar data without facial data still too large for" + << otherAvatar->getSessionUUID() << "-" << avatarByteArray.size() << "bytes"; + + avatarByteArray = otherAvatar->toByteArray(AvatarData::MinimumData, 0, {}, + flagsOut, true, false, + glm::vec3(0), nullptr); + } + } + + if (avatarByteArray.size() <= maxAvatarByteArraySize) { + // start a new segment in the packet list for this avatar + avatarPacketList->startSegment(); + + numAvatarDataBytes += avatarPacketList->write(agentNode->getUUID().toRfc4122()); + numAvatarDataBytes += avatarPacketList->writePrimitive((quint16) avatarByteArray.size()); + numAvatarDataBytes += avatarPacketList->write(avatarByteArray); + + avatarPacketList->endSegment(); + + } else { + qCWarning(avatars) << "Could not fit minimum data avatar for" << otherAvatar->getSessionUUID() + << "to packet list -" << avatarByteArray.size() << "bytes"; + } + + quint64 endAvatarDataPacking = usecTimestampNow(); + _stats.avatarDataPackingElapsedTime += (endAvatarDataPacking - startAvatarDataPacking); + } + }); + + quint64 startPacketSending = usecTimestampNow(); + + // close the current packet so that we're always sending something + avatarPacketList->closeCurrentPacket(true); + + _stats.numPacketsSent += (int)avatarPacketList->getNumPackets(); + _stats.numBytesSent += numAvatarDataBytes; + + // send the replicated bulk avatar data + auto nodeList = DependencyManager::get(); + nodeList->sendPacketList(std::move(avatarPacketList), node->getPublicSocket()); + + quint64 endPacketSending = usecTimestampNow(); + _stats.packetSendingElapsedTime += (endPacketSending - startPacketSending); } diff --git a/libraries/networking/src/NLPacketList.h b/libraries/networking/src/NLPacketList.h index 48ce5ef81a..910d39f71b 100644 --- a/libraries/networking/src/NLPacketList.h +++ b/libraries/networking/src/NLPacketList.h @@ -23,6 +23,8 @@ public: PacketVersion getVersion() const { return _packetVersion; } const QUuid& getSourceID() const { return _sourceID; } + + qint64 getMaxSegmentSize() const override { return NLPacket::maxPayloadSize(_packetType, _isOrdered); } private: NLPacketList(PacketType packetType, QByteArray extendedHeader = QByteArray(), bool isReliable = false, diff --git a/libraries/networking/src/udt/PacketHeaders.cpp b/libraries/networking/src/udt/PacketHeaders.cpp index ad30a184ca..fc6251566e 100644 --- a/libraries/networking/src/udt/PacketHeaders.cpp +++ b/libraries/networking/src/udt/PacketHeaders.cpp @@ -42,7 +42,7 @@ const QSet NON_SOURCED_PACKETS = QSet() << PacketType::DomainServerRemovedNode << PacketType::UsernameFromIDReply << PacketType::OctreeFileReplacement << PacketType::ReplicatedMicrophoneAudioNoEcho << PacketType::ReplicatedMicrophoneAudioWithEcho << PacketType::ReplicatedInjectAudio << PacketType::ReplicatedSilentAudioFrame - << PacketType::ReplicatedAvatarIdentity << PacketType::ReplicatedKillAvatar; + << PacketType::ReplicatedAvatarIdentity << PacketType::ReplicatedKillAvatar << PacketType::ReplicatedBulkAvatarData; PacketVersion versionForPacketType(PacketType packetType) { switch (packetType) { diff --git a/libraries/networking/src/udt/PacketHeaders.h b/libraries/networking/src/udt/PacketHeaders.h index b7d55c3266..9b37a7d76d 100644 --- a/libraries/networking/src/udt/PacketHeaders.h +++ b/libraries/networking/src/udt/PacketHeaders.h @@ -121,7 +121,7 @@ public: ReplicatedSilentAudioFrame, ReplicatedAvatarIdentity, ReplicatedKillAvatar, - + ReplicatedBulkAvatarData, NUM_PACKET_TYPE }; }; diff --git a/libraries/networking/src/udt/PacketList.cpp b/libraries/networking/src/udt/PacketList.cpp index 8651f9eed4..d69ff39197 100644 --- a/libraries/networking/src/udt/PacketList.cpp +++ b/libraries/networking/src/udt/PacketList.cpp @@ -36,8 +36,8 @@ std::unique_ptr PacketList::fromReceivedPackets(std::list> _packets; + + bool _isOrdered = false; private: friend class ::LimitedNodeList; @@ -93,7 +97,6 @@ private: Packet::MessageNumber _messageNumber; bool _isReliable = false; - bool _isOrdered = false; std::unique_ptr _currentPacket; From 01263d9435424dc5a93d6a3f21458ea0b0f978ce Mon Sep 17 00:00:00 2001 From: Stephen Birarda Date: Tue, 13 Jun 2017 15:29:12 -0700 Subject: [PATCH 11/46] only send identity packets directly to agents not upstream --- assignment-client/src/avatars/AvatarMixer.cpp | 14 ++++++++------ .../src/avatars/AvatarMixerSlave.cpp | 18 +++++++++++------- 2 files changed, 19 insertions(+), 13 deletions(-) diff --git a/assignment-client/src/avatars/AvatarMixer.cpp b/assignment-client/src/avatars/AvatarMixer.cpp index 34fcc3c0ed..4b032496b1 100644 --- a/assignment-client/src/avatars/AvatarMixer.cpp +++ b/assignment-client/src/avatars/AvatarMixer.cpp @@ -126,12 +126,14 @@ AvatarMixer::~AvatarMixer() { } void AvatarMixer::sendIdentityPacket(AvatarMixerClientData* nodeData, const SharedNodePointer& destinationNode) { - QByteArray individualData = nodeData->getAvatar().identityByteArray(); - individualData.replace(0, NUM_BYTES_RFC4122_UUID, nodeData->getNodeID().toRfc4122()); - auto identityPackets = NLPacketList::create(PacketType::AvatarIdentity, QByteArray(), true, true); - identityPackets->write(individualData); - DependencyManager::get()->sendPacketList(std::move(identityPackets), *destinationNode); - ++_sumIdentityPackets; + if (destinationNode->getType() == NodeType::DownstreamAvatarMixer || !destinationNode->isUpstream()) { + QByteArray individualData = nodeData->getAvatar().identityByteArray(); + individualData.replace(0, NUM_BYTES_RFC4122_UUID, nodeData->getNodeID().toRfc4122()); + auto identityPackets = NLPacketList::create(PacketType::AvatarIdentity, QByteArray(), true, true); + identityPackets->write(individualData); + DependencyManager::get()->sendPacketList(std::move(identityPackets), *destinationNode); + ++_sumIdentityPackets; + } } std::chrono::microseconds AvatarMixer::timeFrame(p_high_resolution_clock::time_point& timestamp) { diff --git a/assignment-client/src/avatars/AvatarMixerSlave.cpp b/assignment-client/src/avatars/AvatarMixerSlave.cpp index 802ea442c3..581052cab3 100644 --- a/assignment-client/src/avatars/AvatarMixerSlave.cpp +++ b/assignment-client/src/avatars/AvatarMixerSlave.cpp @@ -67,13 +67,17 @@ void AvatarMixerSlave::processIncomingPackets(const SharedNodePointer& node) { int AvatarMixerSlave::sendIdentityPacket(const AvatarMixerClientData* nodeData, const SharedNodePointer& destinationNode) { - QByteArray individualData = nodeData->getConstAvatarData()->identityByteArray(); - individualData.replace(0, NUM_BYTES_RFC4122_UUID, nodeData->getNodeID().toRfc4122()); // FIXME, this looks suspicious - auto identityPackets = NLPacketList::create(PacketType::AvatarIdentity, QByteArray(), true, true); - identityPackets->write(individualData); - DependencyManager::get()->sendPacketList(std::move(identityPackets), *destinationNode); - _stats.numIdentityPackets++; - return individualData.size(); + if (destinationNode->getType() == NodeType::DownstreamAvatarMixer || !destinationNode->isUpstream()) { + QByteArray individualData = nodeData->getConstAvatarData()->identityByteArray(); + individualData.replace(0, NUM_BYTES_RFC4122_UUID, nodeData->getNodeID().toRfc4122()); // FIXME, this looks suspicious + auto identityPackets = NLPacketList::create(PacketType::AvatarIdentity, QByteArray(), true, true); + identityPackets->write(individualData); + DependencyManager::get()->sendPacketList(std::move(identityPackets), *destinationNode); + _stats.numIdentityPackets++; + return individualData.size(); + } else { + return -1; + } } static const int AVATAR_MIXER_BROADCAST_FRAMES_PER_SECOND = 45; From 6c5947d319a809cad6c20b8185316b3693a103ba Mon Sep 17 00:00:00 2001 From: Ryan Huffman Date: Tue, 13 Jun 2017 14:43:59 -0700 Subject: [PATCH 12/46] Add periodic sending of avatar identity data to downstream nodes --- assignment-client/src/avatars/AvatarMixerSlave.cpp | 11 +++++++++++ 1 file changed, 11 insertions(+) diff --git a/assignment-client/src/avatars/AvatarMixerSlave.cpp b/assignment-client/src/avatars/AvatarMixerSlave.cpp index 581052cab3..17fadd8284 100644 --- a/assignment-client/src/avatars/AvatarMixerSlave.cpp +++ b/assignment-client/src/avatars/AvatarMixerSlave.cpp @@ -425,9 +425,13 @@ void AvatarMixerSlave::broadcastAvatarDataToAgent(const SharedNodePointer& node) _stats.packetSendingElapsedTime += (endPacketSending - startPacketSending); } +uint64_t REBROADCAST_IDENTITY_TO_DOWNSTREAM_EVERY_US = 5 * 1000 * 1000; + void AvatarMixerSlave::broadcastAvatarDataToDownstreamMixer(const SharedNodePointer& node) { _stats.nodesBroadcastedTo++; + AvatarMixerClientData* nodeData = reinterpret_cast(node->getLinkedData()); + // setup a PacketList for the replicated bulk avatar data auto avatarPacketList = NLPacketList::create(PacketType::ReplicatedBulkAvatarData); @@ -438,6 +442,13 @@ void AvatarMixerSlave::broadcastAvatarDataToDownstreamMixer(const SharedNodePoin if (agentNode->getType() == NodeType::Agent && agentNode->getLinkedData()) { const AvatarMixerClientData* agentNodeData = reinterpret_cast(agentNode->getLinkedData()); + auto now = usecTimestampNow(); + auto lastBroadcastTime = nodeData->getLastBroadcastTime(agentNode->getUUID()); + if (lastBroadcastTime <= agentNodeData->getIdentityChangeTimestamp() + || (now - lastBroadcastTime) >= REBROADCAST_IDENTITY_TO_DOWNSTREAM_EVERY_US) { + sendIdentityPacket(agentNodeData, node); + } + AvatarSharedPointer otherAvatar = agentNodeData->getAvatarSharedPointer(); quint64 startAvatarDataPacking = usecTimestampNow(); From 52150ad97152bb28851cd60bd3c3cc5bfb4ebffb Mon Sep 17 00:00:00 2001 From: Ryan Huffman Date: Tue, 13 Jun 2017 15:29:25 -0700 Subject: [PATCH 13/46] Add node data to downstream avatar mixers --- assignment-client/src/avatars/AvatarMixer.cpp | 6 +++++- libraries/networking/src/ThreadedAssignment.cpp | 7 ++++--- libraries/networking/src/ThreadedAssignment.h | 5 ++++- 3 files changed, 13 insertions(+), 5 deletions(-) diff --git a/assignment-client/src/avatars/AvatarMixer.cpp b/assignment-client/src/avatars/AvatarMixer.cpp index 4b032496b1..582c39b124 100644 --- a/assignment-client/src/avatars/AvatarMixer.cpp +++ b/assignment-client/src/avatars/AvatarMixer.cpp @@ -851,5 +851,9 @@ void AvatarMixer::parseDomainServerSettings(const QJsonObject& domainSettings) { << "and a maximum avatar scale of" << _domainMaximumScale; - parseDownstreamServers(domainSettings, NodeType::AvatarMixer); + parseDownstreamServers(domainSettings, NodeType::AvatarMixer, [](Node& node) { + if (!node.getLinkedData()) { + node.setLinkedData(std::unique_ptr { new AvatarMixerClientData(node.getUUID()) }); + } + }); } diff --git a/libraries/networking/src/ThreadedAssignment.cpp b/libraries/networking/src/ThreadedAssignment.cpp index 5aeb076c11..24573e0ffc 100644 --- a/libraries/networking/src/ThreadedAssignment.cpp +++ b/libraries/networking/src/ThreadedAssignment.cpp @@ -134,7 +134,7 @@ void ThreadedAssignment::domainSettingsRequestFailed() { setFinished(true); } -void ThreadedAssignment::parseDownstreamServers(const QJsonObject& settingsObject, NodeType_t nodeType) { +void ThreadedAssignment::parseDownstreamServers(const QJsonObject& settingsObject, NodeType_t nodeType, DownstreamNodeFoundCallback callback) { static const QString REPLICATION_GROUP_KEY = "replication"; static const QString DOWNSTREAM_SERVERS_SETTING_KEY = "downstream_servers"; if (settingsObject.contains(REPLICATION_GROUP_KEY)) { @@ -161,8 +161,9 @@ void ThreadedAssignment::parseDownstreamServers(const QJsonObject& settingsObjec }; // manually add the downstream node to our node list - nodeList->addOrUpdateNode(QUuid::createUuid(), NodeType::downstreamType(nodeType), - downstreamServerAddr, downstreamServerAddr); + auto node = nodeList->addOrUpdateNode(QUuid::createUuid(), NodeType::downstreamType(nodeType), + downstreamServerAddr, downstreamServerAddr); + callback(*node); } } } diff --git a/libraries/networking/src/ThreadedAssignment.h b/libraries/networking/src/ThreadedAssignment.h index f96755a776..0cc7b2f40c 100644 --- a/libraries/networking/src/ThreadedAssignment.h +++ b/libraries/networking/src/ThreadedAssignment.h @@ -18,6 +18,8 @@ #include "Assignment.h" +using DownstreamNodeFoundCallback = std::function; + class ThreadedAssignment : public Assignment { Q_OBJECT public: @@ -40,7 +42,8 @@ signals: protected: void commonInit(const QString& targetName, NodeType_t nodeType); - void parseDownstreamServers(const QJsonObject& settingsObject, NodeType_t nodeType); + void parseDownstreamServers(const QJsonObject& settingsObject, NodeType_t nodeType, + DownstreamNodeFoundCallback callback = [](Node& downstreamNode) {}); bool _isFinished; QTimer _domainServerTimer; From da3cd59a968fb8816655d91aa9da08a26fb3d433 Mon Sep 17 00:00:00 2001 From: Ryan Huffman Date: Tue, 13 Jun 2017 15:29:55 -0700 Subject: [PATCH 14/46] Fix avatar mixer not setting last broadcast time for downstream nodes --- .../src/avatars/AvatarMixerSlave.cpp | 15 ++++++++------- libraries/networking/src/LimitedNodeList.cpp | 6 ++++-- 2 files changed, 12 insertions(+), 9 deletions(-) diff --git a/assignment-client/src/avatars/AvatarMixerSlave.cpp b/assignment-client/src/avatars/AvatarMixerSlave.cpp index 17fadd8284..fe28c96340 100644 --- a/assignment-client/src/avatars/AvatarMixerSlave.cpp +++ b/assignment-client/src/avatars/AvatarMixerSlave.cpp @@ -442,13 +442,6 @@ void AvatarMixerSlave::broadcastAvatarDataToDownstreamMixer(const SharedNodePoin if (agentNode->getType() == NodeType::Agent && agentNode->getLinkedData()) { const AvatarMixerClientData* agentNodeData = reinterpret_cast(agentNode->getLinkedData()); - auto now = usecTimestampNow(); - auto lastBroadcastTime = nodeData->getLastBroadcastTime(agentNode->getUUID()); - if (lastBroadcastTime <= agentNodeData->getIdentityChangeTimestamp() - || (now - lastBroadcastTime) >= REBROADCAST_IDENTITY_TO_DOWNSTREAM_EVERY_US) { - sendIdentityPacket(agentNodeData, node); - } - AvatarSharedPointer otherAvatar = agentNodeData->getAvatarSharedPointer(); quint64 startAvatarDataPacking = usecTimestampNow(); @@ -465,6 +458,14 @@ void AvatarMixerSlave::broadcastAvatarDataToDownstreamMixer(const SharedNodePoin quint64 end = usecTimestampNow(); _stats.toByteArrayElapsedTime += (end - start); + auto lastBroadcastTime = nodeData->getLastBroadcastTime(agentNode->getUUID()); + if (lastBroadcastTime <= agentNodeData->getIdentityChangeTimestamp() + || (start - lastBroadcastTime) >= REBROADCAST_IDENTITY_TO_DOWNSTREAM_EVERY_US) { + qDebug() << "Sending identity packet for " << agentNode->getUUID() << " to " << node->getUUID(); + sendIdentityPacket(agentNodeData, node); + nodeData->setLastBroadcastTime(agentNode->getUUID(), start); + } + // figure out how large our avatar byte array can be to fit in the packet list // given that we need it and the avatar UUID and the size of the byte array (16 bit) // to fit in a segment of the packet list diff --git a/libraries/networking/src/LimitedNodeList.cpp b/libraries/networking/src/LimitedNodeList.cpp index 0494efc7b4..9c0754cf26 100644 --- a/libraries/networking/src/LimitedNodeList.cpp +++ b/libraries/networking/src/LimitedNodeList.cpp @@ -446,7 +446,8 @@ qint64 LimitedNodeList::sendPacketList(std::unique_ptr packetList, return _nodeSocket.writePacketList(std::move(packetList), *activeSocket); } else { - qCDebug(networking) << "LimitedNodeList::sendPacketList called without active socket for node. Not sending."; + qCDebug(networking) << "LimitedNodeList::sendPacketList called without active socket for node " + << destinationNode.getUUID() << ". Not sending."; return ERROR_SENDING_PACKET_BYTES; } } @@ -454,7 +455,8 @@ qint64 LimitedNodeList::sendPacketList(std::unique_ptr packetList, qint64 LimitedNodeList::sendPacket(std::unique_ptr packet, const Node& destinationNode, const HifiSockAddr& overridenSockAddr) { if (overridenSockAddr.isNull() && !destinationNode.getActiveSocket()) { - qCDebug(networking) << "LimitedNodeList::sendPacket called without active socket for node. Not sending."; + qCDebug(networking) << "LimitedNodeList::sendPacket called without active socket for node" + << destinationNode.getUUID() << ". Not sending."; return ERROR_SENDING_PACKET_BYTES; } From 76f1a7445e5c4cc7dfb9454d5d6f5d6d3f2ebc00 Mon Sep 17 00:00:00 2001 From: Stephen Birarda Date: Tue, 13 Jun 2017 15:32:07 -0700 Subject: [PATCH 15/46] handle replicated bulk avatar data packets --- assignment-client/src/avatars/AvatarMixer.cpp | 37 +++++++++++++++++++ assignment-client/src/avatars/AvatarMixer.h | 1 + .../src/avatars/AvatarMixerSlave.cpp | 18 ++++----- libraries/networking/src/ReceivedMessage.cpp | 14 +++++++ libraries/networking/src/ReceivedMessage.h | 2 + 5 files changed, 63 insertions(+), 9 deletions(-) diff --git a/assignment-client/src/avatars/AvatarMixer.cpp b/assignment-client/src/avatars/AvatarMixer.cpp index 582c39b124..5aaf633ed9 100644 --- a/assignment-client/src/avatars/AvatarMixer.cpp +++ b/assignment-client/src/avatars/AvatarMixer.cpp @@ -59,6 +59,8 @@ AvatarMixer::AvatarMixer(ReceivedMessage& message) : PacketType::ReplicatedKillAvatar }, this, "handleReplicatedPackets"); + packetReceiver.registerListener(PacketType::ReplicatedBulkAvatarData, this, "handleReplicatedBulkAvatarPacket"); + auto nodeList = DependencyManager::get(); connect(nodeList.data(), &NodeList::packetVersionMismatch, this, &AvatarMixer::handlePacketVersionMismatch); } @@ -91,6 +93,41 @@ void AvatarMixer::handleReplicatedPackets(QSharedPointer messag } +void AvatarMixer::handleReplicatedBulkAvatarPacket(QSharedPointer message) { + auto nodeList = DependencyManager::get(); + + while (message->getBytesLeftToRead()) { + // first, grab the node ID for this replicated avatar + auto nodeID = QUuid::fromRfc4122(message->readWithoutCopy(NUM_BYTES_RFC4122_UUID)); + + // make sure we have an upstream replicated node that matches + auto replicatedNode = nodeList->addOrUpdateNode(nodeID, NodeType::Agent, + message->getSenderSockAddr(), message->getSenderSockAddr(), + DEFAULT_AGENT_PERMISSIONS, true); + + replicatedNode->setLastHeardMicrostamp(usecTimestampNow()); + replicatedNode->setIsUpstream(true); + + // grab the size of the avatar byte array so we know how much to read + quint16 avatarByteArraySize; + message->readPrimitive(&avatarByteArraySize); + + // read the avatar byte array + auto avatarByteArray = message->read(avatarByteArraySize); + + // construct a "fake" avatar data received message from the byte array and packet list information + auto replicatedMessage = QSharedPointer::create(avatarByteArray, PacketType::AvatarData, + versionForPacketType(PacketType::AvatarData), + message->getSenderSockAddr(), nodeID); + + // queue up the replicated avatar data with the client data for the replicated node + auto start = usecTimestampNow(); + getOrCreateClientData(replicatedNode)->queuePacket(replicatedMessage, replicatedNode); + auto end = usecTimestampNow(); + _queueIncomingPacketElapsedTime += (end - start); + } +} + void AvatarMixer::optionallyReplicatePacket(ReceivedMessage& message, const Node& node) { // first, make sure that this is a packet from a node we are supposed to replicate if (node.isReplicated() diff --git a/assignment-client/src/avatars/AvatarMixer.h b/assignment-client/src/avatars/AvatarMixer.h index 17a6db3b08..e0d073a281 100644 --- a/assignment-client/src/avatars/AvatarMixer.h +++ b/assignment-client/src/avatars/AvatarMixer.h @@ -47,6 +47,7 @@ private slots: void handleRadiusIgnoreRequestPacket(QSharedPointer packet, SharedNodePointer sendingNode); void handleRequestsDomainListDataPacket(QSharedPointer message, SharedNodePointer senderNode); void handleReplicatedPackets(QSharedPointer message); + void handleReplicatedBulkAvatarPacket(QSharedPointer message); void domainSettingsRequestComplete(); void handlePacketVersionMismatch(PacketType type, const HifiSockAddr& senderSockAddr, const QUuid& senderUUID); void start(); diff --git a/assignment-client/src/avatars/AvatarMixerSlave.cpp b/assignment-client/src/avatars/AvatarMixerSlave.cpp index fe28c96340..a536eaabd3 100644 --- a/assignment-client/src/avatars/AvatarMixerSlave.cpp +++ b/assignment-client/src/avatars/AvatarMixerSlave.cpp @@ -452,9 +452,11 @@ void AvatarMixerSlave::broadcastAvatarDataToDownstreamMixer(const SharedNodePoin // so we always send a full update for this avatar quint64 start = usecTimestampNow(); AvatarDataPacket::HasFlags flagsOut; - QByteArray avatarByteArray = otherAvatar->toByteArray(AvatarData::SendAllData, 0, {}, - flagsOut, false, false, - glm::vec3(0), nullptr); + + QVector emptyLastJointSendData { otherAvatar->getJointCount() }; + + QByteArray avatarByteArray = otherAvatar->toByteArray(AvatarData::SendAllData, 0, emptyLastJointSendData, + flagsOut, false, false, glm::vec3(0), nullptr); quint64 end = usecTimestampNow(); _stats.toByteArrayElapsedTime += (end - start); @@ -477,17 +479,15 @@ void AvatarMixerSlave::broadcastAvatarDataToDownstreamMixer(const SharedNodePoin qCWarning(avatars) << "Replicated avatar data too large for" << otherAvatar->getSessionUUID() << "-" << avatarByteArray.size() << "bytes"; - avatarByteArray = otherAvatar->toByteArray(AvatarData::SendAllData, 0, {}, - flagsOut, true, false, - glm::vec3(0), nullptr); + avatarByteArray = otherAvatar->toByteArray(AvatarData::SendAllData, 0, emptyLastJointSendData, + flagsOut, true, false, glm::vec3(0), nullptr); if (avatarByteArray.size() > maxAvatarByteArraySize) { qCWarning(avatars) << "Replicated avatar data without facial data still too large for" << otherAvatar->getSessionUUID() << "-" << avatarByteArray.size() << "bytes"; - avatarByteArray = otherAvatar->toByteArray(AvatarData::MinimumData, 0, {}, - flagsOut, true, false, - glm::vec3(0), nullptr); + avatarByteArray = otherAvatar->toByteArray(AvatarData::MinimumData, 0, emptyLastJointSendData, + flagsOut, true, false, glm::vec3(0), nullptr); } } diff --git a/libraries/networking/src/ReceivedMessage.cpp b/libraries/networking/src/ReceivedMessage.cpp index 2c5a11334b..6ca249fb22 100644 --- a/libraries/networking/src/ReceivedMessage.cpp +++ b/libraries/networking/src/ReceivedMessage.cpp @@ -42,6 +42,20 @@ ReceivedMessage::ReceivedMessage(NLPacket& packet) { } +ReceivedMessage::ReceivedMessage(QByteArray byteArray, PacketType packetType, PacketVersion packetVersion, + const HifiSockAddr& senderSockAddr, QUuid sourceID) : + _data(byteArray), + _headData(_data.mid(0, HEAD_DATA_SIZE)), + _numPackets(1), + _sourceID(sourceID), + _packetType(packetType), + _packetVersion(packetVersion), + _senderSockAddr(senderSockAddr), + _isComplete(true) +{ + +} + void ReceivedMessage::setFailed() { _failed = true; _isComplete = true; diff --git a/libraries/networking/src/ReceivedMessage.h b/libraries/networking/src/ReceivedMessage.h index 3acb7163e7..ae51e7592a 100644 --- a/libraries/networking/src/ReceivedMessage.h +++ b/libraries/networking/src/ReceivedMessage.h @@ -24,6 +24,8 @@ class ReceivedMessage : public QObject { public: ReceivedMessage(const NLPacketList& packetList); ReceivedMessage(NLPacket& packet); + ReceivedMessage(QByteArray byteArray, PacketType packetType, PacketVersion packetVersion, + const HifiSockAddr& senderSockAddr, QUuid sourceID = QUuid()); QByteArray getMessage() const { return _data; } const char* getRawMessage() const { return _data.constData(); } From 50f46dafa6fa2bdc463b97f5732588d78eb5b259 Mon Sep 17 00:00:00 2001 From: Stephen Birarda Date: Tue, 13 Jun 2017 15:38:15 -0700 Subject: [PATCH 16/46] add a sequence number to replicated avatar data --- .../src/avatars/AvatarMixerSlave.cpp | 15 ++++++++++++++- 1 file changed, 14 insertions(+), 1 deletion(-) diff --git a/assignment-client/src/avatars/AvatarMixerSlave.cpp b/assignment-client/src/avatars/AvatarMixerSlave.cpp index a536eaabd3..8a3ec2dc5a 100644 --- a/assignment-client/src/avatars/AvatarMixerSlave.cpp +++ b/assignment-client/src/avatars/AvatarMixerSlave.cpp @@ -475,6 +475,9 @@ void AvatarMixerSlave::broadcastAvatarDataToDownstreamMixer(const SharedNodePoin maxAvatarByteArraySize -= NUM_BYTES_RFC4122_UUID; maxAvatarByteArraySize -= sizeof(quint16); + auto sequenceNumberSize = sizeof(agentNodeData->getLastReceivedSequenceNumber()); + maxAvatarByteArraySize -= sequenceNumberSize; + if (avatarByteArray.size() > maxAvatarByteArraySize) { qCWarning(avatars) << "Replicated avatar data too large for" << otherAvatar->getSessionUUID() << "-" << avatarByteArray.size() << "bytes"; @@ -492,11 +495,21 @@ void AvatarMixerSlave::broadcastAvatarDataToDownstreamMixer(const SharedNodePoin } if (avatarByteArray.size() <= maxAvatarByteArraySize) { + // increment the number of avatars sent to this reciever + nodeData->incrementNumAvatarsSentLastFrame(); + + // set the last sent sequence number for this sender on the receiver + nodeData->setLastBroadcastSequenceNumber(agentNode->getUUID(), + agentNodeData->getLastReceivedSequenceNumber()); + // start a new segment in the packet list for this avatar avatarPacketList->startSegment(); + // write the node's UUID, the size of the replicated avatar data, + // the sequence number of the replicated avatar data, and the replicated avatar data numAvatarDataBytes += avatarPacketList->write(agentNode->getUUID().toRfc4122()); - numAvatarDataBytes += avatarPacketList->writePrimitive((quint16) avatarByteArray.size()); + numAvatarDataBytes += avatarPacketList->writePrimitive((quint16) (avatarByteArray.size() + sequenceNumberSize)); + numAvatarDataBytes += avatarPacketList->writePrimitive(agentNodeData->getLastReceivedSequenceNumber()); numAvatarDataBytes += avatarPacketList->write(avatarByteArray); avatarPacketList->endSegment(); From 9085a0896a308f0e197b4a1fd8f05881ade83061 Mon Sep 17 00:00:00 2001 From: Stephen Birarda Date: Tue, 13 Jun 2017 15:46:14 -0700 Subject: [PATCH 17/46] manually activate sockets for downstream servers --- assignment-client/src/audio/AudioMixerClientData.cpp | 2 +- assignment-client/src/avatars/AvatarMixer.cpp | 4 ++-- libraries/networking/src/ThreadedAssignment.cpp | 4 ++++ 3 files changed, 7 insertions(+), 3 deletions(-) diff --git a/assignment-client/src/audio/AudioMixerClientData.cpp b/assignment-client/src/audio/AudioMixerClientData.cpp index 848dc969d8..e976e4176d 100644 --- a/assignment-client/src/audio/AudioMixerClientData.cpp +++ b/assignment-client/src/audio/AudioMixerClientData.cpp @@ -169,7 +169,7 @@ void AudioMixerClientData::optionallyReplicatePacket(ReceivedMessage& message, c packet->write(message.getMessage()); } - nodeList->sendUnreliablePacket(*packet, downstreamNode->getPublicSocket()); + nodeList->sendUnreliablePacket(*packet, *downstreamNode); } }); } diff --git a/assignment-client/src/avatars/AvatarMixer.cpp b/assignment-client/src/avatars/AvatarMixer.cpp index 5aaf633ed9..57c48ee6d4 100644 --- a/assignment-client/src/avatars/AvatarMixer.cpp +++ b/assignment-client/src/avatars/AvatarMixer.cpp @@ -146,7 +146,7 @@ void AvatarMixer::optionallyReplicatePacket(ReceivedMessage& message, const Node packet->write(message.getMessage()); } - nodeList->sendUnreliablePacket(*packet, node->getPublicSocket()); + nodeList->sendUnreliablePacket(*packet, *node); }); } } @@ -402,7 +402,7 @@ void AvatarMixer::nodeKilled(SharedNodePointer killedNode) { replicatedKillPacket->writePrimitive(KillAvatarReason::AvatarDisconnected); } - nodeList->sendUnreliablePacket(*replicatedKillPacket, node->getPublicSocket()); + nodeList->sendUnreliablePacket(*replicatedKillPacket, *node); } }); diff --git a/libraries/networking/src/ThreadedAssignment.cpp b/libraries/networking/src/ThreadedAssignment.cpp index 24573e0ffc..3e679f643a 100644 --- a/libraries/networking/src/ThreadedAssignment.cpp +++ b/libraries/networking/src/ThreadedAssignment.cpp @@ -163,6 +163,10 @@ void ThreadedAssignment::parseDownstreamServers(const QJsonObject& settingsObjec // manually add the downstream node to our node list auto node = nodeList->addOrUpdateNode(QUuid::createUuid(), NodeType::downstreamType(nodeType), downstreamServerAddr, downstreamServerAddr); + + // manually activate the public socket for the downstream node + node->activatePublicSocket(); + callback(*node); } } From 8f7a3595f58f58560c668fc44f78340e89b85e25 Mon Sep 17 00:00:00 2001 From: Stephen Birarda Date: Tue, 13 Jun 2017 15:49:03 -0700 Subject: [PATCH 18/46] only send display name changes back to directly connected agents --- assignment-client/src/avatars/AvatarMixer.cpp | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/assignment-client/src/avatars/AvatarMixer.cpp b/assignment-client/src/avatars/AvatarMixer.cpp index 57c48ee6d4..988ddfed4c 100644 --- a/assignment-client/src/avatars/AvatarMixer.cpp +++ b/assignment-client/src/avatars/AvatarMixer.cpp @@ -163,7 +163,7 @@ AvatarMixer::~AvatarMixer() { } void AvatarMixer::sendIdentityPacket(AvatarMixerClientData* nodeData, const SharedNodePointer& destinationNode) { - if (destinationNode->getType() == NodeType::DownstreamAvatarMixer || !destinationNode->isUpstream()) { + if (destinationNode->getType() == NodeType::Agent && !destinationNode->isUpstream()) { QByteArray individualData = nodeData->getAvatar().identityByteArray(); individualData.replace(0, NUM_BYTES_RFC4122_UUID, nodeData->getNodeID().toRfc4122()); auto identityPackets = NLPacketList::create(PacketType::AvatarIdentity, QByteArray(), true, true); From 0708daa6ccfeb401ef53601eeaabfa3894448004 Mon Sep 17 00:00:00 2001 From: Ryan Huffman Date: Tue, 13 Jun 2017 16:10:08 -0700 Subject: [PATCH 19/46] Add separate sendReplicatedIdentityPacket --- assignment-client/src/avatars/AvatarMixer.cpp | 6 +++--- .../src/avatars/AvatarMixerSlave.cpp | 21 +++++++++++++++---- .../src/avatars/AvatarMixerSlave.h | 1 + 3 files changed, 21 insertions(+), 7 deletions(-) diff --git a/assignment-client/src/avatars/AvatarMixer.cpp b/assignment-client/src/avatars/AvatarMixer.cpp index 988ddfed4c..c88108fb36 100644 --- a/assignment-client/src/avatars/AvatarMixer.cpp +++ b/assignment-client/src/avatars/AvatarMixer.cpp @@ -76,14 +76,14 @@ void AvatarMixer::handleReplicatedPackets(QSharedPointer messag replicatedNode->setLastHeardMicrostamp(usecTimestampNow()); replicatedNode->setIsUpstream(true); + // seek back in the message so the packet handler can start by reading the source ID + message->seek(0); + switch (message->getType()) { case PacketType::ReplicatedAvatarIdentity: handleAvatarIdentityPacket(message, replicatedNode); break; case PacketType::ReplicatedKillAvatar: - // seek back in the message so the kill packet handler can start by reading the source ID - message->seek(0); - handleKillAvatarPacket(message, replicatedNode); break; default: diff --git a/assignment-client/src/avatars/AvatarMixerSlave.cpp b/assignment-client/src/avatars/AvatarMixerSlave.cpp index 8a3ec2dc5a..55975a790d 100644 --- a/assignment-client/src/avatars/AvatarMixerSlave.cpp +++ b/assignment-client/src/avatars/AvatarMixerSlave.cpp @@ -65,18 +65,31 @@ void AvatarMixerSlave::processIncomingPackets(const SharedNodePointer& node) { _stats.processIncomingPacketsElapsedTime += (end - start); } - int AvatarMixerSlave::sendIdentityPacket(const AvatarMixerClientData* nodeData, const SharedNodePointer& destinationNode) { if (destinationNode->getType() == NodeType::DownstreamAvatarMixer || !destinationNode->isUpstream()) { QByteArray individualData = nodeData->getConstAvatarData()->identityByteArray(); individualData.replace(0, NUM_BYTES_RFC4122_UUID, nodeData->getNodeID().toRfc4122()); // FIXME, this looks suspicious - auto identityPackets = NLPacketList::create(PacketType::AvatarIdentity, QByteArray(), true, true); + auto identityPackets = NLPacketList::create(PacketType::ReplicatedAvatarIdentity, QByteArray(), true, true); identityPackets->write(individualData); DependencyManager::get()->sendPacketList(std::move(identityPackets), *destinationNode); _stats.numIdentityPackets++; return individualData.size(); } else { - return -1; + return 0; + } +} + +int AvatarMixerSlave::sendReplicatedIdentityPacket(const AvatarMixerClientData* nodeData, const SharedNodePointer& destinationNode) { + if (destinationNode->getType() == NodeType::DownstreamAvatarMixer || !destinationNode->isUpstream()) { + QByteArray individualData = nodeData->getConstAvatarData()->identityByteArray(); + individualData.replace(0, NUM_BYTES_RFC4122_UUID, nodeData->getNodeID().toRfc4122()); // FIXME, this looks suspicious + auto identityPackets = NLPacketList::create(PacketType::ReplicatedAvatarIdentity, QByteArray(), true, true); + identityPackets->write(individualData); + DependencyManager::get()->sendPacketList(std::move(identityPackets), *destinationNode); + _stats.numIdentityPackets++; + return individualData.size(); + } else { + return 0; } } @@ -464,7 +477,7 @@ void AvatarMixerSlave::broadcastAvatarDataToDownstreamMixer(const SharedNodePoin if (lastBroadcastTime <= agentNodeData->getIdentityChangeTimestamp() || (start - lastBroadcastTime) >= REBROADCAST_IDENTITY_TO_DOWNSTREAM_EVERY_US) { qDebug() << "Sending identity packet for " << agentNode->getUUID() << " to " << node->getUUID(); - sendIdentityPacket(agentNodeData, node); + sendReplicatedIdentityPacket(agentNodeData, node); nodeData->setLastBroadcastTime(agentNode->getUUID(), start); } diff --git a/assignment-client/src/avatars/AvatarMixerSlave.h b/assignment-client/src/avatars/AvatarMixerSlave.h index c11ac3291c..509a8ec94b 100644 --- a/assignment-client/src/avatars/AvatarMixerSlave.h +++ b/assignment-client/src/avatars/AvatarMixerSlave.h @@ -92,6 +92,7 @@ public: private: int sendIdentityPacket(const AvatarMixerClientData* nodeData, const SharedNodePointer& destinationNode); + int sendReplicatedIdentityPacket(const AvatarMixerClientData* nodeData, const SharedNodePointer& destinationNode); void broadcastAvatarDataToAgent(const SharedNodePointer& node); void broadcastAvatarDataToDownstreamMixer(const SharedNodePointer& node); From a476a5b82ea740ff34fa5f144acd4ad38d0b05a4 Mon Sep 17 00:00:00 2001 From: Ryan Huffman Date: Tue, 13 Jun 2017 16:42:38 -0700 Subject: [PATCH 20/46] Change ReplicatedAvatarIdentity packet to be an unreliable packet --- assignment-client/src/avatars/AvatarMixerSlave.cpp | 8 ++++---- 1 file changed, 4 insertions(+), 4 deletions(-) diff --git a/assignment-client/src/avatars/AvatarMixerSlave.cpp b/assignment-client/src/avatars/AvatarMixerSlave.cpp index 55975a790d..b6e76e2b57 100644 --- a/assignment-client/src/avatars/AvatarMixerSlave.cpp +++ b/assignment-client/src/avatars/AvatarMixerSlave.cpp @@ -69,7 +69,7 @@ int AvatarMixerSlave::sendIdentityPacket(const AvatarMixerClientData* nodeData, if (destinationNode->getType() == NodeType::DownstreamAvatarMixer || !destinationNode->isUpstream()) { QByteArray individualData = nodeData->getConstAvatarData()->identityByteArray(); individualData.replace(0, NUM_BYTES_RFC4122_UUID, nodeData->getNodeID().toRfc4122()); // FIXME, this looks suspicious - auto identityPackets = NLPacketList::create(PacketType::ReplicatedAvatarIdentity, QByteArray(), true, true); + auto identityPackets = NLPacketList::create(PacketType::AvatarIdentity, QByteArray(), true, true); identityPackets->write(individualData); DependencyManager::get()->sendPacketList(std::move(identityPackets), *destinationNode); _stats.numIdentityPackets++; @@ -83,9 +83,9 @@ int AvatarMixerSlave::sendReplicatedIdentityPacket(const AvatarMixerClientData* if (destinationNode->getType() == NodeType::DownstreamAvatarMixer || !destinationNode->isUpstream()) { QByteArray individualData = nodeData->getConstAvatarData()->identityByteArray(); individualData.replace(0, NUM_BYTES_RFC4122_UUID, nodeData->getNodeID().toRfc4122()); // FIXME, this looks suspicious - auto identityPackets = NLPacketList::create(PacketType::ReplicatedAvatarIdentity, QByteArray(), true, true); - identityPackets->write(individualData); - DependencyManager::get()->sendPacketList(std::move(identityPackets), *destinationNode); + auto identityPacket = NLPacket::create(PacketType::ReplicatedAvatarIdentity); + identityPacket->write(individualData); + DependencyManager::get()->sendUnreliablePacket(*identityPacket, *destinationNode); _stats.numIdentityPackets++; return individualData.size(); } else { From 29842c67cc1705ae6d0446675426cf31ac0e1fc5 Mon Sep 17 00:00:00 2001 From: Stephen Birarda Date: Tue, 13 Jun 2017 18:04:59 -0700 Subject: [PATCH 21/46] use received message faking for cleaner replication in audio --- assignment-client/src/audio/AudioMixer.cpp | 21 +++++++++++++++- .../src/audio/AudioMixerClientData.cpp | 24 +++++-------------- libraries/audio/src/InboundAudioStream.cpp | 12 +++++++++- 3 files changed, 37 insertions(+), 20 deletions(-) diff --git a/assignment-client/src/audio/AudioMixer.cpp b/assignment-client/src/audio/AudioMixer.cpp index f8c2cf86e8..e397254441 100644 --- a/assignment-client/src/audio/AudioMixer.cpp +++ b/assignment-client/src/audio/AudioMixer.cpp @@ -125,7 +125,26 @@ void AudioMixer::queueReplicatedAudioPacket(QSharedPointer mess replicatedNode->setLastHeardMicrostamp(usecTimestampNow()); replicatedNode->setIsUpstream(true); - getOrCreateClientData(replicatedNode.data())->queuePacket(message, replicatedNode); + // construct a "fake" avatar data received message from the byte array and packet list information + auto audioData = message->getMessage().mid(NUM_BYTES_RFC4122_UUID); + + PacketType rewrittenType; + + if (message->getType() == PacketType::ReplicatedMicrophoneAudioNoEcho) { + rewrittenType = PacketType::MicrophoneAudioNoEcho; + } else if (message->getType() == PacketType::ReplicatedMicrophoneAudioWithEcho) { + rewrittenType = PacketType::MicrophoneAudioWithEcho; + } else if (message->getType() == PacketType::ReplicatedInjectAudio) { + rewrittenType = PacketType::InjectAudio; + } else if (message->getType() == PacketType::ReplicatedSilentAudioFrame) { + rewrittenType = PacketType::SilentAudioFrame; + } + + auto replicatedMessage = QSharedPointer::create(audioData, rewrittenType, + versionForPacketType(rewrittenType), + message->getSenderSockAddr(), nodeID); + + getOrCreateClientData(replicatedNode.data())->queuePacket(replicatedMessage, replicatedNode); } void AudioMixer::handleMuteEnvironmentPacket(QSharedPointer message, SharedNodePointer sendingNode) { diff --git a/assignment-client/src/audio/AudioMixerClientData.cpp b/assignment-client/src/audio/AudioMixerClientData.cpp index e976e4176d..d4d098133d 100644 --- a/assignment-client/src/audio/AudioMixerClientData.cpp +++ b/assignment-client/src/audio/AudioMixerClientData.cpp @@ -160,10 +160,6 @@ void AudioMixerClientData::optionallyReplicatePacket(ReceivedMessage& message, c if (!isReplicatedPacket(message.getType())) { // since this packet will be non-sourced, we add the replicated node's ID here packet->write(node.getUUID().toRfc4122()); - - // we won't negotiate an audio format with the replicant, because we aren't a listener - // so pack the codec string here so that it can statelessly setup a decoder for this string when it needs - packet->writeString(_selectedCodecName); } packet->write(message.getMessage()); @@ -312,6 +308,7 @@ int AudioMixerClientData::parseData(ReceivedMessage& message) { // this is injected audio // grab the stream identifier for this injected audio message.seek(sizeof(quint16)); + QUuid streamIdentifier = QUuid::fromRfc4122(message.readWithoutCopy(NUM_BYTES_RFC4122_UUID)); bool isStereo; @@ -346,18 +343,6 @@ int AudioMixerClientData::parseData(ReceivedMessage& message) { // seek to the beginning of the packet so that the next reader is in the right spot message.seek(0); - if (packetType == PacketType::ReplicatedMicrophoneAudioWithEcho - || packetType == PacketType::ReplicatedMicrophoneAudioNoEcho - || packetType == PacketType::ReplicatedSilentAudioFrame - || packetType == PacketType::ReplicatedInjectAudio) { - - // skip past source ID for the replicated packet - message.seek(NUM_BYTES_RFC4122_UUID); - - // skip past the codec string - message.readString(); - } - // check the overflow count before we parse data auto overflowBefore = matchingStream->getOverflowCount(); auto parseResult = matchingStream->parseData(message); @@ -706,9 +691,9 @@ bool AudioMixerClientData::shouldIgnore(const SharedNodePointer self, const Shar } void AudioMixerClientData::setupCodecForReplicatedAgent(QSharedPointer message) { - // first pull the codec string from the packet + // pull the codec string from the packet + message->seek(sizeof(quint16)); - // read the string for the codec auto codecString = message->readString(); qDebug() << "Manually setting codec for replicated agent" << uuidStringWithoutCurlyBraces(getNodeID()) @@ -718,4 +703,7 @@ void AudioMixerClientData::setupCodecForReplicatedAgent(QSharedPointerseek(0); } diff --git a/libraries/audio/src/InboundAudioStream.cpp b/libraries/audio/src/InboundAudioStream.cpp index 65cccf1fe0..56353e14e3 100644 --- a/libraries/audio/src/InboundAudioStream.cpp +++ b/libraries/audio/src/InboundAudioStream.cpp @@ -127,7 +127,17 @@ int InboundAudioStream::parseData(ReceivedMessage& message) { // parse the info after the seq number and before the audio data (the stream properties) int prePropertyPosition = message.getPosition(); int propertyBytes = parseStreamProperties(message.getType(), message.readWithoutCopy(message.getBytesLeftToRead()), networkFrames); - message.seek(prePropertyPosition + propertyBytes); + + if (message.getType() == PacketType::ReplicatedMicrophoneAudioNoEcho + || message.getType() == PacketType::ReplicatedMicrophoneAudioWithEcho + || message.getType() == PacketType::ReplicatedInjectAudio + || message.getType() == PacketType::ReplicatedSilentAudioFrame) { + message.seek(NUM_BYTES_RFC4122_UUID); + message.readString(); + message.read(sizeof(quint16) + prePropertyPosition + propertyBytes); + } else { + message.seek(prePropertyPosition + propertyBytes); + } // handle this packet based on its arrival status. switch (arrivalInfo._status) { From f6f3087580998aebdfa6338f10c866ea32764449 Mon Sep 17 00:00:00 2001 From: Stephen Birarda Date: Tue, 13 Jun 2017 18:11:50 -0700 Subject: [PATCH 22/46] only broadcast replicated nodes to downstream --- assignment-client/src/avatars/AvatarMixerSlave.cpp | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/assignment-client/src/avatars/AvatarMixerSlave.cpp b/assignment-client/src/avatars/AvatarMixerSlave.cpp index b6e76e2b57..6c50d5a626 100644 --- a/assignment-client/src/avatars/AvatarMixerSlave.cpp +++ b/assignment-client/src/avatars/AvatarMixerSlave.cpp @@ -451,8 +451,8 @@ void AvatarMixerSlave::broadcastAvatarDataToDownstreamMixer(const SharedNodePoin int numAvatarDataBytes = 0; std::for_each(_begin, _end, [&](const SharedNodePointer& agentNode) { - // collect agents that we have avatar data for - if (agentNode->getType() == NodeType::Agent && agentNode->getLinkedData()) { + // collect agents that we have avatar data for that we are supposed to replicate + if (agentNode->getType() == NodeType::Agent && agentNode->getLinkedData() && agentNode->isReplicated()) { const AvatarMixerClientData* agentNodeData = reinterpret_cast(agentNode->getLinkedData()); AvatarSharedPointer otherAvatar = agentNodeData->getAvatarSharedPointer(); From 324a2601fc2ebe94ede06297374ff3fdc5b40c96 Mon Sep 17 00:00:00 2001 From: Stephen Birarda Date: Wed, 14 Jun 2017 11:00:22 -0700 Subject: [PATCH 23/46] send down replicated message to avatar mixer handling --- assignment-client/src/audio/AudioMixer.cpp | 2 +- assignment-client/src/avatars/AvatarMixer.cpp | 22 ++++++++++++++----- 2 files changed, 18 insertions(+), 6 deletions(-) diff --git a/assignment-client/src/audio/AudioMixer.cpp b/assignment-client/src/audio/AudioMixer.cpp index e397254441..a98c172ef3 100644 --- a/assignment-client/src/audio/AudioMixer.cpp +++ b/assignment-client/src/audio/AudioMixer.cpp @@ -125,7 +125,7 @@ void AudioMixer::queueReplicatedAudioPacket(QSharedPointer mess replicatedNode->setLastHeardMicrostamp(usecTimestampNow()); replicatedNode->setIsUpstream(true); - // construct a "fake" avatar data received message from the byte array and packet list information + // construct a "fake" audio received message from the byte array and packet list information auto audioData = message->getMessage().mid(NUM_BYTES_RFC4122_UUID); PacketType rewrittenType; diff --git a/assignment-client/src/avatars/AvatarMixer.cpp b/assignment-client/src/avatars/AvatarMixer.cpp index c88108fb36..9bed3e1351 100644 --- a/assignment-client/src/avatars/AvatarMixer.cpp +++ b/assignment-client/src/avatars/AvatarMixer.cpp @@ -76,15 +76,27 @@ void AvatarMixer::handleReplicatedPackets(QSharedPointer messag replicatedNode->setLastHeardMicrostamp(usecTimestampNow()); replicatedNode->setIsUpstream(true); - // seek back in the message so the packet handler can start by reading the source ID - message->seek(0); + // construct a "fake" avatar data received message from the byte array and packet list information + auto originalPacketData = message->getMessage().mid(NUM_BYTES_RFC4122_UUID); - switch (message->getType()) { + PacketType rewrittenType; + + if (message->getType() == PacketType::ReplicatedAvatarIdentity) { + rewrittenType = PacketType::AvatarIdentity; + } else if (message->getType() == PacketType::ReplicatedKillAvatar) { + rewrittenType = PacketType::KillAvatar; + } + + auto replicatedMessage = QSharedPointer::create(originalPacketData, rewrittenType, + versionForPacketType(rewrittenType), + message->getSenderSockAddr(), nodeID); + + switch (rewrittenType) { case PacketType::ReplicatedAvatarIdentity: - handleAvatarIdentityPacket(message, replicatedNode); + handleAvatarIdentityPacket(replicatedMessage, replicatedNode); break; case PacketType::ReplicatedKillAvatar: - handleKillAvatarPacket(message, replicatedNode); + handleKillAvatarPacket(replicatedMessage, replicatedNode); break; default: // Do nothing From 7d8ddb7b3b5b79dbfbf1a69c26d29418314fefdf Mon Sep 17 00:00:00 2001 From: Stephen Birarda Date: Wed, 14 Jun 2017 11:06:58 -0700 Subject: [PATCH 24/46] cleanup calls in handleReplicatedPackets --- assignment-client/src/avatars/AvatarMixer.cpp | 15 ++++----------- 1 file changed, 4 insertions(+), 11 deletions(-) diff --git a/assignment-client/src/avatars/AvatarMixer.cpp b/assignment-client/src/avatars/AvatarMixer.cpp index 9bed3e1351..152182e5fa 100644 --- a/assignment-client/src/avatars/AvatarMixer.cpp +++ b/assignment-client/src/avatars/AvatarMixer.cpp @@ -91,18 +91,11 @@ void AvatarMixer::handleReplicatedPackets(QSharedPointer messag versionForPacketType(rewrittenType), message->getSenderSockAddr(), nodeID); - switch (rewrittenType) { - case PacketType::ReplicatedAvatarIdentity: - handleAvatarIdentityPacket(replicatedMessage, replicatedNode); - break; - case PacketType::ReplicatedKillAvatar: - handleKillAvatarPacket(replicatedMessage, replicatedNode); - break; - default: - // Do nothing - break; + if (rewrittenType == PacketType::AvatarIdentity) { + handleAvatarIdentityPacket(replicatedMessage, replicatedNode); + } else if (rewrittenType == PacketType::KillAvatar) { + handleKillAvatarPacket(replicatedMessage, replicatedNode); } - } void AvatarMixer::handleReplicatedBulkAvatarPacket(QSharedPointer message) { From b7d8c173c1bda1d4b472b27d61afa983ae5476ef Mon Sep 17 00:00:00 2001 From: Stephen Birarda Date: Wed, 14 Jun 2017 11:22:42 -0700 Subject: [PATCH 25/46] cleanup replicated node add or update --- assignment-client/src/avatars/AvatarMixer.cpp | 26 ++++++++++--------- 1 file changed, 14 insertions(+), 12 deletions(-) diff --git a/assignment-client/src/avatars/AvatarMixer.cpp b/assignment-client/src/avatars/AvatarMixer.cpp index 152182e5fa..27eae8619f 100644 --- a/assignment-client/src/avatars/AvatarMixer.cpp +++ b/assignment-client/src/avatars/AvatarMixer.cpp @@ -65,16 +65,23 @@ AvatarMixer::AvatarMixer(ReceivedMessage& message) : connect(nodeList.data(), &NodeList::packetVersionMismatch, this, &AvatarMixer::handlePacketVersionMismatch); } +SharedNodePointer addOrUpdateReplicatedNode(const QUuid& nodeID, const HifiSockAddr& senderSockAddr) { + auto replicatedNode = DependencyManager::get()->addOrUpdateNode(nodeID, NodeType::Agent, + senderSockAddr, + senderSockAddr, + DEFAULT_AGENT_PERMISSIONS, true); + + replicatedNode->setLastHeardMicrostamp(usecTimestampNow()); + replicatedNode->setIsUpstream(true); + + return replicatedNode; +} + void AvatarMixer::handleReplicatedPackets(QSharedPointer message) { auto nodeList = DependencyManager::get(); auto nodeID = QUuid::fromRfc4122(message->readWithoutCopy(NUM_BYTES_RFC4122_UUID)); - auto replicatedNode = nodeList->addOrUpdateNode(nodeID, NodeType::Agent, - message->getSenderSockAddr(), message->getSenderSockAddr(), - DEFAULT_AGENT_PERMISSIONS, true); - - replicatedNode->setLastHeardMicrostamp(usecTimestampNow()); - replicatedNode->setIsUpstream(true); + auto replicatedNode = addOrUpdateReplicatedNode(nodeID, message->getSenderSockAddr()); // construct a "fake" avatar data received message from the byte array and packet list information auto originalPacketData = message->getMessage().mid(NUM_BYTES_RFC4122_UUID); @@ -106,12 +113,7 @@ void AvatarMixer::handleReplicatedBulkAvatarPacket(QSharedPointerreadWithoutCopy(NUM_BYTES_RFC4122_UUID)); // make sure we have an upstream replicated node that matches - auto replicatedNode = nodeList->addOrUpdateNode(nodeID, NodeType::Agent, - message->getSenderSockAddr(), message->getSenderSockAddr(), - DEFAULT_AGENT_PERMISSIONS, true); - - replicatedNode->setLastHeardMicrostamp(usecTimestampNow()); - replicatedNode->setIsUpstream(true); + auto replicatedNode = addOrUpdateReplicatedNode(nodeID, message->getSenderSockAddr()); // grab the size of the avatar byte array so we know how much to read quint16 avatarByteArraySize; From 6fc1045ee9ca7caab99b417a87e1d09ef0e205f9 Mon Sep 17 00:00:00 2001 From: Stephen Birarda Date: Wed, 14 Jun 2017 11:26:33 -0700 Subject: [PATCH 26/46] make sure kill packet has node ID --- assignment-client/src/avatars/AvatarMixer.cpp | 9 +++++++-- 1 file changed, 7 insertions(+), 2 deletions(-) diff --git a/assignment-client/src/avatars/AvatarMixer.cpp b/assignment-client/src/avatars/AvatarMixer.cpp index 27eae8619f..0c4b614033 100644 --- a/assignment-client/src/avatars/AvatarMixer.cpp +++ b/assignment-client/src/avatars/AvatarMixer.cpp @@ -150,6 +150,13 @@ void AvatarMixer::optionallyReplicatePacket(ReceivedMessage& message, const Node if (!packet) { // construct an NLPacket to send to the replicant that has the contents of the received packet packet = NLPacket::create(replicatedType, message.getSize()); + + if (message.getType() == PacketType::KillAvatar) { + // this was not a replicated packet originally, we need to prepend the session ID + // for the killed node + packet->write(node->getUUID().toRfc4122()); + } + packet->write(message.getMessage()); } @@ -541,8 +548,6 @@ void AvatarMixer::handleAvatarIdentityPacket(QSharedPointer mes } auto end = usecTimestampNow(); _handleAvatarIdentityPacketElapsedTime += (end - start); - - optionallyReplicatePacket(*message, *senderNode); } void AvatarMixer::handleKillAvatarPacket(QSharedPointer message, SharedNodePointer node) { From ae8a0ab0e501363d0e88b1b83c14459ac1d86ae8 Mon Sep 17 00:00:00 2001 From: Stephen Birarda Date: Wed, 14 Jun 2017 11:29:08 -0700 Subject: [PATCH 27/46] kill packets already include killed node ID --- assignment-client/src/avatars/AvatarMixer.cpp | 7 ------- 1 file changed, 7 deletions(-) diff --git a/assignment-client/src/avatars/AvatarMixer.cpp b/assignment-client/src/avatars/AvatarMixer.cpp index 0c4b614033..cb328f025e 100644 --- a/assignment-client/src/avatars/AvatarMixer.cpp +++ b/assignment-client/src/avatars/AvatarMixer.cpp @@ -150,13 +150,6 @@ void AvatarMixer::optionallyReplicatePacket(ReceivedMessage& message, const Node if (!packet) { // construct an NLPacket to send to the replicant that has the contents of the received packet packet = NLPacket::create(replicatedType, message.getSize()); - - if (message.getType() == PacketType::KillAvatar) { - // this was not a replicated packet originally, we need to prepend the session ID - // for the killed node - packet->write(node->getUUID().toRfc4122()); - } - packet->write(message.getMessage()); } From 4a69bc5fc72c0f46857ddac8704f556962850bc3 Mon Sep 17 00:00:00 2001 From: Stephen Birarda Date: Wed, 14 Jun 2017 11:34:33 -0700 Subject: [PATCH 28/46] cleanup conditionals in for avatar mixer slave identity packets --- assignment-client/src/avatars/AvatarMixerSlave.cpp | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/assignment-client/src/avatars/AvatarMixerSlave.cpp b/assignment-client/src/avatars/AvatarMixerSlave.cpp index 6c50d5a626..143e246eb7 100644 --- a/assignment-client/src/avatars/AvatarMixerSlave.cpp +++ b/assignment-client/src/avatars/AvatarMixerSlave.cpp @@ -66,7 +66,7 @@ void AvatarMixerSlave::processIncomingPackets(const SharedNodePointer& node) { } int AvatarMixerSlave::sendIdentityPacket(const AvatarMixerClientData* nodeData, const SharedNodePointer& destinationNode) { - if (destinationNode->getType() == NodeType::DownstreamAvatarMixer || !destinationNode->isUpstream()) { + if (destinationNode->getType() == NodeType::Agent && !destinationNode->isUpstream()) { QByteArray individualData = nodeData->getConstAvatarData()->identityByteArray(); individualData.replace(0, NUM_BYTES_RFC4122_UUID, nodeData->getNodeID().toRfc4122()); // FIXME, this looks suspicious auto identityPackets = NLPacketList::create(PacketType::AvatarIdentity, QByteArray(), true, true); @@ -80,7 +80,7 @@ int AvatarMixerSlave::sendIdentityPacket(const AvatarMixerClientData* nodeData, } int AvatarMixerSlave::sendReplicatedIdentityPacket(const AvatarMixerClientData* nodeData, const SharedNodePointer& destinationNode) { - if (destinationNode->getType() == NodeType::DownstreamAvatarMixer || !destinationNode->isUpstream()) { + if (destinationNode->getType() == NodeType::DownstreamAvatarMixer) { QByteArray individualData = nodeData->getConstAvatarData()->identityByteArray(); individualData.replace(0, NUM_BYTES_RFC4122_UUID, nodeData->getNodeID().toRfc4122()); // FIXME, this looks suspicious auto identityPacket = NLPacket::create(PacketType::ReplicatedAvatarIdentity); From 91cef9bb7ce53f8233dde686b2ae09a90b1fdd7d Mon Sep 17 00:00:00 2001 From: Stephen Birarda Date: Wed, 14 Jun 2017 11:40:54 -0700 Subject: [PATCH 29/46] Add stats to node data for downstream mixers --- assignment-client/src/avatars/AvatarMixerSlave.cpp | 12 +++++++++++- assignment-client/src/avatars/AvatarMixerSlave.h | 3 +++ 2 files changed, 14 insertions(+), 1 deletion(-) diff --git a/assignment-client/src/avatars/AvatarMixerSlave.cpp b/assignment-client/src/avatars/AvatarMixerSlave.cpp index 143e246eb7..dd9f46224b 100644 --- a/assignment-client/src/avatars/AvatarMixerSlave.cpp +++ b/assignment-client/src/avatars/AvatarMixerSlave.cpp @@ -441,7 +441,7 @@ void AvatarMixerSlave::broadcastAvatarDataToAgent(const SharedNodePointer& node) uint64_t REBROADCAST_IDENTITY_TO_DOWNSTREAM_EVERY_US = 5 * 1000 * 1000; void AvatarMixerSlave::broadcastAvatarDataToDownstreamMixer(const SharedNodePointer& node) { - _stats.nodesBroadcastedTo++; + _stats.downstreamMixersBroadcastedTo++; AvatarMixerClientData* nodeData = reinterpret_cast(node->getLinkedData()); @@ -450,6 +450,9 @@ void AvatarMixerSlave::broadcastAvatarDataToDownstreamMixer(const SharedNodePoin int numAvatarDataBytes = 0; + // reset the number of sent avatars + nodeData->resetNumAvatarsSentLastFrame(); + std::for_each(_begin, _end, [&](const SharedNodePointer& agentNode) { // collect agents that we have avatar data for that we are supposed to replicate if (agentNode->getType() == NodeType::Agent && agentNode->getLinkedData() && agentNode->isReplicated()) { @@ -463,6 +466,7 @@ void AvatarMixerSlave::broadcastAvatarDataToDownstreamMixer(const SharedNodePoin // since we have no idea if they're online and receiving our packets // so we always send a full update for this avatar + quint64 start = usecTimestampNow(); AvatarDataPacket::HasFlags flagsOut; @@ -515,6 +519,9 @@ void AvatarMixerSlave::broadcastAvatarDataToDownstreamMixer(const SharedNodePoin nodeData->setLastBroadcastSequenceNumber(agentNode->getUUID(), agentNodeData->getLastReceivedSequenceNumber()); + // increment the number of avatars sent to this reciever + nodeData->incrementNumAvatarsSentLastFrame(); + // start a new segment in the packet list for this avatar avatarPacketList->startSegment(); @@ -549,6 +556,9 @@ void AvatarMixerSlave::broadcastAvatarDataToDownstreamMixer(const SharedNodePoin auto nodeList = DependencyManager::get(); nodeList->sendPacketList(std::move(avatarPacketList), node->getPublicSocket()); + // record the bytes sent for other avatar data in the AvatarMixerClientData + nodeData->recordSentAvatarData(numAvatarDataBytes); + quint64 endPacketSending = usecTimestampNow(); _stats.packetSendingElapsedTime += (endPacketSending - startPacketSending); } diff --git a/assignment-client/src/avatars/AvatarMixerSlave.h b/assignment-client/src/avatars/AvatarMixerSlave.h index 509a8ec94b..69c707bbf1 100644 --- a/assignment-client/src/avatars/AvatarMixerSlave.h +++ b/assignment-client/src/avatars/AvatarMixerSlave.h @@ -21,6 +21,7 @@ public: quint64 processIncomingPacketsElapsedTime { 0 }; int nodesBroadcastedTo { 0 }; + int downstreamMixersBroadcastedTo { 0 }; int numPacketsSent { 0 }; int numBytesSent { 0 }; int numIdentityPackets { 0 }; @@ -41,6 +42,7 @@ public: // sending job stats nodesBroadcastedTo = 0; + downstreamMixersBroadcastedTo = 0; numPacketsSent = 0; numBytesSent = 0; numIdentityPackets = 0; @@ -60,6 +62,7 @@ public: processIncomingPacketsElapsedTime += rhs.processIncomingPacketsElapsedTime; nodesBroadcastedTo += rhs.nodesBroadcastedTo; + downstreamMixersBroadcastedTo += rhs.downstreamMixersBroadcastedTo; numPacketsSent += rhs.numPacketsSent; numBytesSent += rhs.numBytesSent; numIdentityPackets += rhs.numIdentityPackets; From c132e29b0686b4c2abb665481038aeedca08dca0 Mon Sep 17 00:00:00 2001 From: Stephen Birarda Date: Wed, 14 Jun 2017 11:42:19 -0700 Subject: [PATCH 30/46] remove conditional seek from InboundAudioStream --- libraries/audio/src/InboundAudioStream.cpp | 11 +---------- 1 file changed, 1 insertion(+), 10 deletions(-) diff --git a/libraries/audio/src/InboundAudioStream.cpp b/libraries/audio/src/InboundAudioStream.cpp index 56353e14e3..da8997895c 100644 --- a/libraries/audio/src/InboundAudioStream.cpp +++ b/libraries/audio/src/InboundAudioStream.cpp @@ -128,16 +128,7 @@ int InboundAudioStream::parseData(ReceivedMessage& message) { int prePropertyPosition = message.getPosition(); int propertyBytes = parseStreamProperties(message.getType(), message.readWithoutCopy(message.getBytesLeftToRead()), networkFrames); - if (message.getType() == PacketType::ReplicatedMicrophoneAudioNoEcho - || message.getType() == PacketType::ReplicatedMicrophoneAudioWithEcho - || message.getType() == PacketType::ReplicatedInjectAudio - || message.getType() == PacketType::ReplicatedSilentAudioFrame) { - message.seek(NUM_BYTES_RFC4122_UUID); - message.readString(); - message.read(sizeof(quint16) + prePropertyPosition + propertyBytes); - } else { - message.seek(prePropertyPosition + propertyBytes); - } + message.seek(prePropertyPosition + propertyBytes); // handle this packet based on its arrival status. switch (arrivalInfo._status) { From 9be016fd3ea6dac28a4c0a1a5d41311e384ed4d2 Mon Sep 17 00:00:00 2001 From: Stephen Birarda Date: Wed, 14 Jun 2017 13:26:23 -0700 Subject: [PATCH 31/46] don't skip past UUID in identity packets --- assignment-client/src/avatars/AvatarMixer.cpp | 20 +++---------------- 1 file changed, 3 insertions(+), 17 deletions(-) diff --git a/assignment-client/src/avatars/AvatarMixer.cpp b/assignment-client/src/avatars/AvatarMixer.cpp index cb328f025e..337faf2027 100644 --- a/assignment-client/src/avatars/AvatarMixer.cpp +++ b/assignment-client/src/avatars/AvatarMixer.cpp @@ -83,25 +83,10 @@ void AvatarMixer::handleReplicatedPackets(QSharedPointer messag auto replicatedNode = addOrUpdateReplicatedNode(nodeID, message->getSenderSockAddr()); - // construct a "fake" avatar data received message from the byte array and packet list information - auto originalPacketData = message->getMessage().mid(NUM_BYTES_RFC4122_UUID); - - PacketType rewrittenType; - if (message->getType() == PacketType::ReplicatedAvatarIdentity) { - rewrittenType = PacketType::AvatarIdentity; + handleAvatarIdentityPacket(message, replicatedNode); } else if (message->getType() == PacketType::ReplicatedKillAvatar) { - rewrittenType = PacketType::KillAvatar; - } - - auto replicatedMessage = QSharedPointer::create(originalPacketData, rewrittenType, - versionForPacketType(rewrittenType), - message->getSenderSockAddr(), nodeID); - - if (rewrittenType == PacketType::AvatarIdentity) { - handleAvatarIdentityPacket(replicatedMessage, replicatedNode); - } else if (rewrittenType == PacketType::KillAvatar) { - handleKillAvatarPacket(replicatedMessage, replicatedNode); + handleKillAvatarPacket(message, replicatedNode); } } @@ -527,6 +512,7 @@ void AvatarMixer::handleAvatarIdentityPacket(QSharedPointer mes // parse the identity packet and update the change timestamp if appropriate AvatarData::Identity identity; AvatarData::parseAvatarIdentityPacket(message->getMessage(), identity); + bool identityChanged = false; bool displayNameChanged = false; avatar.processAvatarIdentity(identity, identityChanged, displayNameChanged); From 50b56ec7611ab6b1432578908eafa7c8756ec9de Mon Sep 17 00:00:00 2001 From: Stephen Birarda Date: Wed, 14 Jun 2017 14:02:22 -0700 Subject: [PATCH 32/46] don't unpack avatar identity that is old --- assignment-client/src/avatars/AvatarMixer.cpp | 5 +- libraries/avatars/src/AvatarData.cpp | 131 ++++++++++-------- libraries/avatars/src/AvatarData.h | 22 ++- libraries/avatars/src/AvatarHashMap.cpp | 16 ++- .../networking/src/udt/PacketHeaders.cpp | 2 +- libraries/networking/src/udt/PacketHeaders.h | 3 +- libraries/networking/src/udt/SequenceNumber.h | 4 +- 7 files changed, 98 insertions(+), 85 deletions(-) diff --git a/assignment-client/src/avatars/AvatarMixer.cpp b/assignment-client/src/avatars/AvatarMixer.cpp index 337faf2027..ffeba6a2e8 100644 --- a/assignment-client/src/avatars/AvatarMixer.cpp +++ b/assignment-client/src/avatars/AvatarMixer.cpp @@ -510,12 +510,9 @@ void AvatarMixer::handleAvatarIdentityPacket(QSharedPointer mes AvatarData& avatar = nodeData->getAvatar(); // parse the identity packet and update the change timestamp if appropriate - AvatarData::Identity identity; - AvatarData::parseAvatarIdentityPacket(message->getMessage(), identity); - bool identityChanged = false; bool displayNameChanged = false; - avatar.processAvatarIdentity(identity, identityChanged, displayNameChanged); + avatar.processAvatarIdentity(message->getMessage(), identityChanged, displayNameChanged); if (identityChanged) { QMutexLocker nodeDataLocker(&nodeData->getMutex()); nodeData->flagIdentityChange(); diff --git a/libraries/avatars/src/AvatarData.cpp b/libraries/avatars/src/AvatarData.cpp index 4407e12295..d523d1f5dc 100644 --- a/libraries/avatars/src/AvatarData.cpp +++ b/libraries/avatars/src/AvatarData.cpp @@ -1473,27 +1473,6 @@ QStringList AvatarData::getJointNames() const { return _jointNames; } -void AvatarData::parseAvatarIdentityPacket(const QByteArray& data, Identity& identityOut) { - QDataStream packetStream(data); - - packetStream >> identityOut.uuid - >> identityOut.skeletonModelURL - >> identityOut.attachmentData - >> identityOut.displayName - >> identityOut.sessionDisplayName - >> identityOut.avatarEntityData - >> identityOut.sequenceId; - -#ifdef WANT_DEBUG - qCDebug(avatars) << __FUNCTION__ - << "identityOut.uuid:" << identityOut.uuid - << "identityOut.skeletonModelURL:" << identityOut.skeletonModelURL - << "identityOut.displayName:" << identityOut.displayName - << "identityOut.sessionDisplayName:" << identityOut.sessionDisplayName; -#endif - -} - glm::quat AvatarData::getOrientationOutbound() const { return (getLocalOrientation()); } @@ -1504,46 +1483,80 @@ QUrl AvatarData::cannonicalSkeletonModelURL(const QUrl& emptyURL) const { return _skeletonModelURL.scheme() == "file" ? emptyURL : _skeletonModelURL; } -void AvatarData::processAvatarIdentity(const Identity& identity, bool& identityChanged, bool& displayNameChanged) { +void AvatarData::processAvatarIdentity(const QByteArray& identityData, bool& identityChanged, bool& displayNameChanged) { - if (identity.sequenceId < _identitySequenceId) { - qCDebug(avatars) << "Ignoring older identity packet for avatar" << getSessionUUID() - << "_identitySequenceId (" << _identitySequenceId << ") is greater than" << identity.sequenceId; - return; + QDataStream packetStream(identityData); + + QUuid avatarSessionID; + + // peek the sequence number, this will tell us if we should be processing this identity packet at all + udt::SequenceNumber::Type incomingSequenceNumberType; + packetStream >> avatarSessionID >> incomingSequenceNumberType; + udt::SequenceNumber incomingSequenceNumber(incomingSequenceNumberType); + + if (!_hasProcessedFirstIdentity) { + _identitySequenceNumber = incomingSequenceNumber - 1; + _hasProcessedFirstIdentity = true; + qCDebug(avatars) << "Processing first identity packet for" << avatarSessionID << "-" + << (udt::SequenceNumber::Type) incomingSequenceNumber; } - // otherwise, set the identitySequenceId to match the incoming identity - _identitySequenceId = identity.sequenceId; - if (_firstSkeletonCheck || (identity.skeletonModelURL != cannonicalSkeletonModelURL(emptyURL))) { - setSkeletonModelURL(identity.skeletonModelURL); - identityChanged = true; - if (_firstSkeletonCheck) { + if (incomingSequenceNumber > _identitySequenceNumber) { + Identity identity; + + packetStream >> identity.skeletonModelURL + >> identity.attachmentData + >> identity.displayName + >> identity.sessionDisplayName + >> identity.avatarEntityData; + + // set the store identity sequence number to match the incoming identity + _identitySequenceNumber = incomingSequenceNumber; + + if (_firstSkeletonCheck || (identity.skeletonModelURL != cannonicalSkeletonModelURL(emptyURL))) { + setSkeletonModelURL(identity.skeletonModelURL); + identityChanged = true; + if (_firstSkeletonCheck) { + displayNameChanged = true; + } + _firstSkeletonCheck = false; + } + + if (identity.displayName != _displayName) { + _displayName = identity.displayName; + identityChanged = true; displayNameChanged = true; } - _firstSkeletonCheck = false; - } + maybeUpdateSessionDisplayNameFromTransport(identity.sessionDisplayName); - if (identity.displayName != _displayName) { - _displayName = identity.displayName; - identityChanged = true; - displayNameChanged = true; - } - maybeUpdateSessionDisplayNameFromTransport(identity.sessionDisplayName); + if (identity.attachmentData != _attachmentData) { + setAttachmentData(identity.attachmentData); + identityChanged = true; + } - if (identity.attachmentData != _attachmentData) { - setAttachmentData(identity.attachmentData); - identityChanged = true; - } + bool avatarEntityDataChanged = false; + _avatarEntitiesLock.withReadLock([&] { + avatarEntityDataChanged = (identity.avatarEntityData != _avatarEntityData); + }); + + if (avatarEntityDataChanged) { + setAvatarEntityData(identity.avatarEntityData); + identityChanged = true; + } - bool avatarEntityDataChanged = false; - _avatarEntitiesLock.withReadLock([&] { - avatarEntityDataChanged = (identity.avatarEntityData != _avatarEntityData); - }); - if (avatarEntityDataChanged) { - setAvatarEntityData(identity.avatarEntityData); - identityChanged = true; - } +#ifdef WANT_DEBUG + qCDebug(avatars) << __FUNCTION__ + << "identity.uuid:" << identity.uuid + << "identity.skeletonModelURL:" << identity.skeletonModelURL + << "identity.displayName:" << identity.displayName + << "identity.sessionDisplayName:" << identity.sessionDisplayName; +#endif + } else { + qCDebug(avatars) << "Refusing to process identity for" << uuidStringWithoutCurlyBraces(avatarSessionID) << "since" + << (udt::SequenceNumber::Type) _identitySequenceNumber + << "is later than" << (udt::SequenceNumber::Type) incomingSequenceNumber; + } } QByteArray AvatarData::identityByteArray() const { @@ -1553,12 +1566,12 @@ QByteArray AvatarData::identityByteArray() const { _avatarEntitiesLock.withReadLock([&] { identityStream << getSessionUUID() - << urlToSend - << _attachmentData - << _displayName - << getSessionDisplayNameForTransport() // depends on _sessionDisplayName - << _avatarEntityData - << _identitySequenceId; + << (udt::SequenceNumber::Type) _identitySequenceNumber + << urlToSend + << _attachmentData + << _displayName + << getSessionDisplayNameForTransport() // depends on _sessionDisplayName + << _avatarEntityData; }); return identityData; @@ -2340,6 +2353,8 @@ void AvatarData::setAvatarEntityData(const AvatarEntityMap& avatarEntityData) { _avatarEntityData = avatarEntityData; setAvatarEntityDataChanged(true); + qDebug() << "Current avatar entity data is" << _avatarEntityData.keys(); + foreach (auto entityID, previousAvatarEntityIDs) { if (!_avatarEntityData.contains(entityID)) { _avatarEntityDetached.insert(entityID); diff --git a/libraries/avatars/src/AvatarData.h b/libraries/avatars/src/AvatarData.h index 4104615cfe..74aececa66 100644 --- a/libraries/avatars/src/AvatarData.h +++ b/libraries/avatars/src/AvatarData.h @@ -52,15 +52,16 @@ typedef unsigned long long quint64; #include #include #include -#include -#include -#include #include #include -#include +#include #include -#include +#include +#include +#include #include +#include +#include #include "AABox.h" #include "HeadData.h" @@ -525,20 +526,16 @@ public: const HeadData* getHeadData() const { return _headData; } struct Identity { - QUuid uuid; QUrl skeletonModelURL; QVector attachmentData; QString displayName; QString sessionDisplayName; AvatarEntityMap avatarEntityData; - quint64 sequenceId; }; - static void parseAvatarIdentityPacket(const QByteArray& data, Identity& identityOut); - // identityChanged returns true if identity has changed, false otherwise. // displayNameChanged returns true if displayName has changed, false otherwise. - void processAvatarIdentity(const Identity& identity, bool& identityChanged, bool& displayNameChanged); + void processAvatarIdentity(const QByteArray& identityData, bool& identityChanged, bool& displayNameChanged); QByteArray identityByteArray() const; @@ -626,7 +623,7 @@ public: bool getIdentityDataChanged() const { return _identityDataChanged; } // has the identity data changed since the last time sendIdentityPacket() was called void markIdentityDataChanged() { _identityDataChanged = true; - _identitySequenceId++; + ++_identitySequenceNumber; } float getDensity() const { return _density; } @@ -786,7 +783,8 @@ protected: float _audioAverageLoudness { 0.0f }; bool _identityDataChanged { false }; - quint64 _identitySequenceId { 0 }; + udt::SequenceNumber _identitySequenceNumber { 0 }; + bool _hasProcessedFirstIdentity { false }; float _density; private: diff --git a/libraries/avatars/src/AvatarHashMap.cpp b/libraries/avatars/src/AvatarHashMap.cpp index 2ccc64fee2..cfbf2a8806 100644 --- a/libraries/avatars/src/AvatarHashMap.cpp +++ b/libraries/avatars/src/AvatarHashMap.cpp @@ -126,8 +126,9 @@ AvatarSharedPointer AvatarHashMap::parseAvatarData(QSharedPointer message, SharedNodePointer sendingNode) { - AvatarData::Identity identity; - AvatarData::parseAvatarIdentityPacket(message->getMessage(), identity); + + // peek the avatar UUID from the incoming packet + QUuid identityUUID = message->peek(NUM_BYTES_RFC4122_UUID); // make sure this isn't for an ignored avatar auto nodeList = DependencyManager::get(); @@ -136,20 +137,21 @@ void AvatarHashMap::processAvatarIdentityPacket(QSharedPointer { QReadLocker locker(&_hashLock); auto me = _avatarHash.find(EMPTY); - if ((me != _avatarHash.end()) && (identity.uuid == me.value()->getSessionUUID())) { + if ((me != _avatarHash.end()) && (identityUUID == me.value()->getSessionUUID())) { // We add MyAvatar to _avatarHash with an empty UUID. Code relies on this. In order to correctly handle an // identity packet for ourself (such as when we are assigned a sessionDisplayName by the mixer upon joining), // we make things match here. - identity.uuid = EMPTY; + identityUUID = EMPTY; } } - if (!nodeList->isIgnoringNode(identity.uuid) || nodeList->getRequestsDomainListData()) { + + if (!nodeList->isIgnoringNode(identityUUID) || nodeList->getRequestsDomainListData()) { // mesh URL for a UUID, find avatar in our list - auto avatar = newOrExistingAvatar(identity.uuid, sendingNode); + auto avatar = newOrExistingAvatar(identityUUID, sendingNode); bool identityChanged = false; bool displayNameChanged = false; // In this case, the "sendingNode" is the Avatar Mixer. - avatar->processAvatarIdentity(identity, identityChanged, displayNameChanged); + avatar->processAvatarIdentity(message->getMessage(), identityChanged, displayNameChanged); } } diff --git a/libraries/networking/src/udt/PacketHeaders.cpp b/libraries/networking/src/udt/PacketHeaders.cpp index fc6251566e..7985df58bf 100644 --- a/libraries/networking/src/udt/PacketHeaders.cpp +++ b/libraries/networking/src/udt/PacketHeaders.cpp @@ -59,7 +59,7 @@ PacketVersion versionForPacketType(PacketType packetType) { case PacketType::AvatarData: case PacketType::BulkAvatarData: case PacketType::KillAvatar: - return static_cast(AvatarMixerPacketVersion::AvatarIdentitySequenceId); + return static_cast(AvatarMixerPacketVersion::AvatarIdentitySequenceFront); case PacketType::MessagesData: return static_cast(MessageDataVersion::TextOrBinaryData); case PacketType::ICEServerHeartbeat: diff --git a/libraries/networking/src/udt/PacketHeaders.h b/libraries/networking/src/udt/PacketHeaders.h index 9b37a7d76d..23cd3dd3e5 100644 --- a/libraries/networking/src/udt/PacketHeaders.h +++ b/libraries/networking/src/udt/PacketHeaders.h @@ -243,7 +243,8 @@ enum class AvatarMixerPacketVersion : PacketVersion { AvatarAsChildFixes, StickAndBallDefaultAvatar, IdentityPacketsIncludeUpdateTime, - AvatarIdentitySequenceId + AvatarIdentitySequenceId, + AvatarIdentitySequenceFront }; enum class DomainConnectRequestVersion : PacketVersion { diff --git a/libraries/networking/src/udt/SequenceNumber.h b/libraries/networking/src/udt/SequenceNumber.h index 3abc80bdd8..2c82eccfa3 100644 --- a/libraries/networking/src/udt/SequenceNumber.h +++ b/libraries/networking/src/udt/SequenceNumber.h @@ -35,8 +35,8 @@ public: explicit SequenceNumber(char* value) { _value = (*reinterpret_cast(value)) & MAX; } explicit SequenceNumber(Type value) { _value = (value <= MAX) ? ((value >= 0) ? value : 0) : MAX; } explicit SequenceNumber(UType value) { _value = (value <= MAX) ? value : MAX; } - explicit operator Type() { return _value; } - explicit operator UType() { return static_cast(_value); } + explicit operator Type() const { return _value; } + explicit operator UType() const { return static_cast(_value); } inline SequenceNumber& operator++() { _value = (_value + 1) % (MAX + 1); From ddb6db9f6a0b7020c7fe76680c07962a65f155fc Mon Sep 17 00:00:00 2001 From: Stephen Birarda Date: Wed, 14 Jun 2017 14:29:30 -0700 Subject: [PATCH 33/46] fix repeated pushing of avatar identity seq nums --- libraries/avatars/src/AvatarData.cpp | 21 +++++++++++++++------ libraries/avatars/src/AvatarData.h | 8 +++----- 2 files changed, 18 insertions(+), 11 deletions(-) diff --git a/libraries/avatars/src/AvatarData.cpp b/libraries/avatars/src/AvatarData.cpp index d523d1f5dc..b5bc27b64b 100644 --- a/libraries/avatars/src/AvatarData.cpp +++ b/libraries/avatars/src/AvatarData.cpp @@ -1495,15 +1495,18 @@ void AvatarData::processAvatarIdentity(const QByteArray& identityData, bool& ide udt::SequenceNumber incomingSequenceNumber(incomingSequenceNumberType); if (!_hasProcessedFirstIdentity) { - _identitySequenceNumber = incomingSequenceNumber - 1; + _lastIncomingSequenceNumber = incomingSequenceNumber - 1; _hasProcessedFirstIdentity = true; qCDebug(avatars) << "Processing first identity packet for" << avatarSessionID << "-" << (udt::SequenceNumber::Type) incomingSequenceNumber; } - if (incomingSequenceNumber > _identitySequenceNumber) { + if (incomingSequenceNumber > _lastIncomingSequenceNumber) { Identity identity; + qCDebug(avatars) << "Processing an identity packet from" << avatarSessionID + << "-" << (udt::SequenceNumber::Type) incomingSequenceNumber; + packetStream >> identity.skeletonModelURL >> identity.attachmentData >> identity.displayName @@ -1511,7 +1514,7 @@ void AvatarData::processAvatarIdentity(const QByteArray& identityData, bool& ide >> identity.avatarEntityData; // set the store identity sequence number to match the incoming identity - _identitySequenceNumber = incomingSequenceNumber; + _lastIncomingSequenceNumber = incomingSequenceNumber; if (_firstSkeletonCheck || (identity.skeletonModelURL != cannonicalSkeletonModelURL(emptyURL))) { setSkeletonModelURL(identity.skeletonModelURL); @@ -1554,7 +1557,7 @@ void AvatarData::processAvatarIdentity(const QByteArray& identityData, bool& ide } else { qCDebug(avatars) << "Refusing to process identity for" << uuidStringWithoutCurlyBraces(avatarSessionID) << "since" - << (udt::SequenceNumber::Type) _identitySequenceNumber + << (udt::SequenceNumber::Type) _lastIncomingSequenceNumber << "is later than" << (udt::SequenceNumber::Type) incomingSequenceNumber; } } @@ -1566,7 +1569,7 @@ QByteArray AvatarData::identityByteArray() const { _avatarEntitiesLock.withReadLock([&] { identityStream << getSessionUUID() - << (udt::SequenceNumber::Type) _identitySequenceNumber + << (udt::SequenceNumber::Type) _lastOutgoingSequenceNumber << urlToSend << _attachmentData << _displayName @@ -1747,6 +1750,12 @@ void AvatarData::sendAvatarDataPacket() { void AvatarData::sendIdentityPacket() { auto nodeList = DependencyManager::get(); + + if (_identityDataChanged) { + // if the identity data has changed, push the sequence number forwards + ++_lastOutgoingSequenceNumber; + } + QByteArray identityData = identityByteArray(); auto packetList = NLPacketList::create(PacketType::AvatarIdentity, QByteArray(), true, true); @@ -1757,7 +1766,7 @@ void AvatarData::sendIdentityPacket() { }, [&](const SharedNodePointer& node) { nodeList->sendPacketList(std::move(packetList), *node); - }); + }); _avatarEntityDataLocallyEdited = false; _identityDataChanged = false; diff --git a/libraries/avatars/src/AvatarData.h b/libraries/avatars/src/AvatarData.h index 74aececa66..2b65a44b9d 100644 --- a/libraries/avatars/src/AvatarData.h +++ b/libraries/avatars/src/AvatarData.h @@ -621,10 +621,7 @@ public: static float _avatarSortCoefficientAge; bool getIdentityDataChanged() const { return _identityDataChanged; } // has the identity data changed since the last time sendIdentityPacket() was called - void markIdentityDataChanged() { - _identityDataChanged = true; - ++_identitySequenceNumber; - } + void markIdentityDataChanged() { _identityDataChanged = true; } float getDensity() const { return _density; } @@ -783,7 +780,8 @@ protected: float _audioAverageLoudness { 0.0f }; bool _identityDataChanged { false }; - udt::SequenceNumber _identitySequenceNumber { 0 }; + udt::SequenceNumber _lastIncomingSequenceNumber { 0 }; + udt::SequenceNumber _lastOutgoingSequenceNumber { 0 }; bool _hasProcessedFirstIdentity { false }; float _density; From f2a61fa59dcfd1b9d1870745f244398eaadd2924 Mon Sep 17 00:00:00 2001 From: Stephen Birarda Date: Wed, 14 Jun 2017 14:41:25 -0700 Subject: [PATCH 34/46] change incoming/outgoing seq num handling --- libraries/avatars/src/AvatarData.cpp | 13 +++++++++++-- libraries/avatars/src/AvatarData.h | 2 +- 2 files changed, 12 insertions(+), 3 deletions(-) diff --git a/libraries/avatars/src/AvatarData.cpp b/libraries/avatars/src/AvatarData.cpp index b5bc27b64b..27d917c286 100644 --- a/libraries/avatars/src/AvatarData.cpp +++ b/libraries/avatars/src/AvatarData.cpp @@ -1562,14 +1562,23 @@ void AvatarData::processAvatarIdentity(const QByteArray& identityData, bool& ide } } -QByteArray AvatarData::identityByteArray() const { +QByteArray AvatarData::identityByteArray(bool shouldForwardIncomingSequenceNumber) const { QByteArray identityData; QDataStream identityStream(&identityData, QIODevice::Append); const QUrl& urlToSend = cannonicalSkeletonModelURL(emptyURL); // depends on _skeletonModelURL + // we use the boolean flag to determine if this is an identity byte array for a mixer to send to an agent + // or an agent to send to a mixer + + // when mixers send identity packets to agents, they simply forward along the last incoming sequence number they received + // whereas agents send a fresh outgoing sequence number when identity data has changed + + udt::SequenceNumber identitySequenceNumber = + shouldForwardIncomingSequenceNumber ? _lastIncomingSequenceNumber : _lastOutgoingSequenceNumber; + _avatarEntitiesLock.withReadLock([&] { identityStream << getSessionUUID() - << (udt::SequenceNumber::Type) _lastOutgoingSequenceNumber + << (udt::SequenceNumber::Type) identitySequenceNumber << urlToSend << _attachmentData << _displayName diff --git a/libraries/avatars/src/AvatarData.h b/libraries/avatars/src/AvatarData.h index 2b65a44b9d..2d43c4d412 100644 --- a/libraries/avatars/src/AvatarData.h +++ b/libraries/avatars/src/AvatarData.h @@ -537,7 +537,7 @@ public: // displayNameChanged returns true if displayName has changed, false otherwise. void processAvatarIdentity(const QByteArray& identityData, bool& identityChanged, bool& displayNameChanged); - QByteArray identityByteArray() const; + QByteArray identityByteArray(bool shouldForwardIncomingSequenceNumber = false) const; const QUrl& getSkeletonModelURL() const { return _skeletonModelURL; } const QString& getDisplayName() const { return _displayName; } From 9ee1cf88e652f71ad35ed0f05a2e7c4d86080137 Mon Sep 17 00:00:00 2001 From: Stephen Birarda Date: Wed, 14 Jun 2017 14:44:06 -0700 Subject: [PATCH 35/46] force forwarded sequence numbers from AvatarMixerSlave --- assignment-client/src/avatars/AvatarMixerSlave.cpp | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/assignment-client/src/avatars/AvatarMixerSlave.cpp b/assignment-client/src/avatars/AvatarMixerSlave.cpp index dd9f46224b..3f9e466256 100644 --- a/assignment-client/src/avatars/AvatarMixerSlave.cpp +++ b/assignment-client/src/avatars/AvatarMixerSlave.cpp @@ -67,7 +67,7 @@ void AvatarMixerSlave::processIncomingPackets(const SharedNodePointer& node) { int AvatarMixerSlave::sendIdentityPacket(const AvatarMixerClientData* nodeData, const SharedNodePointer& destinationNode) { if (destinationNode->getType() == NodeType::Agent && !destinationNode->isUpstream()) { - QByteArray individualData = nodeData->getConstAvatarData()->identityByteArray(); + QByteArray individualData = nodeData->getConstAvatarData()->identityByteArray(true); individualData.replace(0, NUM_BYTES_RFC4122_UUID, nodeData->getNodeID().toRfc4122()); // FIXME, this looks suspicious auto identityPackets = NLPacketList::create(PacketType::AvatarIdentity, QByteArray(), true, true); identityPackets->write(individualData); @@ -81,7 +81,7 @@ int AvatarMixerSlave::sendIdentityPacket(const AvatarMixerClientData* nodeData, int AvatarMixerSlave::sendReplicatedIdentityPacket(const AvatarMixerClientData* nodeData, const SharedNodePointer& destinationNode) { if (destinationNode->getType() == NodeType::DownstreamAvatarMixer) { - QByteArray individualData = nodeData->getConstAvatarData()->identityByteArray(); + QByteArray individualData = nodeData->getConstAvatarData()->identityByteArray(true); individualData.replace(0, NUM_BYTES_RFC4122_UUID, nodeData->getNodeID().toRfc4122()); // FIXME, this looks suspicious auto identityPacket = NLPacket::create(PacketType::ReplicatedAvatarIdentity); identityPacket->write(individualData); From 5b45beaa3904bd82e16aa315f52643cc37125d7b Mon Sep 17 00:00:00 2001 From: Stephen Birarda Date: Wed, 14 Jun 2017 14:51:28 -0700 Subject: [PATCH 36/46] cleanup some debug from identity packet changes --- libraries/avatars/src/AvatarData.cpp | 5 +---- 1 file changed, 1 insertion(+), 4 deletions(-) diff --git a/libraries/avatars/src/AvatarData.cpp b/libraries/avatars/src/AvatarData.cpp index 27d917c286..97b9071b84 100644 --- a/libraries/avatars/src/AvatarData.cpp +++ b/libraries/avatars/src/AvatarData.cpp @@ -1504,9 +1504,6 @@ void AvatarData::processAvatarIdentity(const QByteArray& identityData, bool& ide if (incomingSequenceNumber > _lastIncomingSequenceNumber) { Identity identity; - qCDebug(avatars) << "Processing an identity packet from" << avatarSessionID - << "-" << (udt::SequenceNumber::Type) incomingSequenceNumber; - packetStream >> identity.skeletonModelURL >> identity.attachmentData >> identity.displayName @@ -1558,7 +1555,7 @@ void AvatarData::processAvatarIdentity(const QByteArray& identityData, bool& ide } else { qCDebug(avatars) << "Refusing to process identity for" << uuidStringWithoutCurlyBraces(avatarSessionID) << "since" << (udt::SequenceNumber::Type) _lastIncomingSequenceNumber - << "is later than" << (udt::SequenceNumber::Type) incomingSequenceNumber; + << "is >=" << (udt::SequenceNumber::Type) incomingSequenceNumber; } } From e3c3831efa7f072712103f46c023a145c3feae74 Mon Sep 17 00:00:00 2001 From: Stephen Birarda Date: Wed, 14 Jun 2017 14:56:15 -0700 Subject: [PATCH 37/46] only output sequence number information if debug is requested --- libraries/avatars/src/AvatarData.cpp | 2 ++ 1 file changed, 2 insertions(+) diff --git a/libraries/avatars/src/AvatarData.cpp b/libraries/avatars/src/AvatarData.cpp index 97b9071b84..99e1767806 100644 --- a/libraries/avatars/src/AvatarData.cpp +++ b/libraries/avatars/src/AvatarData.cpp @@ -1553,9 +1553,11 @@ void AvatarData::processAvatarIdentity(const QByteArray& identityData, bool& ide #endif } else { +#ifdef WANT_DEBUG qCDebug(avatars) << "Refusing to process identity for" << uuidStringWithoutCurlyBraces(avatarSessionID) << "since" << (udt::SequenceNumber::Type) _lastIncomingSequenceNumber << "is >=" << (udt::SequenceNumber::Type) incomingSequenceNumber; +#endif } } From bfa5df778d424b501c65b117522f3e091e97bc39 Mon Sep 17 00:00:00 2001 From: Stephen Birarda Date: Wed, 14 Jun 2017 14:59:49 -0700 Subject: [PATCH 38/46] remove avatar entity data debug --- libraries/avatars/src/AvatarData.cpp | 2 -- 1 file changed, 2 deletions(-) diff --git a/libraries/avatars/src/AvatarData.cpp b/libraries/avatars/src/AvatarData.cpp index 99e1767806..3c048e590d 100644 --- a/libraries/avatars/src/AvatarData.cpp +++ b/libraries/avatars/src/AvatarData.cpp @@ -2370,8 +2370,6 @@ void AvatarData::setAvatarEntityData(const AvatarEntityMap& avatarEntityData) { _avatarEntityData = avatarEntityData; setAvatarEntityDataChanged(true); - qDebug() << "Current avatar entity data is" << _avatarEntityData.keys(); - foreach (auto entityID, previousAvatarEntityIDs) { if (!_avatarEntityData.contains(entityID)) { _avatarEntityDetached.insert(entityID); From 8d5dc21de8d75a2e5abfc5d0206f44a064dfc24d Mon Sep 17 00:00:00 2001 From: Ryan Huffman Date: Wed, 14 Jun 2017 13:53:50 -0700 Subject: [PATCH 39/46] Fix replicated nodes not being properly updated when logging in or changing settings --- domain-server/src/DomainServer.cpp | 19 +++++++++++++------ 1 file changed, 13 insertions(+), 6 deletions(-) diff --git a/domain-server/src/DomainServer.cpp b/domain-server/src/DomainServer.cpp index 65f85215cb..dddc1c2942 100644 --- a/domain-server/src/DomainServer.cpp +++ b/domain-server/src/DomainServer.cpp @@ -2234,17 +2234,24 @@ void DomainServer::updateReplicatedNodes() { } auto nodeList = DependencyManager::get(); - nodeList->eachNode([&](const SharedNodePointer& otherNode) { - if (shouldReplicateNode(*otherNode)) { - qDebug() << "Setting node to replicated: " << otherNode->getUUID(); - otherNode->setIsReplicated(true); + nodeList->eachMatchingNode([this](const SharedNodePointer& otherNode) -> bool { + return otherNode->getType() == NodeType::Agent; + }, [this](const SharedNodePointer& otherNode) { + auto shouldReplicate = shouldReplicateNode(*otherNode); + auto isReplicated = otherNode->isReplicated(); + qDebug() << "Checking " << otherNode->getPermissions().getVerifiedUserName(); + if (isReplicated && !shouldReplicate) { + qDebug() << "Setting node to NOT be replicated: " << otherNode->getUUID(); + } else if (!isReplicated && shouldReplicate) { + qDebug() << "Setting node to replicated: " << otherNode->getUUID(); + } + otherNode->setIsReplicated(shouldReplicate); } - }); + ); } bool DomainServer::shouldReplicateNode(const Node& node) { QString verifiedUsername = node.getPermissions().getVerifiedUserName(); - qDebug() << "Verified username: " << verifiedUsername; auto it = find(_replicatedUsernames.cbegin(), _replicatedUsernames.cend(), verifiedUsername); return it != _replicatedUsernames.end() && node.getType() == NodeType::Agent; }; From 67113330b3f42e12af4e469e1b89f994704ba011 Mon Sep 17 00:00:00 2001 From: Ryan Huffman Date: Wed, 14 Jun 2017 13:57:04 -0700 Subject: [PATCH 40/46] Make replicated username list case insensitive --- domain-server/src/DomainServer.cpp | 4 +++- 1 file changed, 3 insertions(+), 1 deletion(-) diff --git a/domain-server/src/DomainServer.cpp b/domain-server/src/DomainServer.cpp index dddc1c2942..5643a4434a 100644 --- a/domain-server/src/DomainServer.cpp +++ b/domain-server/src/DomainServer.cpp @@ -2228,7 +2228,7 @@ void DomainServer::updateReplicatedNodes() { if (replicationSettings.contains("users")) { auto usersSettings = replicationSettings.value("users").toList(); for (auto& username : usersSettings) { - _replicatedUsernames.push_back(username.toString()); + _replicatedUsernames.push_back(username.toString().toLower()); } } } @@ -2252,6 +2252,8 @@ void DomainServer::updateReplicatedNodes() { bool DomainServer::shouldReplicateNode(const Node& node) { QString verifiedUsername = node.getPermissions().getVerifiedUserName(); + // Both he verified username and usernames in _replicatedUsernames are lowercase, so + // comparisons here are case-insensitive. auto it = find(_replicatedUsernames.cbegin(), _replicatedUsernames.cend(), verifiedUsername); return it != _replicatedUsernames.end() && node.getType() == NodeType::Agent; }; From 54832d995df19ebdb5d7557c8dffc4606fc9fa75 Mon Sep 17 00:00:00 2001 From: Stephen Birarda Date: Wed, 14 Jun 2017 15:05:58 -0700 Subject: [PATCH 41/46] fix a typo in shouldReplicateNode --- domain-server/src/DomainServer.cpp | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/domain-server/src/DomainServer.cpp b/domain-server/src/DomainServer.cpp index 5643a4434a..1afc48af4c 100644 --- a/domain-server/src/DomainServer.cpp +++ b/domain-server/src/DomainServer.cpp @@ -2252,7 +2252,7 @@ void DomainServer::updateReplicatedNodes() { bool DomainServer::shouldReplicateNode(const Node& node) { QString verifiedUsername = node.getPermissions().getVerifiedUserName(); - // Both he verified username and usernames in _replicatedUsernames are lowercase, so + // Both the verified username and usernames in _replicatedUsernames are lowercase, so // comparisons here are case-insensitive. auto it = find(_replicatedUsernames.cbegin(), _replicatedUsernames.cend(), verifiedUsername); return it != _replicatedUsernames.end() && node.getType() == NodeType::Agent; From e818a511b908f69c62cb4d804493fc2c0e57a0ca Mon Sep 17 00:00:00 2001 From: Stephen Birarda Date: Wed, 14 Jun 2017 15:07:48 -0700 Subject: [PATCH 42/46] cleanup debug in updateReplicatedNodes --- domain-server/src/DomainServer.cpp | 7 ++++--- 1 file changed, 4 insertions(+), 3 deletions(-) diff --git a/domain-server/src/DomainServer.cpp b/domain-server/src/DomainServer.cpp index 1afc48af4c..0c0f8a5e55 100644 --- a/domain-server/src/DomainServer.cpp +++ b/domain-server/src/DomainServer.cpp @@ -2239,11 +2239,12 @@ void DomainServer::updateReplicatedNodes() { }, [this](const SharedNodePointer& otherNode) { auto shouldReplicate = shouldReplicateNode(*otherNode); auto isReplicated = otherNode->isReplicated(); - qDebug() << "Checking " << otherNode->getPermissions().getVerifiedUserName(); if (isReplicated && !shouldReplicate) { - qDebug() << "Setting node to NOT be replicated: " << otherNode->getUUID(); + qDebug() << "Setting node to NOT be replicated:" + << otherNode->getPermissions().getVerifiedUserName() << otherNode->getUUID(); } else if (!isReplicated && shouldReplicate) { - qDebug() << "Setting node to replicated: " << otherNode->getUUID(); + qDebug() << "Setting node to replicated:" + << otherNode->getPermissions().getVerifiedUserName() << otherNode->getUUID(); } otherNode->setIsReplicated(shouldReplicate); } From 9407a65f8046027e7ab52ef83e7a3caa455a4d2a Mon Sep 17 00:00:00 2001 From: Stephen Birarda Date: Wed, 14 Jun 2017 15:12:23 -0700 Subject: [PATCH 43/46] peek the nodeID to keep message in right place --- assignment-client/src/avatars/AvatarMixer.cpp | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/assignment-client/src/avatars/AvatarMixer.cpp b/assignment-client/src/avatars/AvatarMixer.cpp index ffeba6a2e8..32222a6406 100644 --- a/assignment-client/src/avatars/AvatarMixer.cpp +++ b/assignment-client/src/avatars/AvatarMixer.cpp @@ -79,7 +79,7 @@ SharedNodePointer addOrUpdateReplicatedNode(const QUuid& nodeID, const HifiSockA void AvatarMixer::handleReplicatedPackets(QSharedPointer message) { auto nodeList = DependencyManager::get(); - auto nodeID = QUuid::fromRfc4122(message->readWithoutCopy(NUM_BYTES_RFC4122_UUID)); + auto nodeID = QUuid::fromRfc4122(message->peek(NUM_BYTES_RFC4122_UUID)); auto replicatedNode = addOrUpdateReplicatedNode(nodeID, message->getSenderSockAddr()); From afc9978841d40dbe40d72d497f594ce50ce7143b Mon Sep 17 00:00:00 2001 From: Stephen Birarda Date: Wed, 14 Jun 2017 15:20:47 -0700 Subject: [PATCH 44/46] add a comment for hop over sequence in AudioMixerClientData --- assignment-client/src/audio/AudioMixerClientData.cpp | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/assignment-client/src/audio/AudioMixerClientData.cpp b/assignment-client/src/audio/AudioMixerClientData.cpp index d4d098133d..7641000a7a 100644 --- a/assignment-client/src/audio/AudioMixerClientData.cpp +++ b/assignment-client/src/audio/AudioMixerClientData.cpp @@ -691,9 +691,10 @@ bool AudioMixerClientData::shouldIgnore(const SharedNodePointer self, const Shar } void AudioMixerClientData::setupCodecForReplicatedAgent(QSharedPointer message) { - // pull the codec string from the packet + // hop past the sequence number that leads the packet message->seek(sizeof(quint16)); + // pull the codec string from the packet auto codecString = message->readString(); qDebug() << "Manually setting codec for replicated agent" << uuidStringWithoutCurlyBraces(getNodeID()) From 4c2fa4be64a3a1ba083c38cbad529981262b34cd Mon Sep 17 00:00:00 2001 From: Stephen Birarda Date: Wed, 14 Jun 2017 15:53:24 -0700 Subject: [PATCH 45/46] fix indentation in AvatarMixerSlave/AvatarMixerSlavePool --- .../src/avatars/AvatarMixerSlave.cpp | 356 +++++++++--------- .../src/avatars/AvatarMixerSlavePool.cpp | 4 +- 2 files changed, 178 insertions(+), 182 deletions(-) diff --git a/assignment-client/src/avatars/AvatarMixerSlave.cpp b/assignment-client/src/avatars/AvatarMixerSlave.cpp index 3f9e466256..e5a0a1f4fd 100644 --- a/assignment-client/src/avatars/AvatarMixerSlave.cpp +++ b/assignment-client/src/avatars/AvatarMixerSlave.cpp @@ -204,216 +204,212 @@ void AvatarMixerSlave::broadcastAvatarDataToAgent(const SharedNodePointer& node) ViewFrustum cameraView = nodeData->getViewFrustom(); std::priority_queue sortedAvatars; AvatarData::sortAvatars(avatarList, cameraView, sortedAvatars, - [&](AvatarSharedPointer avatar)->uint64_t{ - auto avatarNode = avatarDataToNodes[avatar]; - assert(avatarNode); // we can't have gotten here without the avatarData being a valid key in the map - return nodeData->getLastBroadcastTime(avatarNode->getUUID()); - }, + [&](AvatarSharedPointer avatar)->uint64_t { + auto avatarNode = avatarDataToNodes[avatar]; + assert(avatarNode); // we can't have gotten here without the avatarData being a valid key in the map + return nodeData->getLastBroadcastTime(avatarNode->getUUID()); + }, [&](AvatarSharedPointer avatar)->float{ + glm::vec3 nodeBoxHalfScale = (avatar->getPosition() - avatar->getGlobalBoundingBoxCorner()); + return glm::max(nodeBoxHalfScale.x, glm::max(nodeBoxHalfScale.y, nodeBoxHalfScale.z)); + }, [&](AvatarSharedPointer avatar)->bool { + if (avatar == thisAvatar) { + return true; // ignore ourselves... + } - [&](AvatarSharedPointer avatar)->float{ - glm::vec3 nodeBoxHalfScale = (avatar->getPosition() - avatar->getGlobalBoundingBoxCorner()); - return glm::max(nodeBoxHalfScale.x, glm::max(nodeBoxHalfScale.y, nodeBoxHalfScale.z)); - }, + bool shouldIgnore = false; - [&](AvatarSharedPointer avatar)->bool{ - if (avatar == thisAvatar) { - return true; // ignore ourselves... - } + // We will also ignore other nodes for a couple of different reasons: + // 1) ignore bubbles and ignore specific node + // 2) the node hasn't really updated it's frame data recently, this can + // happen if for example the avatar is connected on a desktop and sending + // updates at ~30hz. So every 3 frames we skip a frame. + auto avatarNode = avatarDataToNodes[avatar]; - bool shouldIgnore = false; + assert(avatarNode); // we can't have gotten here without the avatarData being a valid key in the map - // We will also ignore other nodes for a couple of different reasons: - // 1) ignore bubbles and ignore specific node - // 2) the node hasn't really updated it's frame data recently, this can - // happen if for example the avatar is connected on a desktop and sending - // updates at ~30hz. So every 3 frames we skip a frame. - auto avatarNode = avatarDataToNodes[avatar]; + const AvatarMixerClientData* avatarNodeData = reinterpret_cast(avatarNode->getLinkedData()); + assert(avatarNodeData); // we can't have gotten here without avatarNode having valid data + quint64 startIgnoreCalculation = usecTimestampNow(); - assert(avatarNode); // we can't have gotten here without the avatarData being a valid key in the map + // make sure we have data for this avatar, that it isn't the same node, + // and isn't an avatar that the viewing node has ignored + // or that has ignored the viewing node + if (!avatarNode->getLinkedData() + || avatarNode->getUUID() == node->getUUID() + || (node->isIgnoringNodeWithID(avatarNode->getUUID()) && !PALIsOpen) + || (avatarNode->isIgnoringNodeWithID(node->getUUID()) && !getsAnyIgnored)) { + shouldIgnore = true; + } else { - const AvatarMixerClientData* avatarNodeData = reinterpret_cast(avatarNode->getLinkedData()); - assert(avatarNodeData); // we can't have gotten here without avatarNode having valid data - quint64 startIgnoreCalculation = usecTimestampNow(); + // Check to see if the space bubble is enabled + // Don't bother with these checks if the other avatar has their bubble enabled and we're gettingAnyIgnored + if (node->isIgnoreRadiusEnabled() || (avatarNode->isIgnoreRadiusEnabled() && !getsAnyIgnored)) { - // make sure we have data for this avatar, that it isn't the same node, - // and isn't an avatar that the viewing node has ignored - // or that has ignored the viewing node - if (!avatarNode->getLinkedData() - || avatarNode->getUUID() == node->getUUID() - || (node->isIgnoringNodeWithID(avatarNode->getUUID()) && !PALIsOpen) - || (avatarNode->isIgnoringNodeWithID(node->getUUID()) && !getsAnyIgnored)) { - shouldIgnore = true; - } else { - - // Check to see if the space bubble is enabled - // Don't bother with these checks if the other avatar has their bubble enabled and we're gettingAnyIgnored - if (node->isIgnoreRadiusEnabled() || (avatarNode->isIgnoreRadiusEnabled() && !getsAnyIgnored)) { - - // Define the scale of the box for the current other node - glm::vec3 otherNodeBoxScale = (avatarNodeData->getPosition() - avatarNodeData->getGlobalBoundingBoxCorner()) * 2.0f; - // Set up the bounding box for the current other node - AABox otherNodeBox(avatarNodeData->getGlobalBoundingBoxCorner(), otherNodeBoxScale); - // Clamp the size of the bounding box to a minimum scale - if (glm::any(glm::lessThan(otherNodeBoxScale, minBubbleSize))) { - otherNodeBox.setScaleStayCentered(minBubbleSize); - } - // Quadruple the scale of both bounding boxes - otherNodeBox.embiggen(4.0f); - - // Perform the collision check between the two bounding boxes - if (nodeBox.touches(otherNodeBox)) { - nodeData->ignoreOther(node, avatarNode); - shouldIgnore = !getsAnyIgnored; - } + // Define the scale of the box for the current other node + glm::vec3 otherNodeBoxScale = (avatarNodeData->getPosition() - avatarNodeData->getGlobalBoundingBoxCorner()) * 2.0f; + // Set up the bounding box for the current other node + AABox otherNodeBox(avatarNodeData->getGlobalBoundingBoxCorner(), otherNodeBoxScale); + // Clamp the size of the bounding box to a minimum scale + if (glm::any(glm::lessThan(otherNodeBoxScale, minBubbleSize))) { + otherNodeBox.setScaleStayCentered(minBubbleSize); } - // Not close enough to ignore - if (!shouldIgnore) { - nodeData->removeFromRadiusIgnoringSet(node, avatarNode->getUUID()); + // Quadruple the scale of both bounding boxes + otherNodeBox.embiggen(4.0f); + + // Perform the collision check between the two bounding boxes + if (nodeBox.touches(otherNodeBox)) { + nodeData->ignoreOther(node, avatarNode); + shouldIgnore = !getsAnyIgnored; } } - quint64 endIgnoreCalculation = usecTimestampNow(); - _stats.ignoreCalculationElapsedTime += (endIgnoreCalculation - startIgnoreCalculation); - + // Not close enough to ignore if (!shouldIgnore) { - AvatarDataSequenceNumber lastSeqToReceiver = nodeData->getLastBroadcastSequenceNumber(avatarNode->getUUID()); - AvatarDataSequenceNumber lastSeqFromSender = avatarNodeData->getLastReceivedSequenceNumber(); - - // FIXME - This code does appear to be working. But it seems brittle. - // It supports determining if the frame of data for this "other" - // avatar has already been sent to the reciever. This has been - // verified to work on a desktop display that renders at 60hz and - // therefore sends to mixer at 30hz. Each second you'd expect to - // have 15 (45hz-30hz) duplicate frames. In this case, the stat - // avg_other_av_skips_per_second does report 15. - // - // make sure we haven't already sent this data from this sender to this receiver - // or that somehow we haven't sent - if (lastSeqToReceiver == lastSeqFromSender && lastSeqToReceiver != 0) { - ++numAvatarsHeldBack; - shouldIgnore = true; - } else if (lastSeqFromSender - lastSeqToReceiver > 1) { - // this is a skip - we still send the packet but capture the presence of the skip so we see it happening - ++numAvatarsWithSkippedFrames; - } + nodeData->removeFromRadiusIgnoringSet(node, avatarNode->getUUID()); } - return shouldIgnore; - }); + } + quint64 endIgnoreCalculation = usecTimestampNow(); + _stats.ignoreCalculationElapsedTime += (endIgnoreCalculation - startIgnoreCalculation); - // loop through our sorted avatars and allocate our bandwidth to them accordingly - int avatarRank = 0; + if (!shouldIgnore) { + AvatarDataSequenceNumber lastSeqToReceiver = nodeData->getLastBroadcastSequenceNumber(avatarNode->getUUID()); + AvatarDataSequenceNumber lastSeqFromSender = avatarNodeData->getLastReceivedSequenceNumber(); - // this is overly conservative, because it includes some avatars we might not consider - int remainingAvatars = (int)sortedAvatars.size(); - - while (!sortedAvatars.empty()) { - AvatarPriority sortData = sortedAvatars.top(); - sortedAvatars.pop(); - const auto& avatarData = sortData.avatar; - avatarRank++; - remainingAvatars--; - - auto otherNode = avatarDataToNodes[avatarData]; - assert(otherNode); // we can't have gotten here without the avatarData being a valid key in the map - - // NOTE: Here's where we determine if we are over budget and drop to bare minimum data - int minimRemainingAvatarBytes = minimumBytesPerAvatar * remainingAvatars; - bool overBudget = (identityBytesSent + numAvatarDataBytes + minimRemainingAvatarBytes) > maxAvatarBytesPerFrame; - - quint64 startAvatarDataPacking = usecTimestampNow(); - - ++numOtherAvatars; - - const AvatarMixerClientData* otherNodeData = reinterpret_cast(otherNode->getLinkedData()); - - // If the time that the mixer sent AVATAR DATA about Avatar B to Avatar A is BEFORE OR EQUAL TO - // the time that Avatar B flagged an IDENTITY DATA change, send IDENTITY DATA about Avatar B to Avatar A. - if (nodeData->getLastBroadcastTime(otherNode->getUUID()) <= otherNodeData->getIdentityChangeTimestamp()) { - identityBytesSent += sendIdentityPacket(otherNodeData, node); + // FIXME - This code does appear to be working. But it seems brittle. + // It supports determining if the frame of data for this "other" + // avatar has already been sent to the reciever. This has been + // verified to work on a desktop display that renders at 60hz and + // therefore sends to mixer at 30hz. Each second you'd expect to + // have 15 (45hz-30hz) duplicate frames. In this case, the stat + // avg_other_av_skips_per_second does report 15. + // + // make sure we haven't already sent this data from this sender to this receiver + // or that somehow we haven't sent + if (lastSeqToReceiver == lastSeqFromSender && lastSeqToReceiver != 0) { + ++numAvatarsHeldBack; + shouldIgnore = true; + } else if (lastSeqFromSender - lastSeqToReceiver > 1) { + // this is a skip - we still send the packet but capture the presence of the skip so we see it happening + ++numAvatarsWithSkippedFrames; } + } + return shouldIgnore; + }); - const AvatarData* otherAvatar = otherNodeData->getConstAvatarData(); - glm::vec3 otherPosition = otherAvatar->getClientGlobalPosition(); + // loop through our sorted avatars and allocate our bandwidth to them accordingly + int avatarRank = 0; - // determine if avatar is in view, to determine how much data to include... - glm::vec3 otherNodeBoxScale = (otherPosition - otherNodeData->getGlobalBoundingBoxCorner()) * 2.0f; - AABox otherNodeBox(otherNodeData->getGlobalBoundingBoxCorner(), otherNodeBoxScale); - bool isInView = nodeData->otherAvatarInView(otherNodeBox); + // this is overly conservative, because it includes some avatars we might not consider + int remainingAvatars = (int)sortedAvatars.size(); - // start a new segment in the PacketList for this avatar - avatarPacketList->startSegment(); + while (!sortedAvatars.empty()) { + AvatarPriority sortData = sortedAvatars.top(); + sortedAvatars.pop(); + const auto& avatarData = sortData.avatar; + avatarRank++; + remainingAvatars--; - AvatarData::AvatarDataDetail detail; + auto otherNode = avatarDataToNodes[avatarData]; + assert(otherNode); // we can't have gotten here without the avatarData being a valid key in the map - if (overBudget) { - overBudgetAvatars++; - _stats.overBudgetAvatars++; - detail = PALIsOpen ? AvatarData::PALMinimum : AvatarData::NoData; - } else if (!isInView) { - detail = PALIsOpen ? AvatarData::PALMinimum : AvatarData::MinimumData; - nodeData->incrementAvatarOutOfView(); - } else { - detail = distribution(generator) < AVATAR_SEND_FULL_UPDATE_RATIO - ? AvatarData::SendAllData : AvatarData::CullSmallData; - nodeData->incrementAvatarInView(); - } + // NOTE: Here's where we determine if we are over budget and drop to bare minimum data + int minimRemainingAvatarBytes = minimumBytesPerAvatar * remainingAvatars; + bool overBudget = (identityBytesSent + numAvatarDataBytes + minimRemainingAvatarBytes) > maxAvatarBytesPerFrame; - bool includeThisAvatar = true; - auto lastEncodeForOther = nodeData->getLastOtherAvatarEncodeTime(otherNode->getUUID()); - QVector& lastSentJointsForOther = nodeData->getLastOtherAvatarSentJoints(otherNode->getUUID()); - bool distanceAdjust = true; - glm::vec3 viewerPosition = myPosition; - AvatarDataPacket::HasFlags hasFlagsOut; // the result of the toByteArray - bool dropFaceTracking = false; + quint64 startAvatarDataPacking = usecTimestampNow(); - quint64 start = usecTimestampNow(); - QByteArray bytes = otherAvatar->toByteArray(detail, lastEncodeForOther, lastSentJointsForOther, - hasFlagsOut, dropFaceTracking, distanceAdjust, viewerPosition, &lastSentJointsForOther); - quint64 end = usecTimestampNow(); - _stats.toByteArrayElapsedTime += (end - start); + ++numOtherAvatars; + + const AvatarMixerClientData* otherNodeData = reinterpret_cast(otherNode->getLinkedData()); + + // If the time that the mixer sent AVATAR DATA about Avatar B to Avatar A is BEFORE OR EQUAL TO + // the time that Avatar B flagged an IDENTITY DATA change, send IDENTITY DATA about Avatar B to Avatar A. + if (nodeData->getLastBroadcastTime(otherNode->getUUID()) <= otherNodeData->getIdentityChangeTimestamp()) { + identityBytesSent += sendIdentityPacket(otherNodeData, node); + } + + const AvatarData* otherAvatar = otherNodeData->getConstAvatarData(); + glm::vec3 otherPosition = otherAvatar->getClientGlobalPosition(); + + // determine if avatar is in view, to determine how much data to include... + glm::vec3 otherNodeBoxScale = (otherPosition - otherNodeData->getGlobalBoundingBoxCorner()) * 2.0f; + AABox otherNodeBox(otherNodeData->getGlobalBoundingBoxCorner(), otherNodeBoxScale); + bool isInView = nodeData->otherAvatarInView(otherNodeBox); + + // start a new segment in the PacketList for this avatar + avatarPacketList->startSegment(); + + AvatarData::AvatarDataDetail detail; + + if (overBudget) { + overBudgetAvatars++; + _stats.overBudgetAvatars++; + detail = PALIsOpen ? AvatarData::PALMinimum : AvatarData::NoData; + } else if (!isInView) { + detail = PALIsOpen ? AvatarData::PALMinimum : AvatarData::MinimumData; + nodeData->incrementAvatarOutOfView(); + } else { + detail = distribution(generator) < AVATAR_SEND_FULL_UPDATE_RATIO + ? AvatarData::SendAllData : AvatarData::CullSmallData; + nodeData->incrementAvatarInView(); + } + + bool includeThisAvatar = true; + auto lastEncodeForOther = nodeData->getLastOtherAvatarEncodeTime(otherNode->getUUID()); + QVector& lastSentJointsForOther = nodeData->getLastOtherAvatarSentJoints(otherNode->getUUID()); + bool distanceAdjust = true; + glm::vec3 viewerPosition = myPosition; + AvatarDataPacket::HasFlags hasFlagsOut; // the result of the toByteArray + bool dropFaceTracking = false; + + quint64 start = usecTimestampNow(); + QByteArray bytes = otherAvatar->toByteArray(detail, lastEncodeForOther, lastSentJointsForOther, + hasFlagsOut, dropFaceTracking, distanceAdjust, viewerPosition, &lastSentJointsForOther); + quint64 end = usecTimestampNow(); + _stats.toByteArrayElapsedTime += (end - start); + + static const int MAX_ALLOWED_AVATAR_DATA = (1400 - NUM_BYTES_RFC4122_UUID); + if (bytes.size() > MAX_ALLOWED_AVATAR_DATA) { + qCWarning(avatars) << "otherAvatar.toByteArray() resulted in very large buffer:" << bytes.size() << "... attempt to drop facial data"; + + dropFaceTracking = true; // first try dropping the facial data + bytes = otherAvatar->toByteArray(detail, lastEncodeForOther, lastSentJointsForOther, + hasFlagsOut, dropFaceTracking, distanceAdjust, viewerPosition, &lastSentJointsForOther); - static const int MAX_ALLOWED_AVATAR_DATA = (1400 - NUM_BYTES_RFC4122_UUID); if (bytes.size() > MAX_ALLOWED_AVATAR_DATA) { - qCWarning(avatars) << "otherAvatar.toByteArray() resulted in very large buffer:" << bytes.size() << "... attempt to drop facial data"; - - dropFaceTracking = true; // first try dropping the facial data - bytes = otherAvatar->toByteArray(detail, lastEncodeForOther, lastSentJointsForOther, + qCWarning(avatars) << "otherAvatar.toByteArray() without facial data resulted in very large buffer:" << bytes.size() << "... reduce to MinimumData"; + bytes = otherAvatar->toByteArray(AvatarData::MinimumData, lastEncodeForOther, lastSentJointsForOther, hasFlagsOut, dropFaceTracking, distanceAdjust, viewerPosition, &lastSentJointsForOther); - - if (bytes.size() > MAX_ALLOWED_AVATAR_DATA) { - qCWarning(avatars) << "otherAvatar.toByteArray() without facial data resulted in very large buffer:" << bytes.size() << "... reduce to MinimumData"; - bytes = otherAvatar->toByteArray(AvatarData::MinimumData, lastEncodeForOther, lastSentJointsForOther, - hasFlagsOut, dropFaceTracking, distanceAdjust, viewerPosition, &lastSentJointsForOther); - } - - if (bytes.size() > MAX_ALLOWED_AVATAR_DATA) { - qCWarning(avatars) << "otherAvatar.toByteArray() MinimumData resulted in very large buffer:" << bytes.size() << "... FAIL!!"; - includeThisAvatar = false; - } } - if (includeThisAvatar) { - numAvatarDataBytes += avatarPacketList->write(otherNode->getUUID().toRfc4122()); - numAvatarDataBytes += avatarPacketList->write(bytes); - - if (detail != AvatarData::NoData) { - _stats.numOthersIncluded++; - - // increment the number of avatars sent to this reciever - nodeData->incrementNumAvatarsSentLastFrame(); - - // set the last sent sequence number for this sender on the receiver - nodeData->setLastBroadcastSequenceNumber(otherNode->getUUID(), - otherNodeData->getLastReceivedSequenceNumber()); - - // remember the last time we sent details about this other node to the receiver - nodeData->setLastBroadcastTime(otherNode->getUUID(), start); - } + if (bytes.size() > MAX_ALLOWED_AVATAR_DATA) { + qCWarning(avatars) << "otherAvatar.toByteArray() MinimumData resulted in very large buffer:" << bytes.size() << "... FAIL!!"; + includeThisAvatar = false; } + } - avatarPacketList->endSegment(); + if (includeThisAvatar) { + numAvatarDataBytes += avatarPacketList->write(otherNode->getUUID().toRfc4122()); + numAvatarDataBytes += avatarPacketList->write(bytes); - quint64 endAvatarDataPacking = usecTimestampNow(); - _stats.avatarDataPackingElapsedTime += (endAvatarDataPacking - startAvatarDataPacking); + if (detail != AvatarData::NoData) { + _stats.numOthersIncluded++; + + // increment the number of avatars sent to this reciever + nodeData->incrementNumAvatarsSentLastFrame(); + + // set the last sent sequence number for this sender on the receiver + nodeData->setLastBroadcastSequenceNumber(otherNode->getUUID(), + otherNodeData->getLastReceivedSequenceNumber()); + + // remember the last time we sent details about this other node to the receiver + nodeData->setLastBroadcastTime(otherNode->getUUID(), start); + } + } + + avatarPacketList->endSegment(); + + quint64 endAvatarDataPacking = usecTimestampNow(); + _stats.avatarDataPackingElapsedTime += (endAvatarDataPacking - startAvatarDataPacking); }; quint64 startPacketSending = usecTimestampNow(); diff --git a/assignment-client/src/avatars/AvatarMixerSlavePool.cpp b/assignment-client/src/avatars/AvatarMixerSlavePool.cpp index 07d4fa8851..cb5ae7735a 100644 --- a/assignment-client/src/avatars/AvatarMixerSlavePool.cpp +++ b/assignment-client/src/avatars/AvatarMixerSlavePool.cpp @@ -76,8 +76,8 @@ void AvatarMixerSlavePool::processIncomingPackets(ConstIter begin, ConstIter end } void AvatarMixerSlavePool::broadcastAvatarData(ConstIter begin, ConstIter end, - p_high_resolution_clock::time_point lastFrameTimestamp, - float maxKbpsPerNode, float throttlingRatio) { + p_high_resolution_clock::time_point lastFrameTimestamp, + float maxKbpsPerNode, float throttlingRatio) { _function = &AvatarMixerSlave::broadcastAvatarData; _configure = [&](AvatarMixerSlave& slave) { slave.configureBroadcast(begin, end, lastFrameTimestamp, maxKbpsPerNode, throttlingRatio); From 806ebd3401214cc3a735cea2d0c910c3b91a1b5b Mon Sep 17 00:00:00 2001 From: Stephen Birarda Date: Wed, 14 Jun 2017 16:13:20 -0700 Subject: [PATCH 46/46] add return to queueReplicatedAudioPacket if no type match --- assignment-client/src/audio/AudioMixer.cpp | 2 ++ 1 file changed, 2 insertions(+) diff --git a/assignment-client/src/audio/AudioMixer.cpp b/assignment-client/src/audio/AudioMixer.cpp index a98c172ef3..2bbd6a1950 100644 --- a/assignment-client/src/audio/AudioMixer.cpp +++ b/assignment-client/src/audio/AudioMixer.cpp @@ -138,6 +138,8 @@ void AudioMixer::queueReplicatedAudioPacket(QSharedPointer mess rewrittenType = PacketType::InjectAudio; } else if (message->getType() == PacketType::ReplicatedSilentAudioFrame) { rewrittenType = PacketType::SilentAudioFrame; + } else { + return; } auto replicatedMessage = QSharedPointer::create(audioData, rewrittenType,