From 73d688610fc53c16d2ebf553d6fbebd9e3be4011 Mon Sep 17 00:00:00 2001 From: Simon Walton Date: Tue, 8 Jan 2019 18:33:00 -0800 Subject: [PATCH 1/3] Track stats for total AvatarData bytes, traits bytes and identity bytes --- assignment-client/src/avatars/AvatarMixer.cpp | 5 +++- .../src/avatars/AvatarMixerSlave.cpp | 17 ++++++----- .../src/avatars/AvatarMixerSlave.h | 28 +++++++++++++------ libraries/avatars/src/AvatarData.cpp | 4 +-- 4 files changed, 35 insertions(+), 19 deletions(-) diff --git a/assignment-client/src/avatars/AvatarMixer.cpp b/assignment-client/src/avatars/AvatarMixer.cpp index cb2f0636b9..8ace26f2e2 100644 --- a/assignment-client/src/avatars/AvatarMixer.cpp +++ b/assignment-client/src/avatars/AvatarMixer.cpp @@ -767,6 +767,9 @@ void AvatarMixer::sendStatsPacket() { float averageOverBudgetAvatars = averageNodes ? aggregateStats.overBudgetAvatars / averageNodes : 0.0f; slavesAggregatObject["sent_3_averageOverBudgetAvatars"] = TIGHT_LOOP_STAT(averageOverBudgetAvatars); + slavesAggregatObject["sent_4_averageDataBytes"] = TIGHT_LOOP_STAT(aggregateStats.numDataBytesSent); + slavesAggregatObject["sent_5_averageAvatarEntityBytes"] = TIGHT_LOOP_STAT(aggregateStats.numTraitsBytesSent); + slavesAggregatObject["sent_4_averageIdentityBytes"] = TIGHT_LOOP_STAT(aggregateStats.numIdentityBytesSent); slavesAggregatObject["timing_1_processIncomingPackets"] = TIGHT_LOOP_STAT_UINT64(aggregateStats.processIncomingPacketsElapsedTime); slavesAggregatObject["timing_2_ignoreCalculation"] = TIGHT_LOOP_STAT_UINT64(aggregateStats.ignoreCalculationElapsedTime); @@ -775,7 +778,7 @@ void AvatarMixer::sendStatsPacket() { slavesAggregatObject["timing_5_packetSending"] = TIGHT_LOOP_STAT_UINT64(aggregateStats.packetSendingElapsedTime); slavesAggregatObject["timing_6_jobElapsedTime"] = TIGHT_LOOP_STAT_UINT64(aggregateStats.jobElapsedTime); - statsObject["slaves_aggregate"] = slavesAggregatObject; + statsObject["slaves_aggregate (per frame)"] = slavesAggregatObject; _handleViewFrustumPacketElapsedTime = 0; _handleAvatarIdentityPacketElapsedTime = 0; diff --git a/assignment-client/src/avatars/AvatarMixerSlave.cpp b/assignment-client/src/avatars/AvatarMixerSlave.cpp index cd9d164ef7..22a393a06c 100644 --- a/assignment-client/src/avatars/AvatarMixerSlave.cpp +++ b/assignment-client/src/avatars/AvatarMixerSlave.cpp @@ -73,7 +73,8 @@ int AvatarMixerSlave::sendIdentityPacket(NLPacketList& packetList, const AvatarM QByteArray individualData = nodeData->getConstAvatarData()->identityByteArray(); individualData.replace(0, NUM_BYTES_RFC4122_UUID, nodeData->getNodeID().toRfc4122()); // FIXME, this looks suspicious packetList.write(individualData); - _stats.numIdentityPackets++; + _stats.numIdentityPacketsSent++; + _stats.numIdentityBytesSent += individualData.size(); return individualData.size(); } else { return 0; @@ -109,7 +110,6 @@ qint64 AvatarMixerSlave::addChangedTraitsToBulkPacket(AvatarMixerClientData* lis while (simpleReceivedIt != lastReceivedVersions.simpleCEnd()) { auto traitType = static_cast(std::distance(lastReceivedVersions.simpleCBegin(), simpleReceivedIt)); - auto lastReceivedVersion = *simpleReceivedIt; auto& lastSentVersionRef = lastSentVersions[traitType]; @@ -191,7 +191,8 @@ int AvatarMixerSlave::sendReplicatedIdentityPacket(const Node& agentNode, const auto identityPacket = NLPacketList::create(PacketType::ReplicatedAvatarIdentity, QByteArray(), true, true); identityPacket->write(individualData); DependencyManager::get()->sendPacketList(std::move(identityPacket), destinationNode); - _stats.numIdentityPackets++; + _stats.numIdentityPacketsSent++; + _stats.numIdentityBytesSent += individualData.size(); return individualData.size(); } else { return 0; @@ -539,8 +540,8 @@ void AvatarMixerSlave::broadcastAvatarDataToAgent(const SharedNodePointer& node) ++numPacketsSent; } - _stats.numPacketsSent += numPacketsSent; - _stats.numBytesSent += numAvatarDataBytes; + _stats.numDataPacketsSent += numPacketsSent; + _stats.numDataBytesSent += numAvatarDataBytes; // record the bytes sent for other avatar data in the AvatarMixerClientData nodeData->recordSentAvatarData(numAvatarDataBytes); @@ -550,6 +551,8 @@ void AvatarMixerSlave::broadcastAvatarDataToAgent(const SharedNodePointer& node) if (traitsPacketList->getNumPackets() >= 1) { // send the traits packet list + _stats.numTraitsBytesSent += traitBytesSent; + _stats.numTraitsPacketsSent += (int) traitsPacketList->getNumPackets(); nodeList->sendPacketList(std::move(traitsPacketList), *destinationNode); } @@ -685,8 +688,8 @@ void AvatarMixerSlave::broadcastAvatarDataToDownstreamMixer(const SharedNodePoin // close the current packet so that we're always sending something avatarPacketList->closeCurrentPacket(true); - _stats.numPacketsSent += (int)avatarPacketList->getNumPackets(); - _stats.numBytesSent += numAvatarDataBytes; + _stats.numDataPacketsSent += (int)avatarPacketList->getNumPackets(); + _stats.numDataBytesSent += numAvatarDataBytes; // send the replicated bulk avatar data auto nodeList = DependencyManager::get(); diff --git a/assignment-client/src/avatars/AvatarMixerSlave.h b/assignment-client/src/avatars/AvatarMixerSlave.h index 2ef90af38e..5c38cda590 100644 --- a/assignment-client/src/avatars/AvatarMixerSlave.h +++ b/assignment-client/src/avatars/AvatarMixerSlave.h @@ -24,9 +24,12 @@ public: int nodesBroadcastedTo { 0 }; int downstreamMixersBroadcastedTo { 0 }; - int numPacketsSent { 0 }; - int numBytesSent { 0 }; - int numIdentityPackets { 0 }; + int numDataBytesSent { 0 }; + int numTraitsBytesSent { 0 }; + int numIdentityBytesSent { 0 }; + int numDataPacketsSent { 0 }; + int numTraitsPacketsSent { 0 }; + int numIdentityPacketsSent { 0 }; int numOthersIncluded { 0 }; int overBudgetAvatars { 0 }; @@ -45,9 +48,13 @@ public: // sending job stats nodesBroadcastedTo = 0; downstreamMixersBroadcastedTo = 0; - numPacketsSent = 0; - numBytesSent = 0; - numIdentityPackets = 0; + + numDataBytesSent = 0; + numTraitsBytesSent = 0; + numIdentityBytesSent = 0; + numDataPacketsSent = 0; + numTraitsPacketsSent = 0; + numIdentityPacketsSent = 0; numOthersIncluded = 0; overBudgetAvatars = 0; @@ -65,9 +72,12 @@ public: nodesBroadcastedTo += rhs.nodesBroadcastedTo; downstreamMixersBroadcastedTo += rhs.downstreamMixersBroadcastedTo; - numPacketsSent += rhs.numPacketsSent; - numBytesSent += rhs.numBytesSent; - numIdentityPackets += rhs.numIdentityPackets; + numDataBytesSent += rhs.numDataBytesSent; + numTraitsBytesSent += rhs.numTraitsBytesSent; + numIdentityBytesSent += rhs.numIdentityBytesSent; + numDataPacketsSent += rhs.numDataPacketsSent; + numTraitsPacketsSent += rhs.numTraitsPacketsSent; + numIdentityPacketsSent += rhs.numIdentityPacketsSent; numOthersIncluded += rhs.numOthersIncluded; overBudgetAvatars += rhs.overBudgetAvatars; diff --git a/libraries/avatars/src/AvatarData.cpp b/libraries/avatars/src/AvatarData.cpp index e72fa3a6eb..2f1c8d3d82 100644 --- a/libraries/avatars/src/AvatarData.cpp +++ b/libraries/avatars/src/AvatarData.cpp @@ -1987,9 +1987,9 @@ qint64 AvatarData::packTraitInstance(AvatarTraits::TraitType traitType, AvatarTr qint64 bytesWritten = 0; if (traitType == AvatarTraits::AvatarEntity) { - packAvatarEntityTraitInstance(traitType, traitInstanceID, destination, traitVersion); + bytesWritten += packAvatarEntityTraitInstance(traitType, traitInstanceID, destination, traitVersion); } else if (traitType == AvatarTraits::Grab) { - packGrabTraitInstance(traitType, traitInstanceID, destination, traitVersion); + bytesWritten += packGrabTraitInstance(traitType, traitInstanceID, destination, traitVersion); } return bytesWritten; From cb6b86a236f8612eb4cdeeb451cf2628b7c59d32 Mon Sep 17 00:00:00 2001 From: Simon Walton Date: Wed, 9 Jan 2019 09:36:17 -0800 Subject: [PATCH 2/3] Tweak web display --- assignment-client/src/avatars/AvatarMixer.cpp | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/assignment-client/src/avatars/AvatarMixer.cpp b/assignment-client/src/avatars/AvatarMixer.cpp index 8ace26f2e2..1415db7a27 100644 --- a/assignment-client/src/avatars/AvatarMixer.cpp +++ b/assignment-client/src/avatars/AvatarMixer.cpp @@ -768,8 +768,8 @@ void AvatarMixer::sendStatsPacket() { float averageOverBudgetAvatars = averageNodes ? aggregateStats.overBudgetAvatars / averageNodes : 0.0f; slavesAggregatObject["sent_3_averageOverBudgetAvatars"] = TIGHT_LOOP_STAT(averageOverBudgetAvatars); slavesAggregatObject["sent_4_averageDataBytes"] = TIGHT_LOOP_STAT(aggregateStats.numDataBytesSent); - slavesAggregatObject["sent_5_averageAvatarEntityBytes"] = TIGHT_LOOP_STAT(aggregateStats.numTraitsBytesSent); - slavesAggregatObject["sent_4_averageIdentityBytes"] = TIGHT_LOOP_STAT(aggregateStats.numIdentityBytesSent); + slavesAggregatObject["sent_5_averageTraitsBytes"] = TIGHT_LOOP_STAT(aggregateStats.numTraitsBytesSent); + slavesAggregatObject["sent_6_averageIdentityBytes"] = TIGHT_LOOP_STAT(aggregateStats.numIdentityBytesSent); slavesAggregatObject["timing_1_processIncomingPackets"] = TIGHT_LOOP_STAT_UINT64(aggregateStats.processIncomingPacketsElapsedTime); slavesAggregatObject["timing_2_ignoreCalculation"] = TIGHT_LOOP_STAT_UINT64(aggregateStats.ignoreCalculationElapsedTime); From 5bcdf143b9d01f459c689a18a0634225dbf836d1 Mon Sep 17 00:00:00 2001 From: Simon Walton Date: Wed, 9 Jan 2019 13:45:07 -0800 Subject: [PATCH 3/3] Add per-destination-avatar traits-rate stats --- assignment-client/src/avatars/AvatarMixer.cpp | 5 +++-- assignment-client/src/avatars/AvatarMixerClientData.cpp | 3 ++- assignment-client/src/avatars/AvatarMixerClientData.h | 9 ++++++++- assignment-client/src/avatars/AvatarMixerSlave.cpp | 7 ++++--- 4 files changed, 17 insertions(+), 7 deletions(-) diff --git a/assignment-client/src/avatars/AvatarMixer.cpp b/assignment-client/src/avatars/AvatarMixer.cpp index 1415db7a27..62b58f3d73 100644 --- a/assignment-client/src/avatars/AvatarMixer.cpp +++ b/assignment-client/src/avatars/AvatarMixer.cpp @@ -803,7 +803,8 @@ void AvatarMixer::sendStatsPacket() { // add the key to ask the domain-server for a username replacement, if it has it avatarStats[USERNAME_UUID_REPLACEMENT_STATS_KEY] = uuidStringWithoutCurlyBraces(node->getUUID()); - avatarStats[NODE_OUTBOUND_KBPS_STAT_KEY] = node->getOutboundKbps(); + float outboundAvatarDataKbps = node->getOutboundKbps(); + avatarStats[NODE_OUTBOUND_KBPS_STAT_KEY] = outboundAvatarDataKbps; avatarStats[NODE_INBOUND_KBPS_STAT_KEY] = node->getInboundKbps(); AvatarMixerClientData* clientData = static_cast(node->getLinkedData()); @@ -814,7 +815,7 @@ void AvatarMixer::sendStatsPacket() { // add the diff between the full outbound bandwidth and the measured bandwidth for AvatarData send only avatarStats["delta_full_vs_avatar_data_kbps"] = - avatarStats[NODE_OUTBOUND_KBPS_STAT_KEY].toDouble() - avatarStats[OUTBOUND_AVATAR_DATA_STATS_KEY].toDouble(); + (double)outboundAvatarDataKbps - avatarStats[OUTBOUND_AVATAR_DATA_STATS_KEY].toDouble(); } } diff --git a/assignment-client/src/avatars/AvatarMixerClientData.cpp b/assignment-client/src/avatars/AvatarMixerClientData.cpp index f215fd4448..7313634cff 100644 --- a/assignment-client/src/avatars/AvatarMixerClientData.cpp +++ b/assignment-client/src/avatars/AvatarMixerClientData.cpp @@ -317,7 +317,8 @@ void AvatarMixerClientData::loadJSONStats(QJsonObject& jsonObject) const { jsonObject["total_num_out_of_order_sends"] = _numOutOfOrderSends; jsonObject[OUTBOUND_AVATAR_DATA_STATS_KEY] = getOutboundAvatarDataKbps(); - jsonObject[INBOUND_AVATAR_DATA_STATS_KEY] = _avatar->getAverageBytesReceivedPerSecond() / (float) BYTES_PER_KILOBIT; + jsonObject[OUTBOUND_AVATAR_TRAITS_STATS_KEY] = getOutboundAvatarTraitsKbps(); + jsonObject[INBOUND_AVATAR_DATA_STATS_KEY] = _avatar->getAverageBytesReceivedPerSecond() / (float)BYTES_PER_KILOBIT; jsonObject["av_data_receive_rate"] = _avatar->getReceiveRate(); jsonObject["recent_other_av_in_view"] = _recentOtherAvatarsInView; diff --git a/assignment-client/src/avatars/AvatarMixerClientData.h b/assignment-client/src/avatars/AvatarMixerClientData.h index 8a86af384a..18144b1e01 100644 --- a/assignment-client/src/avatars/AvatarMixerClientData.h +++ b/assignment-client/src/avatars/AvatarMixerClientData.h @@ -32,6 +32,7 @@ #include const QString OUTBOUND_AVATAR_DATA_STATS_KEY = "outbound_av_data_kbps"; +const QString OUTBOUND_AVATAR_TRAITS_STATS_KEY = "outbound_av_traits_kbps"; const QString INBOUND_AVATAR_DATA_STATS_KEY = "inbound_av_data_kbps"; struct SlaveSharedData; @@ -85,10 +86,15 @@ public: void incrementNumFramesSinceFRDAdjustment() { ++_numFramesSinceAdjustment; } void resetNumFramesSinceFRDAdjustment() { _numFramesSinceAdjustment = 0; } - void recordSentAvatarData(int numBytes) { _avgOtherAvatarDataRate.updateAverage((float) numBytes); } + void recordSentAvatarData(int numDataBytes, int numTraitsBytes = 0) { + _avgOtherAvatarDataRate.updateAverage(numDataBytes); + _avgOtherAvatarTraitsRate.updateAverage(numTraitsBytes); + } float getOutboundAvatarDataKbps() const { return _avgOtherAvatarDataRate.getAverageSampleValuePerSecond() / (float) BYTES_PER_KILOBIT; } + float getOutboundAvatarTraitsKbps() const + { return _avgOtherAvatarTraitsRate.getAverageSampleValuePerSecond() / BYTES_PER_KILOBIT; } void loadJSONStats(QJsonObject& jsonObject) const; @@ -171,6 +177,7 @@ private: int _numOutOfOrderSends = 0; SimpleMovingAverage _avgOtherAvatarDataRate; + SimpleMovingAverage _avgOtherAvatarTraitsRate; std::vector _radiusIgnoredOthers; ConicalViewFrustums _currentViewFrustums; diff --git a/assignment-client/src/avatars/AvatarMixerSlave.cpp b/assignment-client/src/avatars/AvatarMixerSlave.cpp index 22a393a06c..0460407f01 100644 --- a/assignment-client/src/avatars/AvatarMixerSlave.cpp +++ b/assignment-client/src/avatars/AvatarMixerSlave.cpp @@ -543,9 +543,6 @@ void AvatarMixerSlave::broadcastAvatarDataToAgent(const SharedNodePointer& node) _stats.numDataPacketsSent += numPacketsSent; _stats.numDataBytesSent += numAvatarDataBytes; - // record the bytes sent for other avatar data in the AvatarMixerClientData - nodeData->recordSentAvatarData(numAvatarDataBytes); - // close the current traits packet list traitsPacketList->closeCurrentPacket(); @@ -562,6 +559,10 @@ void AvatarMixerSlave::broadcastAvatarDataToAgent(const SharedNodePointer& node) nodeList->sendPacketList(std::move(identityPacketList), *destinationNode); } + // record the bytes sent for other avatar data in the AvatarMixerClientData + nodeData->recordSentAvatarData(numAvatarDataBytes, traitBytesSent); + + // record the number of avatars held back this frame nodeData->recordNumOtherAvatarStarves(numAvatarsHeldBack); nodeData->recordNumOtherAvatarSkips(numAvatarsWithSkippedFrames);