From 847bc28990cea9580c6e1b0814ce1c4cf3d04878 Mon Sep 17 00:00:00 2001 From: wangyix Date: Tue, 22 Jul 2014 10:27:26 -0700 Subject: [PATCH 01/15] Audio stats now reset when AudioMixer is killed --- interface/src/Application.cpp | 2 +- interface/src/Audio.cpp | 19 ++++++++++++++++++- interface/src/Audio.h | 3 ++- 3 files changed, 21 insertions(+), 3 deletions(-) diff --git a/interface/src/Application.cpp b/interface/src/Application.cpp index 1d5cbabd32..50f0af8723 100644 --- a/interface/src/Application.cpp +++ b/interface/src/Application.cpp @@ -3338,7 +3338,7 @@ void Application::nodeKilled(SharedNodePointer node) { _modelEditSender.nodeKilled(node); if (node->getType() == NodeType::AudioMixer) { - QMetaObject::invokeMethod(&_audio, "resetIncomingMixedAudioSequenceNumberStats"); + QMetaObject::invokeMethod(&_audio, "audioMixerKilled"); } if (node->getType() == NodeType::VoxelServer) { diff --git a/interface/src/Audio.cpp b/interface/src/Audio.cpp index be348ab635..7cf19fbcfe 100644 --- a/interface/src/Audio.cpp +++ b/interface/src/Audio.cpp @@ -152,11 +152,28 @@ void Audio::reset() { _starveCount = 0; _consecutiveNotMixedCount = 0; + _outgoingAvatarAudioSequenceNumber = 0; + + resetStats(); +} + +void Audio::resetStats() { _audioMixerAvatarStreamAudioStats = AudioStreamStats(); _audioMixerInjectedStreamAudioStatsMap.clear(); - _outgoingAvatarAudioSequenceNumber = 0; _incomingMixedAudioSequenceNumberStats.reset(); + + _interframeTimeGapStats.reset(); + + _inputRingBufferFramesAvailableStats.reset(); + + _outputRingBufferFramesAvailableStats.reset(); + _audioOutputBufferFramesAvailableStats.reset(); +} + +void Audio::audioMixerKilled() { + _outgoingAvatarAudioSequenceNumber = 0; + resetStats(); } QAudioDeviceInfo getNamedAudioDeviceForMode(QAudio::Mode mode, const QString& deviceName) { diff --git a/interface/src/Audio.h b/interface/src/Audio.h index ed50815d78..f91208b6e4 100644 --- a/interface/src/Audio.h +++ b/interface/src/Audio.h @@ -93,7 +93,8 @@ public slots: void addSpatialAudioToBuffer(unsigned int sampleTime, const QByteArray& spatialAudio, unsigned int numSamples); void handleAudioInput(); void reset(); - void resetIncomingMixedAudioSequenceNumberStats() { _incomingMixedAudioSequenceNumberStats.reset(); } + void resetStats(); + void audioMixerKilled(); void toggleMute(); void toggleAudioNoiseReduction(); void toggleToneInjection(); From f7381c70bc9713aab8709bcbde1c93771d172cc5 Mon Sep 17 00:00:00 2001 From: wangyix Date: Tue, 22 Jul 2014 11:09:47 -0700 Subject: [PATCH 02/15] starvcount and notmixedcount also reset when audiomixer is killed --- interface/src/Audio.cpp | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/interface/src/Audio.cpp b/interface/src/Audio.cpp index 7cf19fbcfe..8e3662e790 100644 --- a/interface/src/Audio.cpp +++ b/interface/src/Audio.cpp @@ -148,9 +148,6 @@ void Audio::init(QGLWidget *parent) { void Audio::reset() { _ringBuffer.reset(); - - _starveCount = 0; - _consecutiveNotMixedCount = 0; _outgoingAvatarAudioSequenceNumber = 0; @@ -158,6 +155,9 @@ void Audio::reset() { } void Audio::resetStats() { + _starveCount = 0; + _consecutiveNotMixedCount = 0; + _audioMixerAvatarStreamAudioStats = AudioStreamStats(); _audioMixerInjectedStreamAudioStatsMap.clear(); From 1b2dc2cbe1c11768028bf13213dcd1eb6d2cc83a Mon Sep 17 00:00:00 2001 From: wangyix Date: Tue, 22 Jul 2014 14:21:53 -0700 Subject: [PATCH 03/15] added central audio stats including pipeline latency; old stats still there --- interface/src/Audio.cpp | 205 ++++++++++++++++++++++++ interface/src/Audio.h | 6 + interface/src/Menu.cpp | 6 + interface/src/Menu.h | 1 + interface/src/ui/ApplicationOverlay.cpp | 2 + 5 files changed, 220 insertions(+) diff --git a/interface/src/Audio.cpp b/interface/src/Audio.cpp index 8e3662e790..d200bee043 100644 --- a/interface/src/Audio.cpp +++ b/interface/src/Audio.cpp @@ -43,6 +43,7 @@ #include "Audio.h" #include "Menu.h" #include "Util.h" +#include "AudioRingBuffer.h" static const float AUDIO_CALLBACK_MSECS = (float) NETWORK_BUFFER_LENGTH_SAMPLES_PER_CHANNEL / (float)SAMPLE_RATE * 1000.0; @@ -125,6 +126,7 @@ Audio::Audio(int16_t initialJitterBufferSamples, QObject* parent) : _scopeInput(0), _scopeOutputLeft(0), _scopeOutputRight(0), + _statsEnabled(false), _starveCount(0), _consecutiveNotMixedCount(0), _outgoingAvatarAudioSequenceNumber(0), @@ -1303,6 +1305,10 @@ void Audio::toggleScopePause() { _scopeEnabledPause = !_scopeEnabledPause; } +void Audio::toggleStats() { + _statsEnabled = !_statsEnabled; +} + void Audio::selectAudioScopeFiveFrames() { if (Menu::getInstance()->isOptionChecked(MenuOption::AudioScopeFiveFrames)) { reallocateScope(5); @@ -1382,6 +1388,205 @@ void Audio::addBufferToScope( } } +void Audio::renderStats(const float* color, int width, int height) { + if (!_statsEnabled) { + return; + } + + const int MIN_LINES = 20; + const int STATS_BACKGROUND_HEIGHT = STATS_HEIGHT_PER_LINE * MIN_LINES; + + int lines = _audioMixerInjectedStreamAudioStatsMap.size() * 3 + 23; + int statsHeight = STATS_HEIGHT_PER_LINE * std::max(lines, MIN_LINES); + + + static const float backgroundColor[4] = { 0.2f, 0.2f, 0.2f, 0.6f }; + + int x = (width - STATS_WIDTH) / 2; + int y = (height - STATS_BACKGROUND_HEIGHT) / 2; + int w = STATS_WIDTH; + int h = statsHeight; + renderBackground(backgroundColor, x, y, w, h); + + + int horizontalOffset = x + 5; + int verticalOffset = y; + + float scale = 0.10f; + float rotation = 0.0f; + int font = 2; + + + char latencyStatString[512]; + + const float BUFFER_SEND_INTERVAL_MSECS = BUFFER_SEND_INTERVAL_USECS / (float)USECS_PER_MSEC; + + float inputRingBufferLatency = 0.0f, networkRoundtripLatency = 0.0f, mixerRingBufferLatency = 0.0f, outputRingBufferLatency = 0.0f, audioOutputBufferLatency = 0.0f; + + SharedNodePointer audioMixerNodePointer = NodeList::getInstance()->soloNodeOfType(NodeType::AudioMixer); + if (!audioMixerNodePointer.isNull()) { + inputRingBufferLatency = _inputRingBufferFramesAvailableStats.getWindowAverage() * BUFFER_SEND_INTERVAL_MSECS; + networkRoundtripLatency = audioMixerNodePointer->getPingMs(); + mixerRingBufferLatency = _audioMixerAvatarStreamAudioStats._ringBufferFramesAvailableAverage * BUFFER_SEND_INTERVAL_MSECS; + outputRingBufferLatency = _outputRingBufferFramesAvailableStats.getWindowAverage() * BUFFER_SEND_INTERVAL_MSECS; + audioOutputBufferLatency = _audioOutputBufferFramesAvailableStats.getWindowAverage() * BUFFER_SEND_INTERVAL_MSECS; + } + float totalLatency = inputRingBufferLatency + networkRoundtripLatency + mixerRingBufferLatency + outputRingBufferLatency + audioOutputBufferLatency; + + + sprintf(latencyStatString, " Input ring buffer: %.1fms\n", inputRingBufferLatency); + verticalOffset += STATS_HEIGHT_PER_LINE; + drawText(horizontalOffset, verticalOffset, scale, rotation, font, latencyStatString, color); + + sprintf(latencyStatString, " Network to mixer: %.1fms / 2\n", networkRoundtripLatency); + verticalOffset += STATS_HEIGHT_PER_LINE; + drawText(horizontalOffset, verticalOffset, scale, rotation, font, latencyStatString, color); + + sprintf(latencyStatString, " AudioMixer ring buffer: %.1fms\n", mixerRingBufferLatency); + verticalOffset += STATS_HEIGHT_PER_LINE; + drawText(horizontalOffset, verticalOffset, scale, rotation, font, latencyStatString, color); + + sprintf(latencyStatString, " Network to client: %.1fms / 2\n", networkRoundtripLatency); + verticalOffset += STATS_HEIGHT_PER_LINE; + drawText(horizontalOffset, verticalOffset, scale, rotation, font, latencyStatString, color); + + sprintf(latencyStatString, " Output ring buffer: %.1fms\n", outputRingBufferLatency); + verticalOffset += STATS_HEIGHT_PER_LINE; + drawText(horizontalOffset, verticalOffset, scale, rotation, font, latencyStatString, color); + + sprintf(latencyStatString, " Audio output buffer: %.1fms\n", audioOutputBufferLatency); + verticalOffset += STATS_HEIGHT_PER_LINE; + drawText(horizontalOffset, verticalOffset, scale, rotation, font, latencyStatString, color); + + sprintf(latencyStatString, " TOTAL: %.1fms\n", totalLatency); + verticalOffset += STATS_HEIGHT_PER_LINE; + drawText(horizontalOffset, verticalOffset, scale, rotation, font, latencyStatString, color); + + verticalOffset += STATS_HEIGHT_PER_LINE; + + + char inputAudioLabelString[] = "Input: avail_avg_10s/avail"; + + verticalOffset += STATS_HEIGHT_PER_LINE; + drawText(horizontalOffset, verticalOffset, scale, rotation, font, inputAudioLabelString, color); + + char inputAudioStatsString[512]; + sprintf(inputAudioStatsString, " %d/%d", getInputRingBufferAverageFramesAvailable(), getInputRingBufferFramesAvailable()); + + verticalOffset += STATS_HEIGHT_PER_LINE; + drawText(horizontalOffset, verticalOffset, scale, rotation, font, inputAudioStatsString, color); + + + char audioMixerStatsLabelString[] = "AudioMixer stats:"; + char streamStatsFormatLabelString[] = " lost%/lost_30s%, desr/avail_avg_10s/avail"; + char streamStatsFormatLabelString2[] = " gaps: min/max/avg, starv/ovfl"; + char streamStatsFormatLabelString3[] = " gaps_30s: (same), notmix/sdrop"; + + verticalOffset += STATS_HEIGHT_PER_LINE; + drawText(horizontalOffset, verticalOffset, scale, rotation, font, audioMixerStatsLabelString, color); + verticalOffset += STATS_HEIGHT_PER_LINE; + drawText(horizontalOffset, verticalOffset, scale, rotation, font, streamStatsFormatLabelString, color); + verticalOffset += STATS_HEIGHT_PER_LINE; + drawText(horizontalOffset, verticalOffset, scale, rotation, font, streamStatsFormatLabelString2, color); + verticalOffset += STATS_HEIGHT_PER_LINE; + drawText(horizontalOffset, verticalOffset, scale, rotation, font, streamStatsFormatLabelString3, color); + + char downstreamLabelString[] = "Downstream:"; + verticalOffset += STATS_HEIGHT_PER_LINE; + drawText(horizontalOffset, verticalOffset, scale, rotation, font, downstreamLabelString, color); + + char downstreamAudioStatsString[512]; + + AudioStreamStats downstreamAudioStreamStats = getDownstreamAudioStreamStats(); + + sprintf(downstreamAudioStatsString, " mix: %.2f%%/%.2f%%, %u/%u+%d/%u+%d", downstreamAudioStreamStats._packetStreamStats.getLostRate()*100.0f, + downstreamAudioStreamStats._packetStreamWindowStats.getLostRate() * 100.0f, + downstreamAudioStreamStats._ringBufferDesiredJitterBufferFrames, downstreamAudioStreamStats._ringBufferFramesAvailableAverage, + getOutputRingBufferAverageFramesAvailable(), + downstreamAudioStreamStats._ringBufferFramesAvailable, getOutputRingBufferFramesAvailable()); + + verticalOffset += STATS_HEIGHT_PER_LINE; + drawText(horizontalOffset, verticalOffset, scale, rotation, font, downstreamAudioStatsString, color); + + sprintf(downstreamAudioStatsString, " %s/%s/%s, %u/%u", formatUsecTime(downstreamAudioStreamStats._timeGapMin).toLatin1().data(), + formatUsecTime(downstreamAudioStreamStats._timeGapMax).toLatin1().data(), + formatUsecTime(downstreamAudioStreamStats._timeGapAverage).toLatin1().data(), + downstreamAudioStreamStats._ringBufferStarveCount, downstreamAudioStreamStats._ringBufferOverflowCount); + + verticalOffset += STATS_HEIGHT_PER_LINE; + drawText(horizontalOffset, verticalOffset, scale, rotation, font, downstreamAudioStatsString, color); + + sprintf(downstreamAudioStatsString, " %s/%s/%s, %u/?", formatUsecTime(downstreamAudioStreamStats._timeGapWindowMin).toLatin1().data(), + formatUsecTime(downstreamAudioStreamStats._timeGapWindowMax).toLatin1().data(), + formatUsecTime(downstreamAudioStreamStats._timeGapWindowAverage).toLatin1().data(), + downstreamAudioStreamStats._ringBufferConsecutiveNotMixedCount); + + verticalOffset += STATS_HEIGHT_PER_LINE; + drawText(horizontalOffset, verticalOffset, scale, rotation, font, downstreamAudioStatsString, color); + + + char upstreamLabelString[] = "Upstream:"; + verticalOffset += STATS_HEIGHT_PER_LINE; + drawText(horizontalOffset, verticalOffset, scale, rotation, font, upstreamLabelString, color); + + char upstreamAudioStatsString[512]; + + const AudioStreamStats& audioMixerAvatarAudioStreamStats = getAudioMixerAvatarStreamAudioStats(); + + sprintf(upstreamAudioStatsString, " mic: %.2f%%/%.2f%%, %u/%u/%u", audioMixerAvatarAudioStreamStats._packetStreamStats.getLostRate()*100.0f, + audioMixerAvatarAudioStreamStats._packetStreamWindowStats.getLostRate() * 100.0f, + audioMixerAvatarAudioStreamStats._ringBufferDesiredJitterBufferFrames, audioMixerAvatarAudioStreamStats._ringBufferFramesAvailableAverage, + audioMixerAvatarAudioStreamStats._ringBufferFramesAvailable); + + verticalOffset += STATS_HEIGHT_PER_LINE; + drawText(horizontalOffset, verticalOffset, scale, rotation, font, upstreamAudioStatsString, color); + + sprintf(upstreamAudioStatsString, " %s/%s/%s, %u/%u", formatUsecTime(audioMixerAvatarAudioStreamStats._timeGapMin).toLatin1().data(), + formatUsecTime(audioMixerAvatarAudioStreamStats._timeGapMax).toLatin1().data(), + formatUsecTime(audioMixerAvatarAudioStreamStats._timeGapAverage).toLatin1().data(), + audioMixerAvatarAudioStreamStats._ringBufferStarveCount, audioMixerAvatarAudioStreamStats._ringBufferOverflowCount); + + verticalOffset += STATS_HEIGHT_PER_LINE; + drawText(horizontalOffset, verticalOffset, scale, rotation, font, upstreamAudioStatsString, color); + + sprintf(upstreamAudioStatsString, " %s/%s/%s, %u/%u", formatUsecTime(audioMixerAvatarAudioStreamStats._timeGapWindowMin).toLatin1().data(), + formatUsecTime(audioMixerAvatarAudioStreamStats._timeGapWindowMax).toLatin1().data(), + formatUsecTime(audioMixerAvatarAudioStreamStats._timeGapWindowAverage).toLatin1().data(), + audioMixerAvatarAudioStreamStats._ringBufferConsecutiveNotMixedCount, audioMixerAvatarAudioStreamStats._ringBufferSilentFramesDropped); + + verticalOffset += STATS_HEIGHT_PER_LINE; + drawText(horizontalOffset, verticalOffset, scale, rotation, font, upstreamAudioStatsString, color); + + foreach(const AudioStreamStats& injectedStreamAudioStats, _audioMixerInjectedStreamAudioStatsMap) { + + sprintf(upstreamAudioStatsString, " inj: %.2f%%/%.2f%%, %u/%u/%u", injectedStreamAudioStats._packetStreamStats.getLostRate()*100.0f, + injectedStreamAudioStats._packetStreamWindowStats.getLostRate() * 100.0f, + injectedStreamAudioStats._ringBufferDesiredJitterBufferFrames, injectedStreamAudioStats._ringBufferFramesAvailableAverage, + injectedStreamAudioStats._ringBufferFramesAvailable); + + verticalOffset += STATS_HEIGHT_PER_LINE; + drawText(horizontalOffset, verticalOffset, scale, rotation, font, upstreamAudioStatsString, color); + + sprintf(upstreamAudioStatsString, " %s/%s/%s, %u/%u", formatUsecTime(injectedStreamAudioStats._timeGapMin).toLatin1().data(), + formatUsecTime(injectedStreamAudioStats._timeGapMax).toLatin1().data(), + formatUsecTime(injectedStreamAudioStats._timeGapAverage).toLatin1().data(), + injectedStreamAudioStats._ringBufferStarveCount, injectedStreamAudioStats._ringBufferOverflowCount); + + verticalOffset += STATS_HEIGHT_PER_LINE; + drawText(horizontalOffset, verticalOffset, scale, rotation, font, upstreamAudioStatsString, color); + + sprintf(upstreamAudioStatsString, " %s/%s/%s, %u/%u", formatUsecTime(injectedStreamAudioStats._timeGapWindowMin).toLatin1().data(), + formatUsecTime(injectedStreamAudioStats._timeGapWindowMax).toLatin1().data(), + formatUsecTime(injectedStreamAudioStats._timeGapWindowAverage).toLatin1().data(), + injectedStreamAudioStats._ringBufferConsecutiveNotMixedCount, injectedStreamAudioStats._ringBufferSilentFramesDropped); + + verticalOffset += STATS_HEIGHT_PER_LINE; + drawText(horizontalOffset, verticalOffset, scale, rotation, font, upstreamAudioStatsString, color); + } + + +} + void Audio::renderScope(int width, int height) { if (!_scopeEnabled) diff --git a/interface/src/Audio.h b/interface/src/Audio.h index f91208b6e4..fd10eeea11 100644 --- a/interface/src/Audio.h +++ b/interface/src/Audio.h @@ -71,6 +71,7 @@ public: void renderToolBox(int x, int y, bool boxed); void renderScope(int width, int height); + void renderStats(const float* color, int width, int height); int getNetworkSampleRate() { return SAMPLE_RATE; } int getNetworkBufferLengthSamplesPerChannel() { return NETWORK_BUFFER_LENGTH_SAMPLES_PER_CHANNEL; } @@ -100,6 +101,7 @@ public slots: void toggleToneInjection(); void toggleScope(); void toggleScopePause(); + void toggleStats(); void toggleAudioSpatialProcessing(); void toggleStereoInput(); void selectAudioScopeFiveFrames(); @@ -263,6 +265,10 @@ private: QByteArray* _scopeOutputLeft; QByteArray* _scopeOutputRight; + static const unsigned int STATS_WIDTH = 600; + static const unsigned int STATS_HEIGHT_PER_LINE = 20; + bool _statsEnabled; + int _starveCount; int _consecutiveNotMixedCount; diff --git a/interface/src/Menu.cpp b/interface/src/Menu.cpp index e21299da11..21edd11214 100644 --- a/interface/src/Menu.cpp +++ b/interface/src/Menu.cpp @@ -589,6 +589,12 @@ Menu::Menu() : Qt::CTRL | Qt::SHIFT | Qt::Key_U, false); + addCheckableActionToQMenuAndActionHash(audioDebugMenu, MenuOption::AudioStats, + 0, + false, + appInstance->getAudio(), + SLOT(toggleStats())); + addCheckableActionToQMenuAndActionHash(audioDebugMenu, MenuOption::DisableQAudioOutputOverflowCheck, 0, true); addActionToQMenuAndActionHash(developerMenu, MenuOption::PasteToVoxel, diff --git a/interface/src/Menu.h b/interface/src/Menu.h index efec231306..eb6a570ab1 100644 --- a/interface/src/Menu.h +++ b/interface/src/Menu.h @@ -315,6 +315,7 @@ namespace MenuOption { const QString AudioScopeFrames = "Display Frames"; const QString AudioScopePause = "Pause Audio Scope"; const QString AudioScopeTwentyFrames = "Twenty"; + const QString AudioStats = "Audio Stats"; const QString AudioSpatialProcessingAlternateDistanceAttenuate = "Alternate distance attenuation"; const QString AudioSpatialProcessing = "Audio Spatial Processing"; const QString AudioSpatialProcessingDontDistanceAttenuate = "Don't calculate distance attenuation"; diff --git a/interface/src/ui/ApplicationOverlay.cpp b/interface/src/ui/ApplicationOverlay.cpp index 7b3db26d8d..fdaf2aa407 100644 --- a/interface/src/ui/ApplicationOverlay.cpp +++ b/interface/src/ui/ApplicationOverlay.cpp @@ -1003,6 +1003,8 @@ void ApplicationOverlay::renderAudioMeter() { audio->renderScope(glWidget->width(), glWidget->height()); + audio->renderStats(WHITE_TEXT, glWidget->width(), glWidget->height()); + glBegin(GL_QUADS); if (isClipping) { glColor3f(1, 0, 0); From c8ac1732ecaf55c6da48b3490be15acb0f897f99 Mon Sep 17 00:00:00 2001 From: wangyix Date: Tue, 22 Jul 2014 14:53:45 -0700 Subject: [PATCH 04/15] removed audio stats from old location --- interface/src/ui/Stats.cpp | 124 +------------------------------------ 1 file changed, 1 insertion(+), 123 deletions(-) diff --git a/interface/src/ui/Stats.cpp b/interface/src/ui/Stats.cpp index 412de6e7b0..ef4603bc95 100644 --- a/interface/src/ui/Stats.cpp +++ b/interface/src/ui/Stats.cpp @@ -280,7 +280,7 @@ void Stats::display( Audio* audio = Application::getInstance()->getAudio(); const QHash& audioMixerInjectedStreamAudioStatsMap = audio->getAudioMixerInjectedStreamAudioStatsMap(); - lines = _expanded ? 13 + (audioMixerInjectedStreamAudioStatsMap.size() + 2) * 3 : 3; + lines = _expanded ? 4 : 3; drawBackground(backgroundColor, horizontalOffset, 0, _pingStatsWidth, lines * STATS_PELS_PER_LINE + 10); horizontalOffset += 5; @@ -313,128 +313,6 @@ void Stats::display( verticalOffset += STATS_PELS_PER_LINE; drawText(horizontalOffset, verticalOffset, scale, rotation, font, voxelMaxPing, color); - - char inputAudioLabelString[] = "Input: avail_avg_10s/avail"; - - verticalOffset += STATS_PELS_PER_LINE; - drawText(horizontalOffset, verticalOffset, scale, rotation, font, inputAudioLabelString, color); - - char inputAudioStatsString[512]; - sprintf(inputAudioStatsString, " %d/%d", audio->getInputRingBufferAverageFramesAvailable(), - audio->getInputRingBufferFramesAvailable()); - - verticalOffset += STATS_PELS_PER_LINE; - drawText(horizontalOffset, verticalOffset, scale, rotation, font, inputAudioStatsString, color); - - char audioMixerStatsLabelString[] = "AudioMixer stats:"; - char streamStatsFormatLabelString[] = "lost%/lost_30s%"; - char streamStatsFormatLabelString2[] = "desired/avail_avg_10s/avail"; - char streamStatsFormatLabelString3[] = "gaps: min/max/avg, starv/ovfl"; - char streamStatsFormatLabelString4[] = "gaps_30s: (same), notmix/sdrop"; - - verticalOffset += STATS_PELS_PER_LINE; - drawText(horizontalOffset, verticalOffset, scale, rotation, font, audioMixerStatsLabelString, color); - verticalOffset += STATS_PELS_PER_LINE; - drawText(horizontalOffset, verticalOffset, scale, rotation, font, streamStatsFormatLabelString, color); - verticalOffset += STATS_PELS_PER_LINE; - drawText(horizontalOffset, verticalOffset, scale, rotation, font, streamStatsFormatLabelString2, color); - verticalOffset += STATS_PELS_PER_LINE; - drawText(horizontalOffset, verticalOffset, scale, rotation, font, streamStatsFormatLabelString3, color); - verticalOffset += STATS_PELS_PER_LINE; - drawText(horizontalOffset, verticalOffset, scale, rotation, font, streamStatsFormatLabelString4, color); - - char downstreamLabelString[] = " Downstream:"; - verticalOffset += STATS_PELS_PER_LINE; - drawText(horizontalOffset, verticalOffset, scale, rotation, font, downstreamLabelString, color); - - char downstreamAudioStatsString[512]; - - AudioStreamStats downstreamAudioStreamStats = audio->getDownstreamAudioStreamStats(); - - sprintf(downstreamAudioStatsString, " mix: %.2f%%/%.2f%%, %u/%u+%d/%u+%d", downstreamAudioStreamStats._packetStreamStats.getLostRate()*100.0f, - downstreamAudioStreamStats._packetStreamWindowStats.getLostRate() * 100.0f, - downstreamAudioStreamStats._ringBufferDesiredJitterBufferFrames, downstreamAudioStreamStats._ringBufferFramesAvailableAverage, - audio->getOutputRingBufferAverageFramesAvailable(), - downstreamAudioStreamStats._ringBufferFramesAvailable, audio->getOutputRingBufferFramesAvailable()); - - verticalOffset += STATS_PELS_PER_LINE; - drawText(horizontalOffset, verticalOffset, scale, rotation, font, downstreamAudioStatsString, color); - - sprintf(downstreamAudioStatsString, " %s/%s/%s, %u/%u", formatUsecTime(downstreamAudioStreamStats._timeGapMin).toLatin1().data(), - formatUsecTime(downstreamAudioStreamStats._timeGapMax).toLatin1().data(), - formatUsecTime(downstreamAudioStreamStats._timeGapAverage).toLatin1().data(), - downstreamAudioStreamStats._ringBufferStarveCount, downstreamAudioStreamStats._ringBufferOverflowCount); - - verticalOffset += STATS_PELS_PER_LINE; - drawText(horizontalOffset, verticalOffset, scale, rotation, font, downstreamAudioStatsString, color); - - sprintf(downstreamAudioStatsString, " %s/%s/%s, %u/?", formatUsecTime(downstreamAudioStreamStats._timeGapWindowMin).toLatin1().data(), - formatUsecTime(downstreamAudioStreamStats._timeGapWindowMax).toLatin1().data(), - formatUsecTime(downstreamAudioStreamStats._timeGapWindowAverage).toLatin1().data(), - downstreamAudioStreamStats._ringBufferConsecutiveNotMixedCount); - - verticalOffset += STATS_PELS_PER_LINE; - drawText(horizontalOffset, verticalOffset, scale, rotation, font, downstreamAudioStatsString, color); - - - char upstreamLabelString[] = " Upstream:"; - verticalOffset += STATS_PELS_PER_LINE; - drawText(horizontalOffset, verticalOffset, scale, rotation, font, upstreamLabelString, color); - - char upstreamAudioStatsString[512]; - - const AudioStreamStats& audioMixerAvatarAudioStreamStats = audio->getAudioMixerAvatarStreamAudioStats(); - - sprintf(upstreamAudioStatsString, " mic: %.2f%%/%.2f%%, %u/%u/%u", audioMixerAvatarAudioStreamStats._packetStreamStats.getLostRate()*100.0f, - audioMixerAvatarAudioStreamStats._packetStreamWindowStats.getLostRate() * 100.0f, - audioMixerAvatarAudioStreamStats._ringBufferDesiredJitterBufferFrames, audioMixerAvatarAudioStreamStats._ringBufferFramesAvailableAverage, - audioMixerAvatarAudioStreamStats._ringBufferFramesAvailable); - - verticalOffset += STATS_PELS_PER_LINE; - drawText(horizontalOffset, verticalOffset, scale, rotation, font, upstreamAudioStatsString, color); - - sprintf(upstreamAudioStatsString, " %s/%s/%s, %u/%u", formatUsecTime(audioMixerAvatarAudioStreamStats._timeGapMin).toLatin1().data(), - formatUsecTime(audioMixerAvatarAudioStreamStats._timeGapMax).toLatin1().data(), - formatUsecTime(audioMixerAvatarAudioStreamStats._timeGapAverage).toLatin1().data(), - audioMixerAvatarAudioStreamStats._ringBufferStarveCount, audioMixerAvatarAudioStreamStats._ringBufferOverflowCount); - - verticalOffset += STATS_PELS_PER_LINE; - drawText(horizontalOffset, verticalOffset, scale, rotation, font, upstreamAudioStatsString, color); - - sprintf(upstreamAudioStatsString, " %s/%s/%s, %u/%u", formatUsecTime(audioMixerAvatarAudioStreamStats._timeGapWindowMin).toLatin1().data(), - formatUsecTime(audioMixerAvatarAudioStreamStats._timeGapWindowMax).toLatin1().data(), - formatUsecTime(audioMixerAvatarAudioStreamStats._timeGapWindowAverage).toLatin1().data(), - audioMixerAvatarAudioStreamStats._ringBufferConsecutiveNotMixedCount, audioMixerAvatarAudioStreamStats._ringBufferSilentFramesDropped); - - verticalOffset += STATS_PELS_PER_LINE; - drawText(horizontalOffset, verticalOffset, scale, rotation, font, upstreamAudioStatsString, color); - - foreach(const AudioStreamStats& injectedStreamAudioStats, audioMixerInjectedStreamAudioStatsMap) { - - sprintf(upstreamAudioStatsString, " inj: %.2f%%/%.2f%%, %u/%u/%u", injectedStreamAudioStats._packetStreamStats.getLostRate()*100.0f, - injectedStreamAudioStats._packetStreamWindowStats.getLostRate() * 100.0f, - injectedStreamAudioStats._ringBufferDesiredJitterBufferFrames, injectedStreamAudioStats._ringBufferFramesAvailableAverage, - injectedStreamAudioStats._ringBufferFramesAvailable); - - verticalOffset += STATS_PELS_PER_LINE; - drawText(horizontalOffset, verticalOffset, scale, rotation, font, upstreamAudioStatsString, color); - - sprintf(upstreamAudioStatsString, " %s/%s/%s, %u/%u", formatUsecTime(injectedStreamAudioStats._timeGapMin).toLatin1().data(), - formatUsecTime(injectedStreamAudioStats._timeGapMax).toLatin1().data(), - formatUsecTime(injectedStreamAudioStats._timeGapAverage).toLatin1().data(), - injectedStreamAudioStats._ringBufferStarveCount, injectedStreamAudioStats._ringBufferOverflowCount); - - verticalOffset += STATS_PELS_PER_LINE; - drawText(horizontalOffset, verticalOffset, scale, rotation, font, upstreamAudioStatsString, color); - - sprintf(upstreamAudioStatsString, " %s/%s/%s, %u/%u", formatUsecTime(injectedStreamAudioStats._timeGapWindowMin).toLatin1().data(), - formatUsecTime(injectedStreamAudioStats._timeGapWindowMax).toLatin1().data(), - formatUsecTime(injectedStreamAudioStats._timeGapWindowAverage).toLatin1().data(), - injectedStreamAudioStats._ringBufferConsecutiveNotMixedCount, injectedStreamAudioStats._ringBufferSilentFramesDropped); - - verticalOffset += STATS_PELS_PER_LINE; - drawText(horizontalOffset, verticalOffset, scale, rotation, font, upstreamAudioStatsString, color); - } } verticalOffset = 0; From d9692e4af4cd009fc3a67e5c17a95f337b24946a Mon Sep 17 00:00:00 2001 From: wangyix Date: Tue, 22 Jul 2014 17:34:35 -0700 Subject: [PATCH 05/15] changed a > to >= in handleAudioInput; added audio input buffer latency to stats; --- interface/src/Audio.cpp | 59 +++++++++++++++++++---------------------- interface/src/Audio.h | 9 ++++--- 2 files changed, 33 insertions(+), 35 deletions(-) diff --git a/interface/src/Audio.cpp b/interface/src/Audio.cpp index d200bee043..f351c73661 100644 --- a/interface/src/Audio.cpp +++ b/interface/src/Audio.cpp @@ -132,7 +132,8 @@ Audio::Audio(int16_t initialJitterBufferSamples, QObject* parent) : _outgoingAvatarAudioSequenceNumber(0), _incomingMixedAudioSequenceNumberStats(INCOMING_SEQ_STATS_HISTORY_LENGTH), _interframeTimeGapStats(TIME_GAPS_STATS_INTERVAL_SAMPLES, TIME_GAP_STATS_WINDOW_INTERVALS), - _inputRingBufferFramesAvailableStats(1, FRAMES_AVAILABLE_STATS_WINDOW_SECONDS), + _audioInputBufferMsecsDataAvailableStats(MSECS_PER_SECOND / (float)AUDIO_CALLBACK_MSECS * CALLBACK_ACCELERATOR_RATIO, FRAMES_AVAILABLE_STATS_WINDOW_SECONDS), + _inputRingBufferMsecsDataAvailableStats(1, FRAMES_AVAILABLE_STATS_WINDOW_SECONDS), _outputRingBufferFramesAvailableStats(1, FRAMES_AVAILABLE_STATS_WINDOW_SECONDS), _audioOutputBufferFramesAvailableStats(1, FRAMES_AVAILABLE_STATS_WINDOW_SECONDS) { @@ -167,7 +168,8 @@ void Audio::resetStats() { _interframeTimeGapStats.reset(); - _inputRingBufferFramesAvailableStats.reset(); + _audioInputBufferMsecsDataAvailableStats.reset(); + _inputRingBufferMsecsDataAvailableStats.reset(); _outputRingBufferFramesAvailableStats.reset(); _audioOutputBufferFramesAvailableStats.reset(); @@ -518,8 +520,11 @@ void Audio::handleAudioInput() { } _inputRingBuffer.writeData(inputByteArray.data(), inputByteArray.size()); + + float audioInputMsecsRead = inputByteArray.size() / (float)(_inputFormat.channelCount() * _inputFormat.sampleRate() * sizeof(int16_t)) * MSECS_PER_SECOND; + _audioInputBufferMsecsDataAvailableStats.update(audioInputMsecsRead); - while (_inputRingBuffer.samplesAvailable() > inputSamplesRequired) { + while (_inputRingBuffer.samplesAvailable() >= inputSamplesRequired) { int16_t* inputAudioSamples = new int16_t[inputSamplesRequired]; _inputRingBuffer.readSamples(inputAudioSamples, inputSamplesRequired); @@ -830,9 +835,10 @@ AudioStreamStats Audio::getDownstreamAudioStreamStats() const { void Audio::sendDownstreamAudioStatsPacket() { - _inputRingBufferFramesAvailableStats.update(getInputRingBufferFramesAvailable()); + // since this function is called every second, we'll sample some of our stats here + + _inputRingBufferMsecsDataAvailableStats.update(getInputRingBufferMsecsDataAvailable()); - // since this function is called every second, we'll sample the number of audio frames available here. _outputRingBufferFramesAvailableStats.update(_ringBuffer.framesAvailable()); _audioOutputBufferFramesAvailableStats.update(getOutputRingBufferFramesAvailable()); @@ -1421,62 +1427,54 @@ void Audio::renderStats(const float* color, int width, int height) { const float BUFFER_SEND_INTERVAL_MSECS = BUFFER_SEND_INTERVAL_USECS / (float)USECS_PER_MSEC; - float inputRingBufferLatency = 0.0f, networkRoundtripLatency = 0.0f, mixerRingBufferLatency = 0.0f, outputRingBufferLatency = 0.0f, audioOutputBufferLatency = 0.0f; + float audioInputBufferLatency = 0.0f, inputRingBufferLatency = 0.0f, networkRoundtripLatency = 0.0f, mixerRingBufferLatency = 0.0f, outputRingBufferLatency = 0.0f, audioOutputBufferLatency = 0.0f; SharedNodePointer audioMixerNodePointer = NodeList::getInstance()->soloNodeOfType(NodeType::AudioMixer); if (!audioMixerNodePointer.isNull()) { - inputRingBufferLatency = _inputRingBufferFramesAvailableStats.getWindowAverage() * BUFFER_SEND_INTERVAL_MSECS; + audioInputBufferLatency = _audioInputBufferMsecsDataAvailableStats.getWindowAverage(); + inputRingBufferLatency = getInputRingBufferAverageMsecsDataAvailable(); networkRoundtripLatency = audioMixerNodePointer->getPingMs(); mixerRingBufferLatency = _audioMixerAvatarStreamAudioStats._ringBufferFramesAvailableAverage * BUFFER_SEND_INTERVAL_MSECS; outputRingBufferLatency = _outputRingBufferFramesAvailableStats.getWindowAverage() * BUFFER_SEND_INTERVAL_MSECS; audioOutputBufferLatency = _audioOutputBufferFramesAvailableStats.getWindowAverage() * BUFFER_SEND_INTERVAL_MSECS; } - float totalLatency = inputRingBufferLatency + networkRoundtripLatency + mixerRingBufferLatency + outputRingBufferLatency + audioOutputBufferLatency; + float totalLatency = audioInputBufferLatency + inputRingBufferLatency + networkRoundtripLatency + mixerRingBufferLatency + outputRingBufferLatency + audioOutputBufferLatency; - - sprintf(latencyStatString, " Input ring buffer: %.1fms\n", inputRingBufferLatency); + sprintf(latencyStatString, " Audio input buffer: %7.2fms\n", audioInputBufferLatency); verticalOffset += STATS_HEIGHT_PER_LINE; drawText(horizontalOffset, verticalOffset, scale, rotation, font, latencyStatString, color); - sprintf(latencyStatString, " Network to mixer: %.1fms / 2\n", networkRoundtripLatency); + sprintf(latencyStatString, " Input ring buffer: %7.2fms\n", inputRingBufferLatency); verticalOffset += STATS_HEIGHT_PER_LINE; drawText(horizontalOffset, verticalOffset, scale, rotation, font, latencyStatString, color); - sprintf(latencyStatString, " AudioMixer ring buffer: %.1fms\n", mixerRingBufferLatency); + sprintf(latencyStatString, " Network to mixer: %7.2fms\n", networkRoundtripLatency / 2.0f); verticalOffset += STATS_HEIGHT_PER_LINE; drawText(horizontalOffset, verticalOffset, scale, rotation, font, latencyStatString, color); - sprintf(latencyStatString, " Network to client: %.1fms / 2\n", networkRoundtripLatency); + sprintf(latencyStatString, " AudioMixer ring buffer: %7.2fms\n", mixerRingBufferLatency); + verticalOffset += STATS_HEIGHT_PER_LINE; + drawText(horizontalOffset, verticalOffset, scale, rotation, font, latencyStatString, color); + + sprintf(latencyStatString, " Network to client: %7.2fms\n", networkRoundtripLatency / 2.0f); verticalOffset += STATS_HEIGHT_PER_LINE; drawText(horizontalOffset, verticalOffset, scale, rotation, font, latencyStatString, color); - sprintf(latencyStatString, " Output ring buffer: %.1fms\n", outputRingBufferLatency); + sprintf(latencyStatString, " Output ring buffer: %7.2fms\n", outputRingBufferLatency); verticalOffset += STATS_HEIGHT_PER_LINE; drawText(horizontalOffset, verticalOffset, scale, rotation, font, latencyStatString, color); - sprintf(latencyStatString, " Audio output buffer: %.1fms\n", audioOutputBufferLatency); + sprintf(latencyStatString, " Audio output buffer: %7.2fms\n", audioOutputBufferLatency); verticalOffset += STATS_HEIGHT_PER_LINE; drawText(horizontalOffset, verticalOffset, scale, rotation, font, latencyStatString, color); - sprintf(latencyStatString, " TOTAL: %.1fms\n", totalLatency); + sprintf(latencyStatString, " TOTAL: %7.2fms\n", totalLatency); verticalOffset += STATS_HEIGHT_PER_LINE; drawText(horizontalOffset, verticalOffset, scale, rotation, font, latencyStatString, color); verticalOffset += STATS_HEIGHT_PER_LINE; - char inputAudioLabelString[] = "Input: avail_avg_10s/avail"; - - verticalOffset += STATS_HEIGHT_PER_LINE; - drawText(horizontalOffset, verticalOffset, scale, rotation, font, inputAudioLabelString, color); - - char inputAudioStatsString[512]; - sprintf(inputAudioStatsString, " %d/%d", getInputRingBufferAverageFramesAvailable(), getInputRingBufferFramesAvailable()); - - verticalOffset += STATS_HEIGHT_PER_LINE; - drawText(horizontalOffset, verticalOffset, scale, rotation, font, inputAudioStatsString, color); - - char audioMixerStatsLabelString[] = "AudioMixer stats:"; char streamStatsFormatLabelString[] = " lost%/lost_30s%, desr/avail_avg_10s/avail"; char streamStatsFormatLabelString2[] = " gaps: min/max/avg, starv/ovfl"; @@ -1852,7 +1850,6 @@ int Audio::getOutputRingBufferFramesAvailable() const { / (sizeof(int16_t) * _ringBuffer.getNumFrameSamples()); } -int Audio::getInputRingBufferFramesAvailable() const { - float inputToNetworkInputRatio = calculateDeviceToNetworkInputRatio(_numInputCallbackBytes); - return _inputRingBuffer.samplesAvailable() / inputToNetworkInputRatio / _inputRingBuffer.getNumFrameSamples(); +float Audio::getInputRingBufferMsecsDataAvailable() const { + return _inputRingBuffer.samplesAvailable() / (float)(_inputFormat.channelCount() * _inputFormat.sampleRate()) * MSECS_PER_SECOND; } diff --git a/interface/src/Audio.h b/interface/src/Audio.h index fd10eeea11..feaa8d16b1 100644 --- a/interface/src/Audio.h +++ b/interface/src/Audio.h @@ -79,9 +79,9 @@ public: bool getProcessSpatialAudio() const { return _processSpatialAudio; } const SequenceNumberStats& getIncomingMixedAudioSequenceNumberStats() const { return _incomingMixedAudioSequenceNumberStats; } - - int getInputRingBufferFramesAvailable() const; - int getInputRingBufferAverageFramesAvailable() const { return (int)_inputRingBufferFramesAvailableStats.getWindowAverage(); } + + float getInputRingBufferMsecsDataAvailable() const; + float getInputRingBufferAverageMsecsDataAvailable() const { return (float)_inputRingBufferMsecsDataAvailableStats.getWindowAverage(); } int getOutputRingBufferFramesAvailable() const; int getOutputRingBufferAverageFramesAvailable() const { return (int)_audioOutputBufferFramesAvailableStats.getWindowAverage(); } @@ -280,7 +280,8 @@ private: MovingMinMaxAvg _interframeTimeGapStats; - MovingMinMaxAvg _inputRingBufferFramesAvailableStats; + MovingMinMaxAvg _audioInputBufferMsecsDataAvailableStats; + MovingMinMaxAvg _inputRingBufferMsecsDataAvailableStats; MovingMinMaxAvg _outputRingBufferFramesAvailableStats; MovingMinMaxAvg _audioOutputBufferFramesAvailableStats; From 5b0b4847278e47b554994df0755ac5cbc223b59c Mon Sep 17 00:00:00 2001 From: wangyix Date: Wed, 23 Jul 2014 11:00:48 -0700 Subject: [PATCH 06/15] finished new audio stats (looks good on Windows) --- interface/src/Audio.cpp | 175 ++++++++++++++++------------------------ interface/src/Audio.h | 6 +- 2 files changed, 75 insertions(+), 106 deletions(-) diff --git a/interface/src/Audio.cpp b/interface/src/Audio.cpp index f351c73661..9c7a02a0a3 100644 --- a/interface/src/Audio.cpp +++ b/interface/src/Audio.cpp @@ -1399,17 +1399,17 @@ void Audio::renderStats(const float* color, int width, int height) { return; } - const int MIN_LINES = 20; - const int STATS_BACKGROUND_HEIGHT = STATS_HEIGHT_PER_LINE * MIN_LINES; + const int LINES_WHEN_CENTERED = 30; + const int CENTERED_BACKGROUND_HEIGHT = STATS_HEIGHT_PER_LINE * LINES_WHEN_CENTERED; - int lines = _audioMixerInjectedStreamAudioStatsMap.size() * 3 + 23; - int statsHeight = STATS_HEIGHT_PER_LINE * std::max(lines, MIN_LINES); + int lines = _audioMixerInjectedStreamAudioStatsMap.size() * 7 + 23; + int statsHeight = STATS_HEIGHT_PER_LINE * lines; static const float backgroundColor[4] = { 0.2f, 0.2f, 0.2f, 0.6f }; int x = (width - STATS_WIDTH) / 2; - int y = (height - STATS_BACKGROUND_HEIGHT) / 2; + int y = (height - CENTERED_BACKGROUND_HEIGHT) / 2; int w = STATS_WIDTH; int h = statsHeight; renderBackground(backgroundColor, x, y, w, h); @@ -1440,31 +1440,31 @@ void Audio::renderStats(const float* color, int width, int height) { } float totalLatency = audioInputBufferLatency + inputRingBufferLatency + networkRoundtripLatency + mixerRingBufferLatency + outputRingBufferLatency + audioOutputBufferLatency; - sprintf(latencyStatString, " Audio input buffer: %7.2fms\n", audioInputBufferLatency); + sprintf(latencyStatString, " Audio input buffer: %7.2fms - avg msecs of samples read to the input ring buffer in last 10s", audioInputBufferLatency); verticalOffset += STATS_HEIGHT_PER_LINE; drawText(horizontalOffset, verticalOffset, scale, rotation, font, latencyStatString, color); - sprintf(latencyStatString, " Input ring buffer: %7.2fms\n", inputRingBufferLatency); + sprintf(latencyStatString, " Input ring buffer: %7.2fms - avg msecs of samples in input ring buffer in last 10s", inputRingBufferLatency); verticalOffset += STATS_HEIGHT_PER_LINE; drawText(horizontalOffset, verticalOffset, scale, rotation, font, latencyStatString, color); - sprintf(latencyStatString, " Network to mixer: %7.2fms\n", networkRoundtripLatency / 2.0f); + sprintf(latencyStatString, " Network to mixer: %7.2fms - half of last ping value calculated by the node list", networkRoundtripLatency / 2.0f); verticalOffset += STATS_HEIGHT_PER_LINE; drawText(horizontalOffset, verticalOffset, scale, rotation, font, latencyStatString, color); - sprintf(latencyStatString, " AudioMixer ring buffer: %7.2fms\n", mixerRingBufferLatency); + sprintf(latencyStatString, " AudioMixer ring buffer: %7.2fms - avg msecs of samples in audio mixer's ring buffer in last 10s", mixerRingBufferLatency); verticalOffset += STATS_HEIGHT_PER_LINE; drawText(horizontalOffset, verticalOffset, scale, rotation, font, latencyStatString, color); - sprintf(latencyStatString, " Network to client: %7.2fms\n", networkRoundtripLatency / 2.0f); + sprintf(latencyStatString, " Network to client: %7.2fms - half of last ping value calculated by the node list", networkRoundtripLatency / 2.0f); verticalOffset += STATS_HEIGHT_PER_LINE; drawText(horizontalOffset, verticalOffset, scale, rotation, font, latencyStatString, color); - sprintf(latencyStatString, " Output ring buffer: %7.2fms\n", outputRingBufferLatency); + sprintf(latencyStatString, " Output ring buffer: %7.2fms - avg msecs of samples in output ring buffer in last 10s", outputRingBufferLatency); verticalOffset += STATS_HEIGHT_PER_LINE; drawText(horizontalOffset, verticalOffset, scale, rotation, font, latencyStatString, color); - sprintf(latencyStatString, " Audio output buffer: %7.2fms\n", audioOutputBufferLatency); + sprintf(latencyStatString, " Audio output buffer: %7.2fms - avg msecs of samples in audio output buffer in last 10s", audioOutputBufferLatency); verticalOffset += STATS_HEIGHT_PER_LINE; drawText(horizontalOffset, verticalOffset, scale, rotation, font, latencyStatString, color); @@ -1472,119 +1472,84 @@ void Audio::renderStats(const float* color, int width, int height) { verticalOffset += STATS_HEIGHT_PER_LINE; drawText(horizontalOffset, verticalOffset, scale, rotation, font, latencyStatString, color); - verticalOffset += STATS_HEIGHT_PER_LINE; + + verticalOffset += STATS_HEIGHT_PER_LINE; // blank line - char audioMixerStatsLabelString[] = "AudioMixer stats:"; - char streamStatsFormatLabelString[] = " lost%/lost_30s%, desr/avail_avg_10s/avail"; - char streamStatsFormatLabelString2[] = " gaps: min/max/avg, starv/ovfl"; - char streamStatsFormatLabelString3[] = " gaps_30s: (same), notmix/sdrop"; - - verticalOffset += STATS_HEIGHT_PER_LINE; - drawText(horizontalOffset, verticalOffset, scale, rotation, font, audioMixerStatsLabelString, color); - verticalOffset += STATS_HEIGHT_PER_LINE; - drawText(horizontalOffset, verticalOffset, scale, rotation, font, streamStatsFormatLabelString, color); - verticalOffset += STATS_HEIGHT_PER_LINE; - drawText(horizontalOffset, verticalOffset, scale, rotation, font, streamStatsFormatLabelString2, color); - verticalOffset += STATS_HEIGHT_PER_LINE; - drawText(horizontalOffset, verticalOffset, scale, rotation, font, streamStatsFormatLabelString3, color); - - char downstreamLabelString[] = "Downstream:"; + char downstreamLabelString[] = "Downstream mixed audio stats:"; verticalOffset += STATS_HEIGHT_PER_LINE; drawText(horizontalOffset, verticalOffset, scale, rotation, font, downstreamLabelString, color); - char downstreamAudioStatsString[512]; + renderAudioStreamStats(getDownstreamAudioStreamStats(), horizontalOffset, verticalOffset, scale, rotation, font, color); - AudioStreamStats downstreamAudioStreamStats = getDownstreamAudioStreamStats(); - sprintf(downstreamAudioStatsString, " mix: %.2f%%/%.2f%%, %u/%u+%d/%u+%d", downstreamAudioStreamStats._packetStreamStats.getLostRate()*100.0f, - downstreamAudioStreamStats._packetStreamWindowStats.getLostRate() * 100.0f, - downstreamAudioStreamStats._ringBufferDesiredJitterBufferFrames, downstreamAudioStreamStats._ringBufferFramesAvailableAverage, - getOutputRingBufferAverageFramesAvailable(), - downstreamAudioStreamStats._ringBufferFramesAvailable, getOutputRingBufferFramesAvailable()); + verticalOffset += STATS_HEIGHT_PER_LINE; // blank line + char upstreamMicLabelString[] = "Upstream mic audio stats:"; verticalOffset += STATS_HEIGHT_PER_LINE; - drawText(horizontalOffset, verticalOffset, scale, rotation, font, downstreamAudioStatsString, color); + drawText(horizontalOffset, verticalOffset, scale, rotation, font, upstreamMicLabelString, color); - sprintf(downstreamAudioStatsString, " %s/%s/%s, %u/%u", formatUsecTime(downstreamAudioStreamStats._timeGapMin).toLatin1().data(), - formatUsecTime(downstreamAudioStreamStats._timeGapMax).toLatin1().data(), - formatUsecTime(downstreamAudioStreamStats._timeGapAverage).toLatin1().data(), - downstreamAudioStreamStats._ringBufferStarveCount, downstreamAudioStreamStats._ringBufferOverflowCount); + renderAudioStreamStats(_audioMixerAvatarStreamAudioStats, horizontalOffset, verticalOffset, scale, rotation, font, color); - verticalOffset += STATS_HEIGHT_PER_LINE; - drawText(horizontalOffset, verticalOffset, scale, rotation, font, downstreamAudioStatsString, color); - - sprintf(downstreamAudioStatsString, " %s/%s/%s, %u/?", formatUsecTime(downstreamAudioStreamStats._timeGapWindowMin).toLatin1().data(), - formatUsecTime(downstreamAudioStreamStats._timeGapWindowMax).toLatin1().data(), - formatUsecTime(downstreamAudioStreamStats._timeGapWindowAverage).toLatin1().data(), - downstreamAudioStreamStats._ringBufferConsecutiveNotMixedCount); - - verticalOffset += STATS_HEIGHT_PER_LINE; - drawText(horizontalOffset, verticalOffset, scale, rotation, font, downstreamAudioStatsString, color); - - - char upstreamLabelString[] = "Upstream:"; - verticalOffset += STATS_HEIGHT_PER_LINE; - drawText(horizontalOffset, verticalOffset, scale, rotation, font, upstreamLabelString, color); - - char upstreamAudioStatsString[512]; - - const AudioStreamStats& audioMixerAvatarAudioStreamStats = getAudioMixerAvatarStreamAudioStats(); - - sprintf(upstreamAudioStatsString, " mic: %.2f%%/%.2f%%, %u/%u/%u", audioMixerAvatarAudioStreamStats._packetStreamStats.getLostRate()*100.0f, - audioMixerAvatarAudioStreamStats._packetStreamWindowStats.getLostRate() * 100.0f, - audioMixerAvatarAudioStreamStats._ringBufferDesiredJitterBufferFrames, audioMixerAvatarAudioStreamStats._ringBufferFramesAvailableAverage, - audioMixerAvatarAudioStreamStats._ringBufferFramesAvailable); - - verticalOffset += STATS_HEIGHT_PER_LINE; - drawText(horizontalOffset, verticalOffset, scale, rotation, font, upstreamAudioStatsString, color); - - sprintf(upstreamAudioStatsString, " %s/%s/%s, %u/%u", formatUsecTime(audioMixerAvatarAudioStreamStats._timeGapMin).toLatin1().data(), - formatUsecTime(audioMixerAvatarAudioStreamStats._timeGapMax).toLatin1().data(), - formatUsecTime(audioMixerAvatarAudioStreamStats._timeGapAverage).toLatin1().data(), - audioMixerAvatarAudioStreamStats._ringBufferStarveCount, audioMixerAvatarAudioStreamStats._ringBufferOverflowCount); - - verticalOffset += STATS_HEIGHT_PER_LINE; - drawText(horizontalOffset, verticalOffset, scale, rotation, font, upstreamAudioStatsString, color); - - sprintf(upstreamAudioStatsString, " %s/%s/%s, %u/%u", formatUsecTime(audioMixerAvatarAudioStreamStats._timeGapWindowMin).toLatin1().data(), - formatUsecTime(audioMixerAvatarAudioStreamStats._timeGapWindowMax).toLatin1().data(), - formatUsecTime(audioMixerAvatarAudioStreamStats._timeGapWindowAverage).toLatin1().data(), - audioMixerAvatarAudioStreamStats._ringBufferConsecutiveNotMixedCount, audioMixerAvatarAudioStreamStats._ringBufferSilentFramesDropped); - - verticalOffset += STATS_HEIGHT_PER_LINE; - drawText(horizontalOffset, verticalOffset, scale, rotation, font, upstreamAudioStatsString, color); foreach(const AudioStreamStats& injectedStreamAudioStats, _audioMixerInjectedStreamAudioStatsMap) { - sprintf(upstreamAudioStatsString, " inj: %.2f%%/%.2f%%, %u/%u/%u", injectedStreamAudioStats._packetStreamStats.getLostRate()*100.0f, - injectedStreamAudioStats._packetStreamWindowStats.getLostRate() * 100.0f, - injectedStreamAudioStats._ringBufferDesiredJitterBufferFrames, injectedStreamAudioStats._ringBufferFramesAvailableAverage, - injectedStreamAudioStats._ringBufferFramesAvailable); + verticalOffset += STATS_HEIGHT_PER_LINE; // blank line + char upstreamInjectedLabelString[512]; + sprintf(upstreamInjectedLabelString, "Upstream injected audio stats: stream ID: %s", + injectedStreamAudioStats._streamIdentifier.toString().toLatin1().data()); verticalOffset += STATS_HEIGHT_PER_LINE; - drawText(horizontalOffset, verticalOffset, scale, rotation, font, upstreamAudioStatsString, color); + drawText(horizontalOffset, verticalOffset, scale, rotation, font, upstreamInjectedLabelString, color); - sprintf(upstreamAudioStatsString, " %s/%s/%s, %u/%u", formatUsecTime(injectedStreamAudioStats._timeGapMin).toLatin1().data(), - formatUsecTime(injectedStreamAudioStats._timeGapMax).toLatin1().data(), - formatUsecTime(injectedStreamAudioStats._timeGapAverage).toLatin1().data(), - injectedStreamAudioStats._ringBufferStarveCount, injectedStreamAudioStats._ringBufferOverflowCount); - - verticalOffset += STATS_HEIGHT_PER_LINE; - drawText(horizontalOffset, verticalOffset, scale, rotation, font, upstreamAudioStatsString, color); - - sprintf(upstreamAudioStatsString, " %s/%s/%s, %u/%u", formatUsecTime(injectedStreamAudioStats._timeGapWindowMin).toLatin1().data(), - formatUsecTime(injectedStreamAudioStats._timeGapWindowMax).toLatin1().data(), - formatUsecTime(injectedStreamAudioStats._timeGapWindowAverage).toLatin1().data(), - injectedStreamAudioStats._ringBufferConsecutiveNotMixedCount, injectedStreamAudioStats._ringBufferSilentFramesDropped); - - verticalOffset += STATS_HEIGHT_PER_LINE; - drawText(horizontalOffset, verticalOffset, scale, rotation, font, upstreamAudioStatsString, color); + renderAudioStreamStats(injectedStreamAudioStats, horizontalOffset, verticalOffset, scale, rotation, font, color); } - - } +void Audio::renderAudioStreamStats(const AudioStreamStats& streamStats, int horizontalOffset, int& verticalOffset, + float scale, float rotation, int font, const float* color) { + + char stringBuffer[512]; + + sprintf(stringBuffer, " Packet loss | overall: %5.2f%%, last_30s: %5.2f%%", + streamStats._packetStreamStats.getLostRate()*100.0f, + streamStats._packetStreamWindowStats.getLostRate() * 100.0f); + verticalOffset += STATS_HEIGHT_PER_LINE; + drawText(horizontalOffset, verticalOffset, scale, rotation, font, stringBuffer, color); + + sprintf(stringBuffer, " Ringbuffer frames | desired: %u, avg_available(10s): %u+%d, available: %u+%d", + streamStats._ringBufferDesiredJitterBufferFrames, + streamStats._ringBufferFramesAvailableAverage, + getOutputRingBufferAverageFramesAvailable(), + streamStats._ringBufferFramesAvailable, + getOutputRingBufferFramesAvailable()); + verticalOffset += STATS_HEIGHT_PER_LINE; + drawText(horizontalOffset, verticalOffset, scale, rotation, font, stringBuffer, color); + + sprintf(stringBuffer, " Ringbuffer stats | starves: %u, prev_starve_lasted: %u, frames_dropped: %u, overflows: %u", + streamStats._ringBufferStarveCount, + streamStats._ringBufferStarveCount, + streamStats._ringBufferConsecutiveNotMixedCount, + streamStats._ringBufferOverflowCount); + verticalOffset += STATS_HEIGHT_PER_LINE; + drawText(horizontalOffset, verticalOffset, scale, rotation, font, stringBuffer, color); + + sprintf(stringBuffer, " Inter-packet timegaps (overall) | min: %9s, max: %9s, avg: %9s", + formatUsecTime(streamStats._timeGapMin).toLatin1().data(), + formatUsecTime(streamStats._timeGapMax).toLatin1().data(), + formatUsecTime(streamStats._timeGapAverage).toLatin1().data()); + verticalOffset += STATS_HEIGHT_PER_LINE; + drawText(horizontalOffset, verticalOffset, scale, rotation, font, stringBuffer, color); + + sprintf(stringBuffer, " Inter-packet timegaps (last 30s) | min: %9s, max: %9s, avg: %9s", + formatUsecTime(streamStats._timeGapWindowMin).toLatin1().data(), + formatUsecTime(streamStats._timeGapWindowMax).toLatin1().data(), + formatUsecTime(streamStats._timeGapWindowAverage).toLatin1().data()); + verticalOffset += STATS_HEIGHT_PER_LINE; + drawText(horizontalOffset, verticalOffset, scale, rotation, font, stringBuffer, color); +} + + void Audio::renderScope(int width, int height) { if (!_scopeEnabled) diff --git a/interface/src/Audio.h b/interface/src/Audio.h index feaa8d16b1..857044c182 100644 --- a/interface/src/Audio.h +++ b/interface/src/Audio.h @@ -248,6 +248,10 @@ private: void renderGrid(const float* color, int x, int y, int width, int height, int rows, int cols); void renderLineStrip(const float* color, int x, int y, int n, int offset, const QByteArray* byteArray); + // audio stats methods for rendering + void renderAudioStreamStats(const AudioStreamStats& streamStats, int horizontalOffset, int& verticalOffset, + float scale, float rotation, int font, const float* color); + // Audio scope data static const unsigned int NETWORK_SAMPLES_PER_FRAME = NETWORK_BUFFER_LENGTH_SAMPLES_PER_CHANNEL; static const unsigned int DEFAULT_FRAMES_PER_SCOPE = 5; @@ -265,7 +269,7 @@ private: QByteArray* _scopeOutputLeft; QByteArray* _scopeOutputRight; - static const unsigned int STATS_WIDTH = 600; + static const unsigned int STATS_WIDTH = 1500; static const unsigned int STATS_HEIGHT_PER_LINE = 20; bool _statsEnabled; From ba4403b06315d0d5bc201b68ee42b5010c51ce74 Mon Sep 17 00:00:00 2001 From: wangyix Date: Wed, 23 Jul 2014 11:18:25 -0700 Subject: [PATCH 07/15] audio stats look good on Mac now --- interface/src/Audio.cpp | 4 ++-- interface/src/Audio.h | 5 ++++- 2 files changed, 6 insertions(+), 3 deletions(-) diff --git a/interface/src/Audio.cpp b/interface/src/Audio.cpp index 9c7a02a0a3..e55d30d753 100644 --- a/interface/src/Audio.cpp +++ b/interface/src/Audio.cpp @@ -1408,8 +1408,8 @@ void Audio::renderStats(const float* color, int width, int height) { static const float backgroundColor[4] = { 0.2f, 0.2f, 0.2f, 0.6f }; - int x = (width - STATS_WIDTH) / 2; - int y = (height - CENTERED_BACKGROUND_HEIGHT) / 2; + int x = std::max((width - (int)STATS_WIDTH) / 2, 0); + int y = std::max((height - CENTERED_BACKGROUND_HEIGHT) / 2, 0); int w = STATS_WIDTH; int h = statsHeight; renderBackground(backgroundColor, x, y, w, h); diff --git a/interface/src/Audio.h b/interface/src/Audio.h index 857044c182..1692a1b0ce 100644 --- a/interface/src/Audio.h +++ b/interface/src/Audio.h @@ -268,8 +268,11 @@ private: QByteArray* _scopeInput; QByteArray* _scopeOutputLeft; QByteArray* _scopeOutputRight; - +#ifdef _WIN32 static const unsigned int STATS_WIDTH = 1500; +#else + static const unsigned int STATS_WIDTH = 650; +#endif static const unsigned int STATS_HEIGHT_PER_LINE = 20; bool _statsEnabled; From 849c489b4c041aff183b39f4c35f78c735f47ec1 Mon Sep 17 00:00:00 2001 From: wangyix Date: Wed, 23 Jul 2014 11:20:30 -0700 Subject: [PATCH 08/15] removed warnings about parseData() overloading in ringbuffer classes --- assignment-client/src/audio/AudioMixerClientData.cpp | 4 ++-- assignment-client/src/audio/AvatarAudioRingBuffer.h | 2 +- libraries/audio/src/InjectedAudioRingBuffer.h | 2 +- libraries/audio/src/PositionalAudioRingBuffer.h | 2 +- 4 files changed, 5 insertions(+), 5 deletions(-) diff --git a/assignment-client/src/audio/AudioMixerClientData.cpp b/assignment-client/src/audio/AudioMixerClientData.cpp index b42a690d97..1536da7aa6 100644 --- a/assignment-client/src/audio/AudioMixerClientData.cpp +++ b/assignment-client/src/audio/AudioMixerClientData.cpp @@ -95,7 +95,7 @@ int AudioMixerClientData::parseData(const QByteArray& packet) { } case SequenceNumberStats::OnTime: { // ask the AvatarAudioRingBuffer instance to parse the data - avatarRingBuffer->parseData(packet); + avatarRingBuffer->parseData(packet, 0); break; } default: { @@ -139,7 +139,7 @@ int AudioMixerClientData::parseData(const QByteArray& packet) { } case SequenceNumberStats::OnTime: { // ask the AvatarAudioRingBuffer instance to parse the data - matchingInjectedRingBuffer->parseData(packet); + matchingInjectedRingBuffer->parseData(packet, 0); break; } default: { diff --git a/assignment-client/src/audio/AvatarAudioRingBuffer.h b/assignment-client/src/audio/AvatarAudioRingBuffer.h index df9dc3787a..22444cf32a 100644 --- a/assignment-client/src/audio/AvatarAudioRingBuffer.h +++ b/assignment-client/src/audio/AvatarAudioRingBuffer.h @@ -20,7 +20,7 @@ class AvatarAudioRingBuffer : public PositionalAudioRingBuffer { public: AvatarAudioRingBuffer(bool isStereo = false, bool dynamicJitterBuffer = false); - int parseData(const QByteArray& packet, int packetsSkipped = 0); + int parseData(const QByteArray& packet, int packetsSkipped); private: // disallow copying of AvatarAudioRingBuffer objects AvatarAudioRingBuffer(const AvatarAudioRingBuffer&); diff --git a/libraries/audio/src/InjectedAudioRingBuffer.h b/libraries/audio/src/InjectedAudioRingBuffer.h index f226d99b12..856121e2dd 100644 --- a/libraries/audio/src/InjectedAudioRingBuffer.h +++ b/libraries/audio/src/InjectedAudioRingBuffer.h @@ -20,7 +20,7 @@ class InjectedAudioRingBuffer : public PositionalAudioRingBuffer { public: InjectedAudioRingBuffer(const QUuid& streamIdentifier = QUuid(), bool dynamicJitterBuffer = false); - int parseData(const QByteArray& packet, int packetsSkipped = 0); + int parseData(const QByteArray& packet, int packetsSkipped); const QUuid& getStreamIdentifier() const { return _streamIdentifier; } float getRadius() const { return _radius; } diff --git a/libraries/audio/src/PositionalAudioRingBuffer.h b/libraries/audio/src/PositionalAudioRingBuffer.h index f0d6aff80b..7b7d81d216 100644 --- a/libraries/audio/src/PositionalAudioRingBuffer.h +++ b/libraries/audio/src/PositionalAudioRingBuffer.h @@ -45,7 +45,7 @@ public: PositionalAudioRingBuffer(PositionalAudioRingBuffer::Type type, bool isStereo = false, bool dynamicJitterBuffers = false); - virtual int parseData(const QByteArray& packet, int packetsSkipped = 0) = 0; + virtual int parseData(const QByteArray& packet, int packetsSkipped) = 0; int parsePositionalData(const QByteArray& positionalByteArray); int parseListenModeData(const QByteArray& listenModeByteArray); From 4a4d065f00ce19485050b38cbfa8f7c8c0964836 Mon Sep 17 00:00:00 2001 From: wangyix Date: Wed, 23 Jul 2014 12:06:31 -0700 Subject: [PATCH 09/15] another parseData change --- assignment-client/src/audio/AudioMixerClientData.cpp | 8 ++++---- assignment-client/src/audio/AvatarAudioRingBuffer.cpp | 2 +- assignment-client/src/audio/AvatarAudioRingBuffer.h | 2 +- libraries/audio/src/InjectedAudioRingBuffer.cpp | 2 +- libraries/audio/src/InjectedAudioRingBuffer.h | 2 +- libraries/audio/src/PositionalAudioRingBuffer.h | 2 +- 6 files changed, 9 insertions(+), 9 deletions(-) diff --git a/assignment-client/src/audio/AudioMixerClientData.cpp b/assignment-client/src/audio/AudioMixerClientData.cpp index 1536da7aa6..a4983e6a95 100644 --- a/assignment-client/src/audio/AudioMixerClientData.cpp +++ b/assignment-client/src/audio/AudioMixerClientData.cpp @@ -90,12 +90,12 @@ int AudioMixerClientData::parseData(const QByteArray& packet) { switch (packetArrivalInfo._status) { case SequenceNumberStats::Early: { int packetsLost = packetArrivalInfo._seqDiffFromExpected; - avatarRingBuffer->parseData(packet, packetsLost); + avatarRingBuffer->parseDataAndHandleDroppedPackets(packet, packetsLost); break; } case SequenceNumberStats::OnTime: { // ask the AvatarAudioRingBuffer instance to parse the data - avatarRingBuffer->parseData(packet, 0); + avatarRingBuffer->parseDataAndHandleDroppedPackets(packet, 0); break; } default: { @@ -134,12 +134,12 @@ int AudioMixerClientData::parseData(const QByteArray& packet) { switch (packetArrivalInfo._status) { case SequenceNumberStats::Early: { int packetsLost = packetArrivalInfo._seqDiffFromExpected; - matchingInjectedRingBuffer->parseData(packet, packetsLost); + matchingInjectedRingBuffer->parseDataAndHandleDroppedPackets(packet, packetsLost); break; } case SequenceNumberStats::OnTime: { // ask the AvatarAudioRingBuffer instance to parse the data - matchingInjectedRingBuffer->parseData(packet, 0); + matchingInjectedRingBuffer->parseDataAndHandleDroppedPackets(packet, 0); break; } default: { diff --git a/assignment-client/src/audio/AvatarAudioRingBuffer.cpp b/assignment-client/src/audio/AvatarAudioRingBuffer.cpp index f6edde7ac4..382e8de68b 100644 --- a/assignment-client/src/audio/AvatarAudioRingBuffer.cpp +++ b/assignment-client/src/audio/AvatarAudioRingBuffer.cpp @@ -18,7 +18,7 @@ AvatarAudioRingBuffer::AvatarAudioRingBuffer(bool isStereo, bool dynamicJitterBu } -int AvatarAudioRingBuffer::parseData(const QByteArray& packet, int packetsSkipped) { +int AvatarAudioRingBuffer::parseDataAndHandleDroppedPackets(const QByteArray& packet, int packetsSkipped) { frameReceivedUpdateTimingStats(); _shouldLoopbackForNode = (packetTypeForPacket(packet) == PacketTypeMicrophoneAudioWithEcho); diff --git a/assignment-client/src/audio/AvatarAudioRingBuffer.h b/assignment-client/src/audio/AvatarAudioRingBuffer.h index 22444cf32a..96233220cd 100644 --- a/assignment-client/src/audio/AvatarAudioRingBuffer.h +++ b/assignment-client/src/audio/AvatarAudioRingBuffer.h @@ -20,7 +20,7 @@ class AvatarAudioRingBuffer : public PositionalAudioRingBuffer { public: AvatarAudioRingBuffer(bool isStereo = false, bool dynamicJitterBuffer = false); - int parseData(const QByteArray& packet, int packetsSkipped); + int parseDataAndHandleDroppedPackets(const QByteArray& packet, int packetsSkipped); private: // disallow copying of AvatarAudioRingBuffer objects AvatarAudioRingBuffer(const AvatarAudioRingBuffer&); diff --git a/libraries/audio/src/InjectedAudioRingBuffer.cpp b/libraries/audio/src/InjectedAudioRingBuffer.cpp index da2d8336de..4723bca906 100644 --- a/libraries/audio/src/InjectedAudioRingBuffer.cpp +++ b/libraries/audio/src/InjectedAudioRingBuffer.cpp @@ -30,7 +30,7 @@ InjectedAudioRingBuffer::InjectedAudioRingBuffer(const QUuid& streamIdentifier, const uchar MAX_INJECTOR_VOLUME = 255; -int InjectedAudioRingBuffer::parseData(const QByteArray& packet, int packetsSkipped) { +int InjectedAudioRingBuffer::parseDataAndHandleDroppedPackets(const QByteArray& packet, int packetsSkipped) { frameReceivedUpdateTimingStats(); // setup a data stream to read from this packet diff --git a/libraries/audio/src/InjectedAudioRingBuffer.h b/libraries/audio/src/InjectedAudioRingBuffer.h index 856121e2dd..4a1f8b5292 100644 --- a/libraries/audio/src/InjectedAudioRingBuffer.h +++ b/libraries/audio/src/InjectedAudioRingBuffer.h @@ -20,7 +20,7 @@ class InjectedAudioRingBuffer : public PositionalAudioRingBuffer { public: InjectedAudioRingBuffer(const QUuid& streamIdentifier = QUuid(), bool dynamicJitterBuffer = false); - int parseData(const QByteArray& packet, int packetsSkipped); + int parseDataAndHandleDroppedPackets(const QByteArray& packet, int packetsSkipped); const QUuid& getStreamIdentifier() const { return _streamIdentifier; } float getRadius() const { return _radius; } diff --git a/libraries/audio/src/PositionalAudioRingBuffer.h b/libraries/audio/src/PositionalAudioRingBuffer.h index 7b7d81d216..0b14a12858 100644 --- a/libraries/audio/src/PositionalAudioRingBuffer.h +++ b/libraries/audio/src/PositionalAudioRingBuffer.h @@ -45,7 +45,7 @@ public: PositionalAudioRingBuffer(PositionalAudioRingBuffer::Type type, bool isStereo = false, bool dynamicJitterBuffers = false); - virtual int parseData(const QByteArray& packet, int packetsSkipped) = 0; + virtual int parseDataAndHandleDroppedPackets(const QByteArray& packet, int packetsSkipped) = 0; int parsePositionalData(const QByteArray& positionalByteArray); int parseListenModeData(const QByteArray& listenModeByteArray); From 5eaa57b3ade0edbdeb6bd97a0cdec28e97fba7b8 Mon Sep 17 00:00:00 2001 From: wangyix Date: Wed, 23 Jul 2014 12:10:25 -0700 Subject: [PATCH 10/15] removed more warnings about unused vars --- assignment-client/src/audio/AudioMixer.cpp | 8 -------- interface/src/ui/Stats.cpp | 1 - 2 files changed, 9 deletions(-) diff --git a/assignment-client/src/audio/AudioMixer.cpp b/assignment-client/src/audio/AudioMixer.cpp index b008a20aa7..42136f3e9d 100644 --- a/assignment-client/src/audio/AudioMixer.cpp +++ b/assignment-client/src/audio/AudioMixer.cpp @@ -237,14 +237,6 @@ void AudioMixer::addBufferToMixForListeningNodeWithBuffer(PositionalAudioRingBuf _clientSamples[delayedChannelIndex + SINGLE_STEREO_OFFSET] += delayBufferSample[1]; } - // The following code is pretty gross and redundant, but AFAIK it's the best way to avoid - // too many conditionals in handling the delay samples at the beginning of _clientSamples. - // Basically we try to take the samples in batches of four, and then handle the remainder - // conditionally to get rid of the rest. - - const int DOUBLE_STEREO_OFFSET = 4; - const int TRIPLE_STEREO_OFFSET = 6; - if (numSamplesDelay > 0) { // if there was a sample delay for this buffer, we need to pull samples prior to the nextOutput // to stick at the beginning diff --git a/interface/src/ui/Stats.cpp b/interface/src/ui/Stats.cpp index ef4603bc95..a5ff5e9cb1 100644 --- a/interface/src/ui/Stats.cpp +++ b/interface/src/ui/Stats.cpp @@ -278,7 +278,6 @@ void Stats::display( Audio* audio = Application::getInstance()->getAudio(); - const QHash& audioMixerInjectedStreamAudioStatsMap = audio->getAudioMixerInjectedStreamAudioStatsMap(); lines = _expanded ? 4 : 3; drawBackground(backgroundColor, horizontalOffset, 0, _pingStatsWidth, lines * STATS_PELS_PER_LINE + 10); From 142bda3ed481d4e656a9de70c0d4988e9644ce18 Mon Sep 17 00:00:00 2001 From: wangyix Date: Wed, 23 Jul 2014 12:43:26 -0700 Subject: [PATCH 11/15] fixed audio stats typos --- interface/src/Audio.cpp | 32 +++++++++++++++++++++----------- interface/src/Audio.h | 2 +- 2 files changed, 22 insertions(+), 12 deletions(-) diff --git a/interface/src/Audio.cpp b/interface/src/Audio.cpp index e55d30d753..ede27d4b20 100644 --- a/interface/src/Audio.cpp +++ b/interface/src/Audio.cpp @@ -1480,7 +1480,7 @@ void Audio::renderStats(const float* color, int width, int height) { verticalOffset += STATS_HEIGHT_PER_LINE; drawText(horizontalOffset, verticalOffset, scale, rotation, font, downstreamLabelString, color); - renderAudioStreamStats(getDownstreamAudioStreamStats(), horizontalOffset, verticalOffset, scale, rotation, font, color); + renderAudioStreamStats(getDownstreamAudioStreamStats(), horizontalOffset, verticalOffset, scale, rotation, font, color, true); verticalOffset += STATS_HEIGHT_PER_LINE; // blank line @@ -1507,29 +1507,39 @@ void Audio::renderStats(const float* color, int width, int height) { } void Audio::renderAudioStreamStats(const AudioStreamStats& streamStats, int horizontalOffset, int& verticalOffset, - float scale, float rotation, int font, const float* color) { + float scale, float rotation, int font, const float* color, bool isDownstreamStats) { char stringBuffer[512]; - sprintf(stringBuffer, " Packet loss | overall: %5.2f%%, last_30s: %5.2f%%", + sprintf(stringBuffer, " Packet loss | overall: %5.2f%% (%d lost), last_30s: %5.2f%% (%d lost)", streamStats._packetStreamStats.getLostRate()*100.0f, - streamStats._packetStreamWindowStats.getLostRate() * 100.0f); + streamStats._packetStreamStats._numLost, + streamStats._packetStreamWindowStats.getLostRate() * 100.0f, + streamStats._packetStreamWindowStats._numLost); verticalOffset += STATS_HEIGHT_PER_LINE; drawText(horizontalOffset, verticalOffset, scale, rotation, font, stringBuffer, color); - sprintf(stringBuffer, " Ringbuffer frames | desired: %u, avg_available(10s): %u+%d, available: %u+%d", - streamStats._ringBufferDesiredJitterBufferFrames, - streamStats._ringBufferFramesAvailableAverage, - getOutputRingBufferAverageFramesAvailable(), - streamStats._ringBufferFramesAvailable, - getOutputRingBufferFramesAvailable()); + if (isDownstreamStats) { + sprintf(stringBuffer, " Ringbuffer frames | desired: %u, avg_available(10s): %u+%d, available: %u+%d", + streamStats._ringBufferDesiredJitterBufferFrames, + streamStats._ringBufferFramesAvailableAverage, + getOutputRingBufferAverageFramesAvailable(), + streamStats._ringBufferFramesAvailable, + getOutputRingBufferFramesAvailable()); + } else { + sprintf(stringBuffer, " Ringbuffer frames | desired: %u, avg_available(10s): %u, available: %u", + streamStats._ringBufferDesiredJitterBufferFrames, + streamStats._ringBufferFramesAvailableAverage, + streamStats._ringBufferFramesAvailable); + } + verticalOffset += STATS_HEIGHT_PER_LINE; drawText(horizontalOffset, verticalOffset, scale, rotation, font, stringBuffer, color); sprintf(stringBuffer, " Ringbuffer stats | starves: %u, prev_starve_lasted: %u, frames_dropped: %u, overflows: %u", - streamStats._ringBufferStarveCount, streamStats._ringBufferStarveCount, streamStats._ringBufferConsecutiveNotMixedCount, + streamStats._ringBufferSilentFramesDropped, streamStats._ringBufferOverflowCount); verticalOffset += STATS_HEIGHT_PER_LINE; drawText(horizontalOffset, verticalOffset, scale, rotation, font, stringBuffer, color); diff --git a/interface/src/Audio.h b/interface/src/Audio.h index 1692a1b0ce..4346a5f352 100644 --- a/interface/src/Audio.h +++ b/interface/src/Audio.h @@ -250,7 +250,7 @@ private: // audio stats methods for rendering void renderAudioStreamStats(const AudioStreamStats& streamStats, int horizontalOffset, int& verticalOffset, - float scale, float rotation, int font, const float* color); + float scale, float rotation, int font, const float* color, bool isDownstreamStats = false); // Audio scope data static const unsigned int NETWORK_SAMPLES_PER_FRAME = NETWORK_BUFFER_LENGTH_SAMPLES_PER_CHANNEL; From bd8b7eb71085b06d795337f2edd4ba3aec2d9282 Mon Sep 17 00:00:00 2001 From: wangyix Date: Wed, 23 Jul 2014 12:44:16 -0700 Subject: [PATCH 12/15] minor spacing issue --- interface/src/Audio.cpp | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/interface/src/Audio.cpp b/interface/src/Audio.cpp index ede27d4b20..ca98ae194c 100644 --- a/interface/src/Audio.cpp +++ b/interface/src/Audio.cpp @@ -1512,7 +1512,7 @@ void Audio::renderAudioStreamStats(const AudioStreamStats& streamStats, int hori char stringBuffer[512]; sprintf(stringBuffer, " Packet loss | overall: %5.2f%% (%d lost), last_30s: %5.2f%% (%d lost)", - streamStats._packetStreamStats.getLostRate()*100.0f, + streamStats._packetStreamStats.getLostRate() * 100.0f, streamStats._packetStreamStats._numLost, streamStats._packetStreamWindowStats.getLostRate() * 100.0f, streamStats._packetStreamWindowStats._numLost); From 06ed8e4b63867136da7b69a2518b59c4b9f8617c Mon Sep 17 00:00:00 2001 From: wangyix Date: Wed, 23 Jul 2014 13:47:03 -0700 Subject: [PATCH 13/15] fixed some typos in audio stats; cleaned up code --- interface/src/Audio.cpp | 46 +++++++++++++++++++++-------------------- interface/src/Audio.h | 14 ++++++------- 2 files changed, 31 insertions(+), 29 deletions(-) diff --git a/interface/src/Audio.cpp b/interface/src/Audio.cpp index ca98ae194c..27a7dfc6e4 100644 --- a/interface/src/Audio.cpp +++ b/interface/src/Audio.cpp @@ -132,10 +132,10 @@ Audio::Audio(int16_t initialJitterBufferSamples, QObject* parent) : _outgoingAvatarAudioSequenceNumber(0), _incomingMixedAudioSequenceNumberStats(INCOMING_SEQ_STATS_HISTORY_LENGTH), _interframeTimeGapStats(TIME_GAPS_STATS_INTERVAL_SAMPLES, TIME_GAP_STATS_WINDOW_INTERVALS), - _audioInputBufferMsecsDataAvailableStats(MSECS_PER_SECOND / (float)AUDIO_CALLBACK_MSECS * CALLBACK_ACCELERATOR_RATIO, FRAMES_AVAILABLE_STATS_WINDOW_SECONDS), - _inputRingBufferMsecsDataAvailableStats(1, FRAMES_AVAILABLE_STATS_WINDOW_SECONDS), + _audioInputMsecsReadStats(MSECS_PER_SECOND / (float)AUDIO_CALLBACK_MSECS * CALLBACK_ACCELERATOR_RATIO, FRAMES_AVAILABLE_STATS_WINDOW_SECONDS), + _inputRingBufferMsecsAvailableStats(1, FRAMES_AVAILABLE_STATS_WINDOW_SECONDS), _outputRingBufferFramesAvailableStats(1, FRAMES_AVAILABLE_STATS_WINDOW_SECONDS), - _audioOutputBufferFramesAvailableStats(1, FRAMES_AVAILABLE_STATS_WINDOW_SECONDS) + _audioOutputMsecsUnplayedStats(1, FRAMES_AVAILABLE_STATS_WINDOW_SECONDS) { // clear the array of locally injected samples memset(_localProceduralSamples, 0, NETWORK_BUFFER_LENGTH_BYTES_PER_CHANNEL); @@ -168,11 +168,11 @@ void Audio::resetStats() { _interframeTimeGapStats.reset(); - _audioInputBufferMsecsDataAvailableStats.reset(); - _inputRingBufferMsecsDataAvailableStats.reset(); + _audioInputMsecsReadStats.reset(); + _inputRingBufferMsecsAvailableStats.reset(); _outputRingBufferFramesAvailableStats.reset(); - _audioOutputBufferFramesAvailableStats.reset(); + _audioOutputMsecsUnplayedStats.reset(); } void Audio::audioMixerKilled() { @@ -522,7 +522,7 @@ void Audio::handleAudioInput() { _inputRingBuffer.writeData(inputByteArray.data(), inputByteArray.size()); float audioInputMsecsRead = inputByteArray.size() / (float)(_inputFormat.channelCount() * _inputFormat.sampleRate() * sizeof(int16_t)) * MSECS_PER_SECOND; - _audioInputBufferMsecsDataAvailableStats.update(audioInputMsecsRead); + _audioInputMsecsReadStats.update(audioInputMsecsRead); while (_inputRingBuffer.samplesAvailable() >= inputSamplesRequired) { @@ -837,10 +837,10 @@ void Audio::sendDownstreamAudioStatsPacket() { // since this function is called every second, we'll sample some of our stats here - _inputRingBufferMsecsDataAvailableStats.update(getInputRingBufferMsecsDataAvailable()); + _inputRingBufferMsecsAvailableStats.update(getInputRingBufferMsecsAvailable()); _outputRingBufferFramesAvailableStats.update(_ringBuffer.framesAvailable()); - _audioOutputBufferFramesAvailableStats.update(getOutputRingBufferFramesAvailable()); + _audioOutputMsecsUnplayedStats.update(getAudioOutputMsecsUnplayed()); // push the current seq number stats into history, which moves the history window forward 1s // (since that's how often pushStatsToHistory() is called) @@ -1431,12 +1431,12 @@ void Audio::renderStats(const float* color, int width, int height) { SharedNodePointer audioMixerNodePointer = NodeList::getInstance()->soloNodeOfType(NodeType::AudioMixer); if (!audioMixerNodePointer.isNull()) { - audioInputBufferLatency = _audioInputBufferMsecsDataAvailableStats.getWindowAverage(); - inputRingBufferLatency = getInputRingBufferAverageMsecsDataAvailable(); + audioInputBufferLatency = _audioInputMsecsReadStats.getWindowAverage(); + inputRingBufferLatency = getInputRingBufferAverageMsecsAvailable(); networkRoundtripLatency = audioMixerNodePointer->getPingMs(); mixerRingBufferLatency = _audioMixerAvatarStreamAudioStats._ringBufferFramesAvailableAverage * BUFFER_SEND_INTERVAL_MSECS; outputRingBufferLatency = _outputRingBufferFramesAvailableStats.getWindowAverage() * BUFFER_SEND_INTERVAL_MSECS; - audioOutputBufferLatency = _audioOutputBufferFramesAvailableStats.getWindowAverage() * BUFFER_SEND_INTERVAL_MSECS; + audioOutputBufferLatency = _audioOutputMsecsUnplayedStats.getWindowAverage(); } float totalLatency = audioInputBufferLatency + inputRingBufferLatency + networkRoundtripLatency + mixerRingBufferLatency + outputRingBufferLatency + audioOutputBufferLatency; @@ -1520,12 +1520,14 @@ void Audio::renderAudioStreamStats(const AudioStreamStats& streamStats, int hori drawText(horizontalOffset, verticalOffset, scale, rotation, font, stringBuffer, color); if (isDownstreamStats) { + + const float BUFFER_SEND_INTERVAL_MSECS = BUFFER_SEND_INTERVAL_USECS / (float)USECS_PER_MSEC; sprintf(stringBuffer, " Ringbuffer frames | desired: %u, avg_available(10s): %u+%d, available: %u+%d", streamStats._ringBufferDesiredJitterBufferFrames, streamStats._ringBufferFramesAvailableAverage, - getOutputRingBufferAverageFramesAvailable(), + (int)(getAudioOutputAverageMsecsUnplayed() / BUFFER_SEND_INTERVAL_MSECS), streamStats._ringBufferFramesAvailable, - getOutputRingBufferFramesAvailable()); + (int)(getAudioOutputMsecsUnplayed() / BUFFER_SEND_INTERVAL_MSECS)); } else { sprintf(stringBuffer, " Ringbuffer frames | desired: %u, avg_available(10s): %u, available: %u", streamStats._ringBufferDesiredJitterBufferFrames, @@ -1817,14 +1819,14 @@ int Audio::calculateNumberOfFrameSamples(int numBytes) const { return frameSamples; } -int Audio::getOutputRingBufferFramesAvailable() const { - float networkOutputToOutputRatio = (_desiredOutputFormat.sampleRate() / (float)_outputFormat.sampleRate()) - * (_desiredOutputFormat.channelCount() / (float)_outputFormat.channelCount()); - - return (_audioOutput->bufferSize() - _audioOutput->bytesFree()) * networkOutputToOutputRatio - / (sizeof(int16_t) * _ringBuffer.getNumFrameSamples()); +float Audio::getAudioOutputMsecsUnplayed() const { + int bytesAudioOutputUnplayed = _audioOutput->bufferSize() - _audioOutput->bytesFree(); + float msecsAudioOutputUnplayed = bytesAudioOutputUnplayed / (float)_outputFormat.bytesForDuration(USECS_PER_MSEC); + return msecsAudioOutputUnplayed; } -float Audio::getInputRingBufferMsecsDataAvailable() const { - return _inputRingBuffer.samplesAvailable() / (float)(_inputFormat.channelCount() * _inputFormat.sampleRate()) * MSECS_PER_SECOND; +float Audio::getInputRingBufferMsecsAvailable() const { + int bytesInInputRingBuffer = _inputRingBuffer.samplesAvailable() * sizeof(int16_t); + float msecsInInputRingBuffer = bytesInInputRingBuffer / (float)(_inputFormat.bytesForDuration(USECS_PER_MSEC)); + return msecsInInputRingBuffer; } diff --git a/interface/src/Audio.h b/interface/src/Audio.h index 4346a5f352..67a951b8d9 100644 --- a/interface/src/Audio.h +++ b/interface/src/Audio.h @@ -80,11 +80,11 @@ public: const SequenceNumberStats& getIncomingMixedAudioSequenceNumberStats() const { return _incomingMixedAudioSequenceNumberStats; } - float getInputRingBufferMsecsDataAvailable() const; - float getInputRingBufferAverageMsecsDataAvailable() const { return (float)_inputRingBufferMsecsDataAvailableStats.getWindowAverage(); } + float getInputRingBufferMsecsAvailable() const; + float getInputRingBufferAverageMsecsAvailable() const { return (float)_inputRingBufferMsecsAvailableStats.getWindowAverage(); } - int getOutputRingBufferFramesAvailable() const; - int getOutputRingBufferAverageFramesAvailable() const { return (int)_audioOutputBufferFramesAvailableStats.getWindowAverage(); } + float getAudioOutputMsecsUnplayed() const; + float getAudioOutputAverageMsecsUnplayed() const { return (float)_audioOutputMsecsUnplayedStats.getWindowAverage(); } public slots: void start(); @@ -287,11 +287,11 @@ private: MovingMinMaxAvg _interframeTimeGapStats; - MovingMinMaxAvg _audioInputBufferMsecsDataAvailableStats; - MovingMinMaxAvg _inputRingBufferMsecsDataAvailableStats; + MovingMinMaxAvg _audioInputMsecsReadStats; + MovingMinMaxAvg _inputRingBufferMsecsAvailableStats; MovingMinMaxAvg _outputRingBufferFramesAvailableStats; - MovingMinMaxAvg _audioOutputBufferFramesAvailableStats; + MovingMinMaxAvg _audioOutputMsecsUnplayedStats; }; From 663cbb76dd8b122a4373848a6b607835f60a8c47 Mon Sep 17 00:00:00 2001 From: wangyix Date: Wed, 23 Jul 2014 14:37:11 -0700 Subject: [PATCH 14/15] removed seq num reset from Audio::reset() --- interface/src/Audio.cpp | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/interface/src/Audio.cpp b/interface/src/Audio.cpp index 27a7dfc6e4..bd6e51b820 100644 --- a/interface/src/Audio.cpp +++ b/interface/src/Audio.cpp @@ -152,7 +152,8 @@ void Audio::init(QGLWidget *parent) { void Audio::reset() { _ringBuffer.reset(); - _outgoingAvatarAudioSequenceNumber = 0; + // we don't want to reset seq numbers when space-bar reset occurs. + //_outgoingAvatarAudioSequenceNumber = 0; resetStats(); } From 3abd383cd3feac91d8a8aac37c38de3d2a7ba6df Mon Sep 17 00:00:00 2001 From: wangyix Date: Wed, 23 Jul 2014 15:26:49 -0700 Subject: [PATCH 15/15] cleaned up code more --- interface/src/Audio.cpp | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/interface/src/Audio.cpp b/interface/src/Audio.cpp index bd6e51b820..5054537287 100644 --- a/interface/src/Audio.cpp +++ b/interface/src/Audio.cpp @@ -522,7 +522,7 @@ void Audio::handleAudioInput() { _inputRingBuffer.writeData(inputByteArray.data(), inputByteArray.size()); - float audioInputMsecsRead = inputByteArray.size() / (float)(_inputFormat.channelCount() * _inputFormat.sampleRate() * sizeof(int16_t)) * MSECS_PER_SECOND; + float audioInputMsecsRead = inputByteArray.size() / (float)(_inputFormat.bytesForDuration(USECS_PER_MSEC)); _audioInputMsecsReadStats.update(audioInputMsecsRead); while (_inputRingBuffer.samplesAvailable() >= inputSamplesRequired) {