From a4f5ce2215116e433f321b7bd36650de939e3d13 Mon Sep 17 00:00:00 2001 From: wangyix Date: Mon, 23 Jun 2014 17:48:57 -0700 Subject: [PATCH] added stats for audiomixer jitter buffers lengths in Application --- assignment-client/src/audio/AudioMixer.cpp | 15 ++++++++--- .../src/audio/AudioMixerClientData.cpp | 25 +++++++++++++++++++ .../src/audio/AudioMixerClientData.h | 15 ++++++++++- interface/src/Audio.cpp | 12 ++++++++- interface/src/Audio.h | 6 +++++ interface/src/ui/Stats.cpp | 20 +++++++++++++-- libraries/audio/src/AudioRingBuffer.cpp | 8 ++++-- .../audio/src/PositionalAudioRingBuffer.cpp | 2 +- .../audio/src/PositionalAudioRingBuffer.h | 1 + libraries/networking/src/PacketHeaders.cpp | 2 ++ 10 files changed, 96 insertions(+), 10 deletions(-) diff --git a/assignment-client/src/audio/AudioMixer.cpp b/assignment-client/src/audio/AudioMixer.cpp index 42be1aea5a..4c2d7198c6 100644 --- a/assignment-client/src/audio/AudioMixer.cpp +++ b/assignment-client/src/audio/AudioMixer.cpp @@ -472,7 +472,7 @@ void AudioMixer::run() { QElapsedTimer timer; timer.start(); - char* clientMixBuffer = new char[NETWORK_BUFFER_LENGTH_BYTES_STEREO + char* clientMixBuffer = new char[NETWORK_BUFFER_LENGTH_BYTES_STEREO + sizeof(AudioMixerJitterBuffersStats) + numBytesForPacketHeaderGivenPacketType(PacketTypeMixedAudio)]; int usecToSleep = BUFFER_SEND_INTERVAL_USECS; @@ -546,10 +546,19 @@ void AudioMixer::run() { && ((AudioMixerClientData*) node->getLinkedData())->getAvatarAudioRingBuffer()) { prepareMixForListeningNode(node.data()); + // pack header int numBytesPacketHeader = populatePacketHeader(clientMixBuffer, PacketTypeMixedAudio); + char* dataAt = clientMixBuffer + numBytesPacketHeader; - memcpy(clientMixBuffer + numBytesPacketHeader, _clientSamples, NETWORK_BUFFER_LENGTH_BYTES_STEREO); - nodeList->writeDatagram(clientMixBuffer, NETWORK_BUFFER_LENGTH_BYTES_STEREO + numBytesPacketHeader, node); + // calculate and pack the jitter buffer size stats for this node + AudioMixerJitterBuffersStats jitterBuffersStats; + ((AudioMixerClientData*)node->getLinkedData())->calculateJitterBuffersStats(jitterBuffersStats); + memcpy(dataAt, &jitterBuffersStats, sizeof(AudioMixerJitterBuffersStats)); + dataAt += sizeof(AudioMixerJitterBuffersStats); + + // pack mixed audio + memcpy(dataAt, _clientSamples, NETWORK_BUFFER_LENGTH_BYTES_STEREO); + nodeList->writeDatagram(clientMixBuffer, dataAt - clientMixBuffer, node); ++_sumListeners; } diff --git a/assignment-client/src/audio/AudioMixerClientData.cpp b/assignment-client/src/audio/AudioMixerClientData.cpp index 0c41cc70f9..ebe12ead65 100644 --- a/assignment-client/src/audio/AudioMixerClientData.cpp +++ b/assignment-client/src/audio/AudioMixerClientData.cpp @@ -15,6 +15,7 @@ #include #include "InjectedAudioRingBuffer.h" +#include "SharedUtil.h" #include "AudioMixerClientData.h" @@ -138,3 +139,27 @@ void AudioMixerClientData::pushBuffersAfterFrameSend() { i++; } } + +void AudioMixerClientData::calculateJitterBuffersStats(AudioMixerJitterBuffersStats& stats) const { + int avatarJitterBufferFrames = 0; + int maxJitterBufferFrames = 0; + int sumJitterBufferFrames = 0; + + for (int i = 0; i < _ringBuffers.size(); i++) { + + int bufferJitterFrames = _ringBuffers[i]->getCurrentJitterBufferFrames(); + if (_ringBuffers[i]->getType() == PositionalAudioRingBuffer::Microphone) { + avatarJitterBufferFrames = bufferJitterFrames; + } + + if (bufferJitterFrames > maxJitterBufferFrames) { + maxJitterBufferFrames = bufferJitterFrames; + } + + sumJitterBufferFrames += bufferJitterFrames; + } + + stats.avatarJitterBufferFrames = avatarJitterBufferFrames; + stats.maxJitterBufferFrames = maxJitterBufferFrames; + stats.avgJitterBufferFrames = (float)sumJitterBufferFrames / (float)_ringBuffers.size(); +} diff --git a/assignment-client/src/audio/AudioMixerClientData.h b/assignment-client/src/audio/AudioMixerClientData.h index 3c4ddd3459..928f635643 100644 --- a/assignment-client/src/audio/AudioMixerClientData.h +++ b/assignment-client/src/audio/AudioMixerClientData.h @@ -14,10 +14,21 @@ #include #include -#include +#include "PositionalAudioRingBuffer.h" #include "AvatarAudioRingBuffer.h" +class AudioMixerJitterBuffersStats { +public: + AudioMixerJitterBuffersStats() + : avatarJitterBufferFrames(0), maxJitterBufferFrames(0), avgJitterBufferFrames(0) + {} + + int avatarJitterBufferFrames; + int maxJitterBufferFrames; + float avgJitterBufferFrames; +}; + class AudioMixerClientData : public NodeData { public: AudioMixerClientData(); @@ -29,6 +40,8 @@ public: int parseData(const QByteArray& packet); void checkBuffersBeforeFrameSend(AABox* checkSourceZone = NULL, AABox* listenerZone = NULL); void pushBuffersAfterFrameSend(); + + void calculateJitterBuffersStats(AudioMixerJitterBuffersStats& stats) const; private: QList _ringBuffers; }; diff --git a/interface/src/Audio.cpp b/interface/src/Audio.cpp index a0a85f8888..234ec31407 100644 --- a/interface/src/Audio.cpp +++ b/interface/src/Audio.cpp @@ -102,7 +102,8 @@ Audio::Audio(int16_t initialJitterBufferSamples, QObject* parent) : _samplesPerScope(NETWORK_SAMPLES_PER_FRAME * _framesPerScope), _scopeInput(0), _scopeOutputLeft(0), - _scopeOutputRight(0) + _scopeOutputRight(0), + _audioMixerJitterBufferStats() { // clear the array of locally injected samples memset(_localProceduralSamples, 0, NETWORK_BUFFER_LENGTH_BYTES_PER_CHANNEL); @@ -800,7 +801,16 @@ void Audio::toggleStereoInput() { } } +void Audio::parseAudioMixerJitterBuffersStats(const QByteArray& audioByteArray) { + + int numBytesPacketHeader = numBytesForPacketHeader(audioByteArray); + const char* dataAt = reinterpret_cast(audioByteArray.data() + numBytesPacketHeader); + + memcpy(&_audioMixerJitterBufferStats, dataAt, sizeof(AudioMixerJitterBuffersStats)); +} + void Audio::processReceivedAudio(const QByteArray& audioByteArray) { + parseAudioMixerJitterBuffersStats(audioByteArray); _ringBuffer.parseData(audioByteArray); float networkOutputToOutputRatio = (_desiredOutputFormat.sampleRate() / (float) _outputFormat.sampleRate()) diff --git a/interface/src/Audio.h b/interface/src/Audio.h index 74fc373cb0..8ba42b0c07 100644 --- a/interface/src/Audio.h +++ b/interface/src/Audio.h @@ -16,6 +16,7 @@ #include #include "InterfaceConfig.h" +#include "../../assignment-client/src/audio/AudioMixerClientData.h" #include #include @@ -102,6 +103,8 @@ public slots: float getInputVolume() const { return (_audioInput) ? _audioInput->volume() : 0.0f; } void setInputVolume(float volume) { if (_audioInput) _audioInput->setVolume(volume); } + const AudioMixerJitterBuffersStats& getAudioMixerJitterBuffersStats() const { return _audioMixerJitterBufferStats; } + signals: bool muteToggled(); void preProcessOriginalInboundAudio(unsigned int sampleTime, QByteArray& samples, const QAudioFormat& format); @@ -216,6 +219,8 @@ private: void renderGrid(const float* color, int x, int y, int width, int height, int rows, int cols); void renderLineStrip(const float* color, int x, int y, int n, int offset, const QByteArray* byteArray); + void parseAudioMixerJitterBuffersStats(const QByteArray& audioByteArray); + // Audio scope data static const unsigned int NETWORK_SAMPLES_PER_FRAME = NETWORK_BUFFER_LENGTH_SAMPLES_PER_CHANNEL; static const unsigned int DEFAULT_FRAMES_PER_SCOPE = 5; @@ -233,6 +238,7 @@ private: QByteArray* _scopeOutputLeft; QByteArray* _scopeOutputRight; + AudioMixerJitterBuffersStats _audioMixerJitterBufferStats; }; diff --git a/interface/src/ui/Stats.cpp b/interface/src/ui/Stats.cpp index fa62ecdb9b..17938a0f56 100644 --- a/interface/src/ui/Stats.cpp +++ b/interface/src/ui/Stats.cpp @@ -286,7 +286,7 @@ void Stats::display( pingVoxel = totalPingVoxel/voxelServerCount; } - lines = _expanded ? 4 : 3; + lines = _expanded ? 6 : 5; drawBackground(backgroundColor, horizontalOffset, 0, _pingStatsWidth, lines * STATS_PELS_PER_LINE + 10); horizontalOffset += 5; @@ -302,7 +302,6 @@ void Stats::display( char audioPing[30]; sprintf(audioPing, "Audio ping: %d", pingAudio); - char avatarPing[30]; sprintf(avatarPing, "Avatar ping: %d", pingAvatar); @@ -324,10 +323,27 @@ void Stats::display( drawText(horizontalOffset, verticalOffset, scale, rotation, font, voxelMaxPing, color); } + static const float MSECS_PER_FRAME = (float)NETWORK_BUFFER_LENGTH_SAMPLES_PER_CHANNEL * (float)MSECS_PER_SECOND / (float)SAMPLE_RATE; + + const AudioMixerJitterBuffersStats& audioMixerJitterBufferStats = + Application::getInstance()->getAudio()->getAudioMixerJitterBuffersStats(); + + char* audioMixerJitterBuffersStatsLabel = "AudioMixer j-buffers msecs:"; + char audioMixerJitterBuffersStats[30]; + sprintf(audioMixerJitterBuffersStats, "mic/max/avg: %.1f / %.1f / %.1f", audioMixerJitterBufferStats.avatarJitterBufferFrames * MSECS_PER_FRAME, + audioMixerJitterBufferStats.maxJitterBufferFrames * MSECS_PER_FRAME, audioMixerJitterBufferStats.avgJitterBufferFrames * MSECS_PER_FRAME); + + verticalOffset += STATS_PELS_PER_LINE; + drawText(horizontalOffset, verticalOffset, scale, rotation, font, audioMixerJitterBuffersStatsLabel, color); + verticalOffset += STATS_PELS_PER_LINE; + drawText(horizontalOffset, verticalOffset, scale, rotation, font, audioMixerJitterBuffersStats, color); + + verticalOffset = 0; horizontalOffset = _lastHorizontalOffset + _generalStatsWidth + _pingStatsWidth + 2; } + MyAvatar* myAvatar = Application::getInstance()->getAvatar(); glm::vec3 avatarPos = myAvatar->getPosition(); diff --git a/libraries/audio/src/AudioRingBuffer.cpp b/libraries/audio/src/AudioRingBuffer.cpp index 1b6bdaa5d8..8f7d3c5663 100644 --- a/libraries/audio/src/AudioRingBuffer.cpp +++ b/libraries/audio/src/AudioRingBuffer.cpp @@ -16,6 +16,7 @@ #include #include "PacketHeaders.h" +#include "../../../assignment-client/src/audio/AudioMixerClientData.h" #include "AudioRingBuffer.h" @@ -63,8 +64,11 @@ void AudioRingBuffer::resizeForFrameSize(qint64 numFrameSamples) { } int AudioRingBuffer::parseData(const QByteArray& packet) { - int numBytesPacketHeader = numBytesForPacketHeader(packet); - return writeData(packet.data() + numBytesPacketHeader, packet.size() - numBytesPacketHeader); + int numBytesBeforeAudioData = numBytesForPacketHeader(packet); + if (packetTypeForPacket(packet) == PacketTypeMixedAudio) { + numBytesBeforeAudioData += sizeof(AudioMixerJitterBuffersStats); + } + return writeData(packet.data() + numBytesBeforeAudioData, packet.size() - numBytesBeforeAudioData); } qint64 AudioRingBuffer::readSamples(int16_t* destination, qint64 maxSamples) { diff --git a/libraries/audio/src/PositionalAudioRingBuffer.cpp b/libraries/audio/src/PositionalAudioRingBuffer.cpp index 0e2e785973..264af75753 100644 --- a/libraries/audio/src/PositionalAudioRingBuffer.cpp +++ b/libraries/audio/src/PositionalAudioRingBuffer.cpp @@ -232,7 +232,7 @@ bool PositionalAudioRingBuffer::shouldBeAddedToMix() { void PositionalAudioRingBuffer::updateDesiredJitterBufferFrames() { - const float USECS_PER_FRAME = NETWORK_BUFFER_LENGTH_SAMPLES_PER_CHANNEL * USECS_PER_SECOND / (float)SAMPLE_RATE; + static const float USECS_PER_FRAME = NETWORK_BUFFER_LENGTH_SAMPLES_PER_CHANNEL * USECS_PER_SECOND / (float)SAMPLE_RATE; if (_interframeTimeGapStats.hasNewWindowMaxGapAvailable()) { _desiredJitterBufferFrames = ceilf((float)_interframeTimeGapStats.getWindowMaxGap() / USECS_PER_FRAME); diff --git a/libraries/audio/src/PositionalAudioRingBuffer.h b/libraries/audio/src/PositionalAudioRingBuffer.h index 8c2122f29e..92f8ca9576 100644 --- a/libraries/audio/src/PositionalAudioRingBuffer.h +++ b/libraries/audio/src/PositionalAudioRingBuffer.h @@ -76,6 +76,7 @@ public: void setListenerUnattenuatedZone(AABox* listenerUnattenuatedZone) { _listenerUnattenuatedZone = listenerUnattenuatedZone; } int getSamplesPerFrame() const { return _isStereo ? NETWORK_BUFFER_LENGTH_SAMPLES_STEREO : NETWORK_BUFFER_LENGTH_SAMPLES_PER_CHANNEL; } + int getCurrentJitterBufferFrames() const { return _currentJitterBufferFrames; } protected: // disallow copying of PositionalAudioRingBuffer objects diff --git a/libraries/networking/src/PacketHeaders.cpp b/libraries/networking/src/PacketHeaders.cpp index e2bc46b3be..293b5b0dfb 100644 --- a/libraries/networking/src/PacketHeaders.cpp +++ b/libraries/networking/src/PacketHeaders.cpp @@ -51,6 +51,8 @@ PacketVersion versionForPacketType(PacketType type) { case PacketTypeMicrophoneAudioWithEcho: case PacketTypeSilentAudioFrame: return 1; + case PacketTypeMixedAudio: + return 1; case PacketTypeAvatarData: return 3; case PacketTypeAvatarIdentity: