From 71c23eac1e33d1efc98bc3994e4ee6b070b50901 Mon Sep 17 00:00:00 2001 From: wangyix Date: Tue, 29 Jul 2014 13:51:46 -0700 Subject: [PATCH] added TimeWeightedAvg to InboundAudioStream _maxFramesOverDesired hardcoded right now --- .../src/audio/AudioMixerClientData.cpp | 4 +- interface/src/Audio.cpp | 2 +- libraries/audio/src/AudioStreamStats.h | 4 +- libraries/audio/src/InboundAudioStream.cpp | 85 ++++++++++++------- libraries/audio/src/InboundAudioStream.h | 38 ++++++--- libraries/shared/src/TimeWeightedAvg.h | 80 +++++++++++++++++ 6 files changed, 164 insertions(+), 49 deletions(-) create mode 100644 libraries/shared/src/TimeWeightedAvg.h diff --git a/assignment-client/src/audio/AudioMixerClientData.cpp b/assignment-client/src/audio/AudioMixerClientData.cpp index e681a6ccbf..d3c16dc04e 100644 --- a/assignment-client/src/audio/AudioMixerClientData.cpp +++ b/assignment-client/src/audio/AudioMixerClientData.cpp @@ -224,7 +224,7 @@ QString AudioMixerClientData::getAudioStreamStatsString() const { + " starves:" + QString::number(streamStats._starveCount) + " not_mixed:" + QString::number(streamStats._consecutiveNotMixedCount) + " overflows:" + QString::number(streamStats._overflowCount) - + " silents_dropped:" + QString::number(streamStats._silentFramesDropped) + + " silents_dropped:" + QString::number(streamStats._framesDropped) + " lost%:" + QString::number(streamStats._packetStreamStats.getLostRate() * 100.0f, 'f', 2) + " lost%_30s:" + QString::number(streamStats._packetStreamWindowStats.getLostRate() * 100.0f, 'f', 2) + " min_gap:" + formatUsecTime(streamStats._timeGapMin) @@ -248,7 +248,7 @@ QString AudioMixerClientData::getAudioStreamStatsString() const { + " starves:" + QString::number(streamStats._starveCount) + " not_mixed:" + QString::number(streamStats._consecutiveNotMixedCount) + " overflows:" + QString::number(streamStats._overflowCount) - + " silents_dropped:" + QString::number(streamStats._silentFramesDropped) + + " silents_dropped:" + QString::number(streamStats._framesDropped) + " lost%:" + QString::number(streamStats._packetStreamStats.getLostRate() * 100.0f, 'f', 2) + " lost%_30s:" + QString::number(streamStats._packetStreamWindowStats.getLostRate() * 100.0f, 'f', 2) + " min_gap:" + formatUsecTime(streamStats._timeGapMin) diff --git a/interface/src/Audio.cpp b/interface/src/Audio.cpp index e830e5f6d4..1be38377a6 100644 --- a/interface/src/Audio.cpp +++ b/interface/src/Audio.cpp @@ -1443,7 +1443,7 @@ void Audio::renderAudioStreamStats(const AudioStreamStats& streamStats, int hori sprintf(stringBuffer, " Ringbuffer stats | starves: %u, prev_starve_lasted: %u, frames_dropped: %u, overflows: %u", streamStats._starveCount, streamStats._consecutiveNotMixedCount, - streamStats._silentFramesDropped, + streamStats._framesDropped, streamStats._overflowCount); verticalOffset += STATS_HEIGHT_PER_LINE; drawText(horizontalOffset, verticalOffset, scale, rotation, font, stringBuffer, color); diff --git a/libraries/audio/src/AudioStreamStats.h b/libraries/audio/src/AudioStreamStats.h index 784e163b3b..148fad6557 100644 --- a/libraries/audio/src/AudioStreamStats.h +++ b/libraries/audio/src/AudioStreamStats.h @@ -31,7 +31,7 @@ public: _starveCount(0), _consecutiveNotMixedCount(0), _overflowCount(0), - _silentFramesDropped(0), + _framesDropped(0), _packetStreamStats(), _packetStreamWindowStats() {} @@ -52,7 +52,7 @@ public: quint32 _starveCount; quint32 _consecutiveNotMixedCount; quint32 _overflowCount; - quint32 _silentFramesDropped; + quint32 _framesDropped; PacketStreamStats _packetStreamStats; PacketStreamStats _packetStreamWindowStats; diff --git a/libraries/audio/src/InboundAudioStream.cpp b/libraries/audio/src/InboundAudioStream.cpp index bfaa4c6d63..c70fd090ed 100644 --- a/libraries/audio/src/InboundAudioStream.cpp +++ b/libraries/audio/src/InboundAudioStream.cpp @@ -12,7 +12,8 @@ #include "InboundAudioStream.h" #include "PacketHeaders.h" -InboundAudioStream::InboundAudioStream(int numFrameSamples, int numFramesCapacity, bool dynamicJitterBuffers, bool useStDevForJitterCalc) : +InboundAudioStream::InboundAudioStream(int numFrameSamples, int numFramesCapacity, + bool dynamicJitterBuffers, /*int maxFramesOverDesired,*/ bool useStDevForJitterCalc) : _ringBuffer(numFrameSamples, false, numFramesCapacity), _lastPopSucceeded(false), _lastPopOutput(), @@ -22,16 +23,19 @@ InboundAudioStream::InboundAudioStream(int numFrameSamples, int numFramesCapacit _calculatedJitterBufferFramesUsingMaxGap(0), _calculatedJitterBufferFramesUsingStDev(0), _desiredJitterBufferFrames(1), + _maxFramesOverDesired(20),//maxFramesOverDesired), // PLACEHOLDER!!!!!!!!! _isStarved(true), _hasStarted(false), _consecutiveNotMixedCount(0), _starveCount(0), _silentFramesDropped(0), + _oldFramesDropped(0), _incomingSequenceNumberStats(INCOMING_SEQ_STATS_HISTORY_LENGTH_SECONDS), _lastFrameReceivedTime(0), _interframeTimeGapStatsForJitterCalc(TIME_GAPS_FOR_JITTER_CALC_INTERVAL_SAMPLES, TIME_GAPS_FOR_JITTER_CALC_WINDOW_INTERVALS), _interframeTimeGapStatsForStatsPacket(TIME_GAPS_FOR_STATS_PACKET_INTERVAL_SAMPLES, TIME_GAPS_FOR_STATS_PACKET_WINDOW_INTERVALS), - _framesAvailableStats(FRAMES_AVAILABLE_STATS_INTERVAL_SAMPLES, FRAMES_AVAILABLE_STATS_WINDOW_INTERVALS) + _framesAvailableStat(), + _framesAvailableAvg(0) { } @@ -49,16 +53,19 @@ void InboundAudioStream::resetStats() { _consecutiveNotMixedCount = 0; _starveCount = 0; _silentFramesDropped = 0; + _oldFramesDropped = 0; _incomingSequenceNumberStats.reset(); _lastFrameReceivedTime = 0; _interframeTimeGapStatsForJitterCalc.reset(); _interframeTimeGapStatsForStatsPacket.reset(); - _framesAvailableStats.reset(); + _framesAvailableStat.reset(); + _framesAvailableAvg = 0; } void InboundAudioStream::clearBuffer() { _ringBuffer.clear(); - _framesAvailableStats.reset(); + _framesAvailableStat.reset(); + _framesAvailableAvg = 0; } int InboundAudioStream::parseData(const QByteArray& packet) { @@ -99,11 +106,24 @@ int InboundAudioStream::parseData(const QByteArray& packet) { } } - if (_isStarved && _ringBuffer.framesAvailable() >= _desiredJitterBufferFrames) { + int framesAvailable = _ringBuffer.framesAvailable(); + // if this stream was starved, check if we're still starved. + if (_isStarved && framesAvailable >= _desiredJitterBufferFrames) { _isStarved = false; } + // if the ringbuffer exceeds the desired size by more than the threshold specified, + // drop the oldest frames so the ringbuffer is down to the desired size. + if (framesAvailable > _desiredJitterBufferFrames + _maxFramesOverDesired) { + int framesToDrop = framesAvailable - (_desiredJitterBufferFrames + DESIRED_JITTER_BUFFER_FRAMES_PADDING); + _ringBuffer.shiftReadPosition(framesToDrop * _ringBuffer.getNumFrameSamples()); + printf("dropped %d old frames\n", framesToDrop); + _framesAvailableStat.reset(); + _framesAvailableAvg = 0; - _framesAvailableStats.update(_ringBuffer.framesAvailable()); + _oldFramesDropped += framesToDrop; + } + + framesAvailableChanged(); return readBytes; } @@ -119,6 +139,7 @@ bool InboundAudioStream::popFrames(int numFrames, bool starveOnFail) { // we have enough samples to pop, so we're good to mix _lastPopOutput = _ringBuffer.nextOutput(); _ringBuffer.shiftReadPosition(numSamplesRequested); + framesAvailableChanged(); _hasStarted = true; _lastPopSucceeded = true; @@ -135,6 +156,15 @@ bool InboundAudioStream::popFrames(int numFrames, bool starveOnFail) { return _lastPopSucceeded; } +void InboundAudioStream::framesAvailableChanged() { + _framesAvailableStat.updateWithSample(_ringBuffer.framesAvailable()); + if (_framesAvailableStat.getElapsedUsecs() >= FRAMES_AVAILABLE_STATS_WINDOW_USECS) { + _framesAvailableAvg = (int)ceil(_framesAvailableStat.getAverage()); + _framesAvailableStat.reset(); + printf("10s samples filled; frames avail avg = %d\n", _framesAvailableAvg); + } +} + void InboundAudioStream::setToStarved() { if (!_isStarved && _ringBuffer.framesAvailable() < _desiredJitterBufferFrames) { starved(); @@ -209,37 +239,26 @@ SequenceNumberStats::ArrivalInfo InboundAudioStream::frameReceivedUpdateNetworkS } int InboundAudioStream::writeDroppableSilentSamples(int numSilentSamples) { - - // This adds some number of frames to the desired jitter buffer frames target we use. - // The larger this value is, the less aggressive we are about reducing the jitter buffer length. - // Setting this to 0 will try to get the jitter buffer to be exactly _desiredJitterBufferFrames long, - // which could lead immediately to a starve. - const int DESIRED_JITTER_BUFFER_FRAMES_PADDING = 1; - - // calculate how many silent frames we should drop. We only drop silent frames if - // the running avg num frames available has stabilized and it's more than - // our desired number of frames by the margin defined above. + + // calculate how many silent frames we should drop. int samplesPerFrame = _ringBuffer.getNumFrameSamples(); + int desiredJitterBufferFramesPlusPadding = _desiredJitterBufferFrames + DESIRED_JITTER_BUFFER_FRAMES_PADDING; int numSilentFramesToDrop = 0; - if (_framesAvailableStats.getNewStatsAvailableFlag() && _framesAvailableStats.isWindowFilled() - && numSilentSamples >= samplesPerFrame) { - _framesAvailableStats.clearNewStatsAvailableFlag(); - int averageJitterBufferFrames = (int)getFramesAvailableAverage(); - int desiredJitterBufferFramesPlusPadding = _desiredJitterBufferFrames + DESIRED_JITTER_BUFFER_FRAMES_PADDING; - if (averageJitterBufferFrames > desiredJitterBufferFramesPlusPadding) { - // our avg jitter buffer size exceeds its desired value, so ignore some silent - // frames to get that size as close to desired as possible - int numSilentFramesToDropDesired = averageJitterBufferFrames - desiredJitterBufferFramesPlusPadding; - int numSilentFramesReceived = numSilentSamples / samplesPerFrame; - numSilentFramesToDrop = std::min(numSilentFramesToDropDesired, numSilentFramesReceived); + if (numSilentSamples >= samplesPerFrame && _framesAvailableAvg > desiredJitterBufferFramesPlusPadding) { - // since we now have a new jitter buffer length, reset the frames available stats. - _framesAvailableStats.reset(); + // our avg jitter buffer size exceeds its desired value, so ignore some silent + // frames to get that size as close to desired as possible + int numSilentFramesToDropDesired = _framesAvailableAvg - desiredJitterBufferFramesPlusPadding; + int numSilentFramesReceived = numSilentSamples / samplesPerFrame; + numSilentFramesToDrop = std::min(numSilentFramesToDropDesired, numSilentFramesReceived); - _silentFramesDropped += numSilentFramesToDrop; - } + // dont reset _framesAvailableAvg here; we want to be able to drop further silent frames + // without waiting for _framesAvailableStat to fill up to 10s of samples. + _framesAvailableAvg -= numSilentFramesToDrop; + _framesAvailableStat.reset(); } + return _ringBuffer.addSilentFrame(numSilentSamples - numSilentFramesToDrop * samplesPerFrame); } @@ -258,12 +277,12 @@ AudioStreamStats InboundAudioStream::getAudioStreamStats() const { streamStats._timeGapWindowAverage = _interframeTimeGapStatsForStatsPacket.getWindowAverage(); streamStats._framesAvailable = _ringBuffer.framesAvailable(); - streamStats._framesAvailableAverage = _framesAvailableStats.getWindowAverage(); + streamStats._framesAvailableAverage = _framesAvailableAvg; streamStats._desiredJitterBufferFrames = _desiredJitterBufferFrames; streamStats._starveCount = _starveCount; streamStats._consecutiveNotMixedCount = _consecutiveNotMixedCount; streamStats._overflowCount = _ringBuffer.getOverflowCount(); - streamStats._silentFramesDropped = _silentFramesDropped; + streamStats._framesDropped = _silentFramesDropped + _oldFramesDropped; // TODO: add separate stat for old frames dropped streamStats._packetStreamStats = _incomingSequenceNumberStats.getStats(); streamStats._packetStreamWindowStats = _incomingSequenceNumberStats.getStatsForHistoryWindow(); diff --git a/libraries/audio/src/InboundAudioStream.h b/libraries/audio/src/InboundAudioStream.h index 958491bca1..c8d854eeb1 100644 --- a/libraries/audio/src/InboundAudioStream.h +++ b/libraries/audio/src/InboundAudioStream.h @@ -19,6 +19,13 @@ #include "AudioStreamStats.h" #include "PacketHeaders.h" #include "StdDev.h" +#include "TimeWeightedAvg.h" + +// This adds some number of frames to the desired jitter buffer frames target we use when we're dropping frames. +// The larger this value is, the less aggressive we are about reducing the jitter buffer length. +// Setting this to 0 will try to get the jitter buffer to be exactly _desiredJitterBufferFrames long when dropping frames, +// which could lead to a starve soon after. +const int DESIRED_JITTER_BUFFER_FRAMES_PADDING = 1; // the time gaps stats for _desiredJitterBufferFrames calculation // will recalculate the max for the past 5000 samples every 500 samples @@ -30,10 +37,7 @@ const int TIME_GAPS_FOR_JITTER_CALC_WINDOW_INTERVALS = 10; const int TIME_GAPS_FOR_STATS_PACKET_INTERVAL_SAMPLES = USECS_PER_SECOND / BUFFER_SEND_INTERVAL_USECS; const int TIME_GAPS_FOR_STATS_PACKET_WINDOW_INTERVALS = 30; -// the stats for calculating the average frames available will recalculate every ~1 second -// and will include data for the past ~2 seconds -const int FRAMES_AVAILABLE_STATS_INTERVAL_SAMPLES = USECS_PER_SECOND / BUFFER_SEND_INTERVAL_USECS; -const int FRAMES_AVAILABLE_STATS_WINDOW_INTERVALS = 10; +const int FRAMES_AVAILABLE_STATS_WINDOW_USECS = 10 * USECS_PER_SECOND; // the internal history buffer of the incoming seq stats will cover 30s to calculate // packet loss % over last 30s @@ -45,7 +49,9 @@ const int INBOUND_RING_BUFFER_FRAME_CAPACITY = 100; class InboundAudioStream : public NodeData { Q_OBJECT public: - InboundAudioStream(int numFrameSamples, int numFramesCapacity, bool dynamicJitterBuffers, bool useStDevForJitterCalc = false); + InboundAudioStream(int numFrameSamples, int numFramesCapacity, + bool dynamicJitterBuffers, //int maxFramesOverDesired, + bool useStDevForJitterCalc = false); void reset(); void resetStats(); @@ -85,7 +91,7 @@ public: int getNumFrameSamples() const { return _ringBuffer.getNumFrameSamples(); } int getFrameCapacity() const { return _ringBuffer.getFrameCapacity(); } int getFramesAvailable() const { return _ringBuffer.framesAvailable(); } - double getFramesAvailableAverage() const { return _framesAvailableStats.getWindowAverage(); } + double getFramesAvailableAverage() const { return _framesAvailableAvg; } bool isStarved() const { return _isStarved; } bool hasStarted() const { return _hasStarted; } @@ -103,6 +109,8 @@ private: int writeSamplesForDroppedPackets(int numSamples); + void framesAvailableChanged(); + protected: // disallow copying of InboundAudioStream objects InboundAudioStream(const InboundAudioStream&); @@ -124,14 +132,21 @@ protected: bool _lastPopSucceeded; AudioRingBuffer::ConstIterator _lastPopOutput; - bool _dynamicJitterBuffers; - bool _dynamicJitterBuffersOverride; + bool _dynamicJitterBuffers; // if false, _desiredJitterBufferFrames is locked at 1 (old behavior) + bool _dynamicJitterBuffersOverride; // used for locking the _desiredJitterBufferFrames to some number while running + + // if jitter buffer is dynamic, this determines what method of calculating _desiredJitterBufferFrames + // if true, Philip's timegap std dev calculation is used. Otherwise, Freddy's max timegap calculation is used bool _useStDevForJitterCalc; - int _calculatedJitterBufferFramesUsingMaxGap; int _calculatedJitterBufferFramesUsingStDev; + int _desiredJitterBufferFrames; + // if there are more than _desiredJitterBufferFrames + _maxFramesOverDesired frames, old ringbuffer frames + // will be dropped to keep audio delay from building up + int _maxFramesOverDesired; + bool _isStarved; bool _hasStarted; @@ -140,6 +155,7 @@ protected: int _consecutiveNotMixedCount; int _starveCount; int _silentFramesDropped; + int _oldFramesDropped; SequenceNumberStats _incomingSequenceNumberStats; @@ -148,8 +164,8 @@ protected: StDev _stdev; MovingMinMaxAvg _interframeTimeGapStatsForStatsPacket; - // TODO: change this to time-weighted moving avg - MovingMinMaxAvg _framesAvailableStats; + TimeWeightedAvg _framesAvailableStat; + int _framesAvailableAvg; }; #endif // hifi_InboundAudioStream_h diff --git a/libraries/shared/src/TimeWeightedAvg.h b/libraries/shared/src/TimeWeightedAvg.h new file mode 100644 index 0000000000..9412078413 --- /dev/null +++ b/libraries/shared/src/TimeWeightedAvg.h @@ -0,0 +1,80 @@ +// +// TimeWeightedAvg.h +// libraries/shared/src +// +// Created by Yixin Wang on 7/29/2014 +// Copyright 2013 High Fidelity, Inc. +// +// Distributed under the Apache License, Version 2.0. +// See the accompanying file LICENSE or http://www.apache.org/licenses/LICENSE-2.0.html +// + +#ifndef hifi_TimeWeightedAvg_h +#define hifi_TimeWeightedAvg_h + +#include "SharedUtil.h" + +template +class TimeWeightedAvg { + +public: + + TimeWeightedAvg() + : _firstSampleTime(0), + _lastSample(), + _lastSampleTime(0), + _weightedSampleSumExcludingLastSample(0.0) + {} + + void reset() { + _firstSampleTime = 0; + _lastSampleTime = 0; + _weightedSampleSumExcludingLastSample = 0.0; + } + + void updateWithSample(T sample) { + quint64 now = usecTimestampNow(); + + if (_firstSampleTime == 0) { + _firstSampleTime = now; + } else { + _weightedSampleSumExcludingLastSample = getWeightedSampleSum(now); + } + + _lastSample = sample; + _lastSampleTime = now; + } + + double getAverage() const { + if (_firstSampleTime == 0) { + return 0.0; + } + quint64 now = usecTimestampNow(); + quint64 elapsed = now - _firstSampleTime; + return getWeightedSampleSum(now) / (double)elapsed; + } + + quint64 getElapsedUsecs() const { + if (_firstSampleTime == 0) { + return 0; + } + return usecTimestampNow() - _firstSampleTime; + } + +private: + // if no sample has been collected yet, the return value is undefined + double getWeightedSampleSum(quint64 now) const { + quint64 lastSampleLasted = now - _lastSampleTime; + return _weightedSampleSumExcludingLastSample + (double)_lastSample * (double)lastSampleLasted; + } + +private: + quint64 _firstSampleTime; + + T _lastSample; + quint64 _lastSampleTime; + + double _weightedSampleSumExcludingLastSample; +}; + +#endif // hifi_TimeWeightedAvg_h