added TimeWeightedAvg to InboundAudioStream

_maxFramesOverDesired hardcoded right now

assignment-client/src/audio/AudioMixerClientData.cpp

@@ -224,7 +224,7 @@ QString AudioMixerClientData::getAudioStreamStatsString() const {
             + " starves:" + QString::number(streamStats._starveCount)
             + " not_mixed:" + QString::number(streamStats._consecutiveNotMixedCount)
             + " overflows:" + QString::number(streamStats._overflowCount)
-            + " silents_dropped:" + QString::number(streamStats._silentFramesDropped)
+            + " silents_dropped:" + QString::number(streamStats._framesDropped)
             + " lost%:" + QString::number(streamStats._packetStreamStats.getLostRate() * 100.0f, 'f', 2)
             + " lost%_30s:" + QString::number(streamStats._packetStreamWindowStats.getLostRate() * 100.0f, 'f', 2)
             + " min_gap:" + formatUsecTime(streamStats._timeGapMin)
@@ -248,7 +248,7 @@ QString AudioMixerClientData::getAudioStreamStatsString() const {
                 + " starves:" + QString::number(streamStats._starveCount)
                 + " not_mixed:" + QString::number(streamStats._consecutiveNotMixedCount)
                 + " overflows:" + QString::number(streamStats._overflowCount)
-                + " silents_dropped:" + QString::number(streamStats._silentFramesDropped)
+                + " silents_dropped:" + QString::number(streamStats._framesDropped)
                 + " lost%:" + QString::number(streamStats._packetStreamStats.getLostRate() * 100.0f, 'f', 2)
                 + " lost%_30s:" + QString::number(streamStats._packetStreamWindowStats.getLostRate() * 100.0f, 'f', 2)
                 + " min_gap:" + formatUsecTime(streamStats._timeGapMin)

interface/src/Audio.cpp

@@ -1443,7 +1443,7 @@ void Audio::renderAudioStreamStats(const AudioStreamStats& streamStats, int hori
     sprintf(stringBuffer, "                 Ringbuffer stats | starves: %u, prev_starve_lasted: %u, frames_dropped: %u, overflows: %u",
         streamStats._starveCount,
         streamStats._consecutiveNotMixedCount,
-        streamStats._silentFramesDropped,
+        streamStats._framesDropped,
         streamStats._overflowCount);
     verticalOffset += STATS_HEIGHT_PER_LINE;
     drawText(horizontalOffset, verticalOffset, scale, rotation, font, stringBuffer, color);

libraries/audio/src/AudioStreamStats.h

@@ -31,7 +31,7 @@ public:
         _starveCount(0),
         _consecutiveNotMixedCount(0),
         _overflowCount(0),
-        _silentFramesDropped(0),
+        _framesDropped(0),
         _packetStreamStats(),
         _packetStreamWindowStats()
     {}
@@ -52,7 +52,7 @@ public:
     quint32 _starveCount;
     quint32 _consecutiveNotMixedCount;
     quint32 _overflowCount;
-    quint32 _silentFramesDropped;
+    quint32 _framesDropped;
 
     PacketStreamStats _packetStreamStats;
     PacketStreamStats _packetStreamWindowStats;

libraries/audio/src/InboundAudioStream.cpp

@@ -12,7 +12,8 @@
 #include "InboundAudioStream.h"
 #include "PacketHeaders.h"
 
-InboundAudioStream::InboundAudioStream(int numFrameSamples, int numFramesCapacity, bool dynamicJitterBuffers, bool useStDevForJitterCalc) :
+InboundAudioStream::InboundAudioStream(int numFrameSamples, int numFramesCapacity,
+    bool dynamicJitterBuffers, /*int maxFramesOverDesired,*/ bool useStDevForJitterCalc) :
     _ringBuffer(numFrameSamples, false, numFramesCapacity),
     _lastPopSucceeded(false),
     _lastPopOutput(),
@@ -22,16 +23,19 @@ InboundAudioStream::InboundAudioStream(int numFrameSamples, int numFramesCapacit
     _calculatedJitterBufferFramesUsingMaxGap(0),
     _calculatedJitterBufferFramesUsingStDev(0),
     _desiredJitterBufferFrames(1),
+    _maxFramesOverDesired(20),//maxFramesOverDesired),  // PLACEHOLDER!!!!!!!!!
     _isStarved(true),
     _hasStarted(false),
     _consecutiveNotMixedCount(0),
     _starveCount(0),
     _silentFramesDropped(0),
+    _oldFramesDropped(0),
     _incomingSequenceNumberStats(INCOMING_SEQ_STATS_HISTORY_LENGTH_SECONDS),
     _lastFrameReceivedTime(0),
     _interframeTimeGapStatsForJitterCalc(TIME_GAPS_FOR_JITTER_CALC_INTERVAL_SAMPLES, TIME_GAPS_FOR_JITTER_CALC_WINDOW_INTERVALS),
     _interframeTimeGapStatsForStatsPacket(TIME_GAPS_FOR_STATS_PACKET_INTERVAL_SAMPLES, TIME_GAPS_FOR_STATS_PACKET_WINDOW_INTERVALS),
-    _framesAvailableStats(FRAMES_AVAILABLE_STATS_INTERVAL_SAMPLES, FRAMES_AVAILABLE_STATS_WINDOW_INTERVALS)
+    _framesAvailableStat(),
+    _framesAvailableAvg(0)
 {
 }
 
@@ -49,16 +53,19 @@ void InboundAudioStream::resetStats() {
     _consecutiveNotMixedCount = 0;
     _starveCount = 0;
     _silentFramesDropped = 0;
+    _oldFramesDropped = 0;
     _incomingSequenceNumberStats.reset();
     _lastFrameReceivedTime = 0;
     _interframeTimeGapStatsForJitterCalc.reset();
     _interframeTimeGapStatsForStatsPacket.reset();
-    _framesAvailableStats.reset();
+    _framesAvailableStat.reset();
+    _framesAvailableAvg = 0;
 }
 
 void InboundAudioStream::clearBuffer() {
     _ringBuffer.clear();
-    _framesAvailableStats.reset();
+    _framesAvailableStat.reset();
+    _framesAvailableAvg = 0;
 }
 
 int InboundAudioStream::parseData(const QByteArray& packet) {
@@ -99,11 +106,24 @@ int InboundAudioStream::parseData(const QByteArray& packet) {
         }
     }
 
-    if (_isStarved && _ringBuffer.framesAvailable() >= _desiredJitterBufferFrames) {
+    int framesAvailable = _ringBuffer.framesAvailable();
+    // if this stream was starved, check if we're still starved.
+    if (_isStarved && framesAvailable >= _desiredJitterBufferFrames) {
         _isStarved = false;
     }
+    // if the ringbuffer exceeds the desired size by more than the threshold specified,
+    // drop the oldest frames so the ringbuffer is down to the desired size.
+    if (framesAvailable > _desiredJitterBufferFrames + _maxFramesOverDesired) {
+        int framesToDrop = framesAvailable - (_desiredJitterBufferFrames + DESIRED_JITTER_BUFFER_FRAMES_PADDING);
+        _ringBuffer.shiftReadPosition(framesToDrop * _ringBuffer.getNumFrameSamples());
+        printf("dropped %d old frames\n", framesToDrop);
+        _framesAvailableStat.reset();
+        _framesAvailableAvg = 0;
 
-    _framesAvailableStats.update(_ringBuffer.framesAvailable());
+        _oldFramesDropped += framesToDrop;
+    }
+
+    framesAvailableChanged();
 
     return readBytes;
 }
@@ -119,6 +139,7 @@ bool InboundAudioStream::popFrames(int numFrames, bool starveOnFail) {
             // we have enough samples to pop, so we're good to mix
             _lastPopOutput = _ringBuffer.nextOutput();
             _ringBuffer.shiftReadPosition(numSamplesRequested);
+            framesAvailableChanged();
 
             _hasStarted = true;
             _lastPopSucceeded = true;
@@ -135,6 +156,15 @@ bool InboundAudioStream::popFrames(int numFrames, bool starveOnFail) {
     return _lastPopSucceeded;
 }
 
+void InboundAudioStream::framesAvailableChanged() {
+    _framesAvailableStat.updateWithSample(_ringBuffer.framesAvailable());
+    if (_framesAvailableStat.getElapsedUsecs() >= FRAMES_AVAILABLE_STATS_WINDOW_USECS) {
+        _framesAvailableAvg = (int)ceil(_framesAvailableStat.getAverage());
+        _framesAvailableStat.reset();
+        printf("10s samples filled; frames avail avg = %d\n", _framesAvailableAvg);
+    }
+}
+
 void InboundAudioStream::setToStarved() {
     if (!_isStarved && _ringBuffer.framesAvailable() < _desiredJitterBufferFrames) {
         starved();
@@ -209,37 +239,26 @@ SequenceNumberStats::ArrivalInfo InboundAudioStream::frameReceivedUpdateNetworkS
 }
 
 int InboundAudioStream::writeDroppableSilentSamples(int numSilentSamples) {
-
+    
-    // This adds some number of frames to the desired jitter buffer frames target we use.
+    // calculate how many silent frames we should drop.
-    // The larger this value is, the less aggressive we are about reducing the jitter buffer length.
-    // Setting this to 0 will try to get the jitter buffer to be exactly _desiredJitterBufferFrames long,
-    // which could lead immediately to a starve.
-    const int DESIRED_JITTER_BUFFER_FRAMES_PADDING = 1;
-
-    // calculate how many silent frames we should drop.  We only drop silent frames if
-    // the running avg num frames available has stabilized and it's more than
-    // our desired number of frames by the margin defined above.
     int samplesPerFrame = _ringBuffer.getNumFrameSamples();
+    int desiredJitterBufferFramesPlusPadding = _desiredJitterBufferFrames + DESIRED_JITTER_BUFFER_FRAMES_PADDING;
     int numSilentFramesToDrop = 0;
-    if (_framesAvailableStats.getNewStatsAvailableFlag() && _framesAvailableStats.isWindowFilled()
-        && numSilentSamples >= samplesPerFrame) {
-        _framesAvailableStats.clearNewStatsAvailableFlag();
-        int averageJitterBufferFrames = (int)getFramesAvailableAverage();
-        int desiredJitterBufferFramesPlusPadding = _desiredJitterBufferFrames + DESIRED_JITTER_BUFFER_FRAMES_PADDING;
 
-        if (averageJitterBufferFrames > desiredJitterBufferFramesPlusPadding) {
+    if (numSilentSamples >= samplesPerFrame && _framesAvailableAvg > desiredJitterBufferFramesPlusPadding) {
-            // our avg jitter buffer size exceeds its desired value, so ignore some silent
-            // frames to get that size as close to desired as possible
-            int numSilentFramesToDropDesired = averageJitterBufferFrames - desiredJitterBufferFramesPlusPadding;
-            int numSilentFramesReceived = numSilentSamples / samplesPerFrame;
-            numSilentFramesToDrop = std::min(numSilentFramesToDropDesired, numSilentFramesReceived);
 
-            // since we now have a new jitter buffer length, reset the frames available stats.
+        // our avg jitter buffer size exceeds its desired value, so ignore some silent
-            _framesAvailableStats.reset();
+        // frames to get that size as close to desired as possible
+        int numSilentFramesToDropDesired = _framesAvailableAvg - desiredJitterBufferFramesPlusPadding;
+        int numSilentFramesReceived = numSilentSamples / samplesPerFrame;
+        numSilentFramesToDrop = std::min(numSilentFramesToDropDesired, numSilentFramesReceived);
 
-            _silentFramesDropped += numSilentFramesToDrop;
+        // dont reset _framesAvailableAvg here; we want to be able to drop further silent frames
-        }
+        // without waiting for _framesAvailableStat to fill up to 10s of samples.
+        _framesAvailableAvg -= numSilentFramesToDrop;
+        _framesAvailableStat.reset();
     }
+
     return _ringBuffer.addSilentFrame(numSilentSamples - numSilentFramesToDrop * samplesPerFrame);
 }
 
@@ -258,12 +277,12 @@ AudioStreamStats InboundAudioStream::getAudioStreamStats() const {
     streamStats._timeGapWindowAverage = _interframeTimeGapStatsForStatsPacket.getWindowAverage();
 
     streamStats._framesAvailable = _ringBuffer.framesAvailable();
-    streamStats._framesAvailableAverage = _framesAvailableStats.getWindowAverage();
+    streamStats._framesAvailableAverage = _framesAvailableAvg;
     streamStats._desiredJitterBufferFrames = _desiredJitterBufferFrames;
     streamStats._starveCount = _starveCount;
     streamStats._consecutiveNotMixedCount = _consecutiveNotMixedCount;
     streamStats._overflowCount = _ringBuffer.getOverflowCount();
-    streamStats._silentFramesDropped = _silentFramesDropped;
+    streamStats._framesDropped = _silentFramesDropped + _oldFramesDropped;    // TODO: add separate stat for old frames dropped
 
     streamStats._packetStreamStats = _incomingSequenceNumberStats.getStats();
     streamStats._packetStreamWindowStats = _incomingSequenceNumberStats.getStatsForHistoryWindow();

libraries/audio/src/InboundAudioStream.h

@@ -19,6 +19,13 @@
 #include "AudioStreamStats.h"
 #include "PacketHeaders.h"
 #include "StdDev.h"
+#include "TimeWeightedAvg.h"
+
+// This adds some number of frames to the desired jitter buffer frames target we use when we're dropping frames.
+// The larger this value is, the less aggressive we are about reducing the jitter buffer length.
+// Setting this to 0 will try to get the jitter buffer to be exactly _desiredJitterBufferFrames long when dropping frames,
+// which could lead to a starve soon after.
+const int DESIRED_JITTER_BUFFER_FRAMES_PADDING = 1;
 
 // the time gaps stats for _desiredJitterBufferFrames calculation
 // will recalculate the max for the past 5000 samples every 500 samples
@@ -30,10 +37,7 @@ const int TIME_GAPS_FOR_JITTER_CALC_WINDOW_INTERVALS = 10;
 const int TIME_GAPS_FOR_STATS_PACKET_INTERVAL_SAMPLES = USECS_PER_SECOND / BUFFER_SEND_INTERVAL_USECS;
 const int TIME_GAPS_FOR_STATS_PACKET_WINDOW_INTERVALS = 30;
 
-// the stats for calculating the average frames available  will recalculate every ~1 second
+const int FRAMES_AVAILABLE_STATS_WINDOW_USECS = 10 * USECS_PER_SECOND;
-// and will include data for the past ~2 seconds 
-const int FRAMES_AVAILABLE_STATS_INTERVAL_SAMPLES = USECS_PER_SECOND / BUFFER_SEND_INTERVAL_USECS;
-const int FRAMES_AVAILABLE_STATS_WINDOW_INTERVALS = 10;
 
 // the internal history buffer of the incoming seq stats will cover 30s to calculate
 // packet loss % over last 30s
@@ -45,7 +49,9 @@ const int INBOUND_RING_BUFFER_FRAME_CAPACITY = 100;
 class InboundAudioStream : public NodeData {
     Q_OBJECT
 public:
-    InboundAudioStream(int numFrameSamples, int numFramesCapacity, bool dynamicJitterBuffers, bool useStDevForJitterCalc = false);
+    InboundAudioStream(int numFrameSamples, int numFramesCapacity,
+        bool dynamicJitterBuffers, //int maxFramesOverDesired,
+        bool useStDevForJitterCalc = false);
 
     void reset();
     void resetStats();
@@ -85,7 +91,7 @@ public:
     int getNumFrameSamples() const { return _ringBuffer.getNumFrameSamples(); }
     int getFrameCapacity() const { return _ringBuffer.getFrameCapacity(); }
     int getFramesAvailable() const { return _ringBuffer.framesAvailable(); }
-    double getFramesAvailableAverage() const { return _framesAvailableStats.getWindowAverage(); }
+    double getFramesAvailableAverage() const { return _framesAvailableAvg; }
 
     bool isStarved() const { return _isStarved; }
     bool hasStarted() const { return _hasStarted; }
@@ -103,6 +109,8 @@ private:
 
     int writeSamplesForDroppedPackets(int numSamples);
 
+    void framesAvailableChanged();
+
 protected:
     // disallow copying of InboundAudioStream objects
     InboundAudioStream(const InboundAudioStream&);
@@ -124,14 +132,21 @@ protected:
     bool _lastPopSucceeded;
     AudioRingBuffer::ConstIterator _lastPopOutput;
 
-    bool _dynamicJitterBuffers;
+    bool _dynamicJitterBuffers;         // if false, _desiredJitterBufferFrames is locked at 1 (old behavior)
-    bool _dynamicJitterBuffersOverride;
+    bool _dynamicJitterBuffersOverride; // used for locking the _desiredJitterBufferFrames to some number while running
+
+    // if jitter buffer is dynamic, this determines what method of calculating _desiredJitterBufferFrames 
+    // if true, Philip's timegap std dev calculation is used.  Otherwise, Freddy's max timegap calculation is used
     bool _useStDevForJitterCalc;
-    
     int _calculatedJitterBufferFramesUsingMaxGap;
     int _calculatedJitterBufferFramesUsingStDev;
+
     int _desiredJitterBufferFrames;
 
+    // if there are more than _desiredJitterBufferFrames + _maxFramesOverDesired frames, old ringbuffer frames
+    // will be dropped to keep audio delay from building up
+    int _maxFramesOverDesired;
+
     bool _isStarved;
     bool _hasStarted;
 
@@ -140,6 +155,7 @@ protected:
     int _consecutiveNotMixedCount;
     int _starveCount;
     int _silentFramesDropped;
+    int _oldFramesDropped;
 
     SequenceNumberStats _incomingSequenceNumberStats;
 
@@ -148,8 +164,8 @@ protected:
     StDev _stdev;
     MovingMinMaxAvg<quint64> _interframeTimeGapStatsForStatsPacket;
     
-    // TODO: change this to time-weighted moving avg
+    TimeWeightedAvg<int> _framesAvailableStat;
-    MovingMinMaxAvg<int> _framesAvailableStats;
+    int _framesAvailableAvg;
 };
 
 #endif // hifi_InboundAudioStream_h

libraries/shared/src/TimeWeightedAvg.h

@@ -0,0 +1,80 @@
+//
+//  TimeWeightedAvg.h
+//  libraries/shared/src
+//
+//  Created by Yixin Wang on 7/29/2014
+//  Copyright 2013 High Fidelity, Inc.
+//
+//  Distributed under the Apache License, Version 2.0.
+//  See the accompanying file LICENSE or http://www.apache.org/licenses/LICENSE-2.0.html
+//
+
+#ifndef hifi_TimeWeightedAvg_h
+#define hifi_TimeWeightedAvg_h
+
+#include "SharedUtil.h"
+
+template <typename T>
+class TimeWeightedAvg {
+
+public:
+
+    TimeWeightedAvg()
+        : _firstSampleTime(0),
+        _lastSample(),
+        _lastSampleTime(0),
+        _weightedSampleSumExcludingLastSample(0.0)
+    {}
+
+    void reset() {
+        _firstSampleTime = 0;
+        _lastSampleTime = 0;
+        _weightedSampleSumExcludingLastSample = 0.0;
+    }
+
+    void  updateWithSample(T sample) {
+        quint64 now = usecTimestampNow();
+
+        if (_firstSampleTime == 0) {
+            _firstSampleTime = now;
+        } else {
+            _weightedSampleSumExcludingLastSample = getWeightedSampleSum(now);
+        }
+
+        _lastSample = sample;
+        _lastSampleTime = now;
+    }
+
+    double getAverage() const {
+        if (_firstSampleTime == 0) {
+            return 0.0;
+        }
+        quint64 now = usecTimestampNow();
+        quint64 elapsed = now - _firstSampleTime;
+        return getWeightedSampleSum(now) / (double)elapsed;
+    }
+
+    quint64 getElapsedUsecs() const {
+        if (_firstSampleTime == 0) {
+            return 0;
+        }
+        return usecTimestampNow() - _firstSampleTime;
+    }
+
+private:
+    // if no sample has been collected yet, the return value is undefined
+    double getWeightedSampleSum(quint64 now) const {
+        quint64 lastSampleLasted = now - _lastSampleTime;
+        return _weightedSampleSumExcludingLastSample + (double)_lastSample * (double)lastSampleLasted;
+    }
+
+private:
+    quint64 _firstSampleTime;
+
+    T _lastSample;
+    quint64 _lastSampleTime;
+    
+    double _weightedSampleSumExcludingLastSample;
+};
+
+#endif // hifi_TimeWeightedAvg_h