Merge branch 'master' of https://github.com/highfidelity/hifi

assignment-client/src/audio/AudioMixerClientData.cpp

@@ -61,7 +61,7 @@ int AudioMixerClientData::parseData(const QByteArray& packet) {
         || packetType == PacketTypeMicrophoneAudioNoEcho
         || packetType == PacketTypeSilentAudioFrame) {
 
-        _incomingAvatarAudioSequenceNumberStats.sequenceNumberReceived(sequence);
+        SequenceNumberStats::ArrivalInfo packetArrivalInfo = _incomingAvatarAudioSequenceNumberStats.sequenceNumberReceived(sequence);
 
         // grab the AvatarAudioRingBuffer from the vector (or create it if it doesn't exist)
         AvatarAudioRingBuffer* avatarRingBuffer = getAvatarAudioRingBuffer();
@@ -84,8 +84,24 @@ int AudioMixerClientData::parseData(const QByteArray& packet) {
             _ringBuffers.push_back(avatarRingBuffer);
         }
 
-        // ask the AvatarAudioRingBuffer instance to parse the data
+
-        avatarRingBuffer->parseData(packet);
+        // for now, late packets are simply discarded.  In the future, it may be good to insert them into their correct place
+        // in the ring buffer (if that frame hasn't been mixed yet)
+        switch (packetArrivalInfo._status) {
+            case SequenceNumberStats::Early: {
+                int packetsLost = packetArrivalInfo._seqDiffFromExpected;
+                avatarRingBuffer->parseData(packet, packetsLost);
+                break;
+            }
+            case SequenceNumberStats::OnTime: {
+                // ask the AvatarAudioRingBuffer instance to parse the data
+                avatarRingBuffer->parseData(packet);
+                break;
+            }
+            default: {
+                break;
+            }
+        }
     } else if (packetType == PacketTypeInjectAudio) {
         // this is injected audio
 
@@ -95,7 +111,8 @@ int AudioMixerClientData::parseData(const QByteArray& packet) {
         if (!_incomingInjectedAudioSequenceNumberStatsMap.contains(streamIdentifier)) {
             _incomingInjectedAudioSequenceNumberStatsMap.insert(streamIdentifier, SequenceNumberStats(INCOMING_SEQ_STATS_HISTORY_LENGTH));
         }
-        _incomingInjectedAudioSequenceNumberStatsMap[streamIdentifier].sequenceNumberReceived(sequence);
+        SequenceNumberStats::ArrivalInfo packetArrivalInfo = 
+            _incomingInjectedAudioSequenceNumberStatsMap[streamIdentifier].sequenceNumberReceived(sequence);
 
         InjectedAudioRingBuffer* matchingInjectedRingBuffer = NULL;
 
@@ -112,7 +129,23 @@ int AudioMixerClientData::parseData(const QByteArray& packet) {
             _ringBuffers.push_back(matchingInjectedRingBuffer);
         }
 
-        matchingInjectedRingBuffer->parseData(packet);
+        // for now, late packets are simply discarded.  In the future, it may be good to insert them into their correct place
+        // in the ring buffer (if that frame hasn't been mixed yet)
+        switch (packetArrivalInfo._status) {
+            case SequenceNumberStats::Early: {
+                int packetsLost = packetArrivalInfo._seqDiffFromExpected;
+                matchingInjectedRingBuffer->parseData(packet, packetsLost);
+                break;
+            }
+            case SequenceNumberStats::OnTime: {
+                // ask the AvatarAudioRingBuffer instance to parse the data
+                matchingInjectedRingBuffer->parseData(packet);
+                break;
+            }
+            default: {
+                break;
+            }
+        }
     } else if (packetType == PacketTypeAudioStreamStats) {
 
         const char* dataAt = packet.data();
@@ -198,7 +231,7 @@ AudioStreamStats AudioMixerClientData::getAudioStreamStatsOfStream(const Positio
     streamStats._timeGapWindowAverage = timeGapStats.getWindowAverage();
 
     streamStats._ringBufferFramesAvailable = ringBuffer->framesAvailable();
-    streamStats._ringBufferCurrentJitterBufferFrames = ringBuffer->getCurrentJitterBufferFrames();
+    streamStats._ringBufferFramesAvailableAverage = ringBuffer->getFramesAvailableAverage();
     streamStats._ringBufferDesiredJitterBufferFrames = ringBuffer->getDesiredJitterBufferFrames();
     streamStats._ringBufferStarveCount = ringBuffer->getStarveCount();
     streamStats._ringBufferConsecutiveNotMixedCount = ringBuffer->getConsecutiveNotMixedCount();
@@ -272,7 +305,7 @@ QString AudioMixerClientData::getAudioStreamStatsString() const {
     QString result;
     AudioStreamStats streamStats = _downstreamAudioStreamStats;
     result += "DOWNSTREAM.desired:" + QString::number(streamStats._ringBufferDesiredJitterBufferFrames)
-        + " current: ?"
+        + " available_avg_10s:" + QString::number(streamStats._ringBufferFramesAvailableAverage)
         + " available:" + QString::number(streamStats._ringBufferFramesAvailable)
         + " starves:" + QString::number(streamStats._ringBufferStarveCount)
         + " not_mixed:" + QString::number(streamStats._ringBufferConsecutiveNotMixedCount)
@@ -291,7 +324,8 @@ QString AudioMixerClientData::getAudioStreamStatsString() const {
     if (avatarRingBuffer) {
         AudioStreamStats streamStats = getAudioStreamStatsOfStream(avatarRingBuffer);
         result += " UPSTREAM.mic.desired:" + QString::number(streamStats._ringBufferDesiredJitterBufferFrames)
-            + " current:" + QString::number(streamStats._ringBufferCurrentJitterBufferFrames)
+            + " desired_calc:" + QString::number(avatarRingBuffer->getCalculatedDesiredJitterBufferFrames())
+            + " available_avg_10s:" + QString::number(streamStats._ringBufferFramesAvailableAverage)
             + " available:" + QString::number(streamStats._ringBufferFramesAvailable)
             + " starves:" + QString::number(streamStats._ringBufferStarveCount)
             + " not_mixed:" + QString::number(streamStats._ringBufferConsecutiveNotMixedCount)
@@ -313,7 +347,8 @@ QString AudioMixerClientData::getAudioStreamStatsString() const {
         if (_ringBuffers[i]->getType() == PositionalAudioRingBuffer::Injector) {
             AudioStreamStats streamStats = getAudioStreamStatsOfStream(_ringBuffers[i]);
             result += " UPSTREAM.inj.desired:" + QString::number(streamStats._ringBufferDesiredJitterBufferFrames)
-                + " current:" + QString::number(streamStats._ringBufferCurrentJitterBufferFrames)
+                + " desired_calc:" + QString::number(_ringBuffers[i]->getCalculatedDesiredJitterBufferFrames())
+                + " available_avg_10s:" + QString::number(streamStats._ringBufferFramesAvailableAverage)
                 + " available:" + QString::number(streamStats._ringBufferFramesAvailable)
                 + " starves:" + QString::number(streamStats._ringBufferStarveCount)
                 + " not_mixed:" + QString::number(streamStats._ringBufferConsecutiveNotMixedCount)

assignment-client/src/audio/AvatarAudioRingBuffer.cpp

@@ -18,10 +18,53 @@ AvatarAudioRingBuffer::AvatarAudioRingBuffer(bool isStereo, bool dynamicJitterBu
     
 }
 
-int AvatarAudioRingBuffer::parseData(const QByteArray& packet) {
+int AvatarAudioRingBuffer::parseData(const QByteArray& packet, int packetsSkipped) {
-    timeGapStatsFrameReceived();
+    frameReceivedUpdateTimingStats();
-    updateDesiredJitterBufferFrames();
 
     _shouldLoopbackForNode = (packetTypeForPacket(packet) == PacketTypeMicrophoneAudioWithEcho);
-    return PositionalAudioRingBuffer::parseData(packet);
+
+    // skip the packet header (includes the source UUID)
+    int readBytes = numBytesForPacketHeader(packet);
+
+    // skip the sequence number
+    readBytes += sizeof(quint16);
+
+    // hop over the channel flag that has already been read in AudioMixerClientData
+    readBytes += sizeof(quint8);
+    // read the positional data
+    readBytes += parsePositionalData(packet.mid(readBytes));
+
+    if (packetTypeForPacket(packet) == PacketTypeSilentAudioFrame) {
+        // this source had no audio to send us, but this counts as a packet
+        // write silence equivalent to the number of silent samples they just sent us
+        int16_t numSilentSamples;
+
+        memcpy(&numSilentSamples, packet.data() + readBytes, sizeof(int16_t));
+        readBytes += sizeof(int16_t);
+
+        // add silent samples for the dropped packets as well. 
+        // ASSUME that each dropped packet had same number of silent samples as this one
+        numSilentSamples *= (packetsSkipped + 1);
+
+        // NOTE: fixes a bug in old clients that would send garbage for their number of silentSamples
+        // CAN'T DO THIS because ScriptEngine.cpp sends frames of different size due to having a different sending interval
+        // (every 16.667ms) than Audio.cpp (every 10.667ms)
+        //numSilentSamples = getSamplesPerFrame();
+
+        addDroppableSilentSamples(numSilentSamples);
+
+    } else {
+        int numAudioBytes = packet.size() - readBytes;
+        int numAudioSamples = numAudioBytes / sizeof(int16_t);
+
+        // add silent samples for the dropped packets.
+        // ASSUME that each dropped packet had same number of samples as this one
+        if (packetsSkipped > 0) {
+            addDroppableSilentSamples(packetsSkipped * numAudioSamples);
+        }
+
+        // there is audio data to read
+        readBytes += writeData(packet.data() + readBytes, numAudioBytes);
+    }
+    return readBytes;
 }

assignment-client/src/audio/AvatarAudioRingBuffer.h

@@ -20,7 +20,7 @@ class AvatarAudioRingBuffer : public PositionalAudioRingBuffer {
 public:
     AvatarAudioRingBuffer(bool isStereo = false, bool dynamicJitterBuffer = false);
     
-    int parseData(const QByteArray& packet);
+    int parseData(const QByteArray& packet, int packetsSkipped = 0);
 private:
     // disallow copying of AvatarAudioRingBuffer objects
     AvatarAudioRingBuffer(const AvatarAudioRingBuffer&);

interface/src/Audio.cpp

@@ -56,6 +56,10 @@ static const int TIME_GAP_STATS_WINDOW_INTERVALS = 30;
 static const int INCOMING_SEQ_STATS_HISTORY_LENGTH = INCOMING_SEQ_STATS_HISTORY_LENGTH_SECONDS /
         (TOO_LONG_SINCE_LAST_SEND_DOWNSTREAM_AUDIO_STATS / USECS_PER_SECOND);
 
+// the stats for the total frames available in the ring buffer and the audio output buffer
+// will sample every second, update every second, and have a moving window covering 10 seconds
+static const int FRAMES_AVAILABLE_STATS_WINDOW_SECONDS = 10;
+
 // Mute icon configration
 static const int MUTE_ICON_SIZE = 24;
 
@@ -116,7 +120,8 @@ Audio::Audio(int16_t initialJitterBufferSamples, QObject* parent) :
     _consecutiveNotMixedCount(0),
     _outgoingAvatarAudioSequenceNumber(0),
     _incomingMixedAudioSequenceNumberStats(INCOMING_SEQ_STATS_HISTORY_LENGTH),
-    _interframeTimeGapStats(TIME_GAPS_STATS_INTERVAL_SAMPLES, TIME_GAP_STATS_WINDOW_INTERVALS)
+    _interframeTimeGapStats(TIME_GAPS_STATS_INTERVAL_SAMPLES, TIME_GAP_STATS_WINDOW_INTERVALS),
+    _framesAvailableStats(1, FRAMES_AVAILABLE_STATS_WINDOW_SECONDS)
 {
     // clear the array of locally injected samples
     memset(_localProceduralSamples, 0, NETWORK_BUFFER_LENGTH_BYTES_PER_CHANNEL);
@@ -779,8 +784,8 @@ AudioStreamStats Audio::getDownstreamAudioStreamStats() const {
     stats._timeGapWindowMax = _interframeTimeGapStats.getWindowMax();
     stats._timeGapWindowAverage = _interframeTimeGapStats.getWindowAverage();
 
-    stats._ringBufferFramesAvailable = _ringBuffer.framesAvailable();
+    stats._ringBufferFramesAvailable = getFramesAvailableInRingAndAudioOutputBuffers();
-    stats._ringBufferCurrentJitterBufferFrames = 0;
+    stats._ringBufferFramesAvailableAverage = _framesAvailableStats.getWindowAverage();
     stats._ringBufferDesiredJitterBufferFrames = getDesiredJitterBufferFrames();
     stats._ringBufferStarveCount = _starveCount;
     stats._ringBufferConsecutiveNotMixedCount = _consecutiveNotMixedCount;
@@ -795,6 +800,9 @@ AudioStreamStats Audio::getDownstreamAudioStreamStats() const {
 
 void Audio::sendDownstreamAudioStatsPacket() {
 
+    // since this function is called every second, we'll sample the number of audio frames available here.
+    _framesAvailableStats.update(getFramesAvailableInRingAndAudioOutputBuffers());
+
     // push the current seq number stats into history, which moves the history window forward 1s
     // (since that's how often pushStatsToHistory() is called)
     _incomingMixedAudioSequenceNumberStats.pushStatsToHistory();
@@ -1598,3 +1606,9 @@ int Audio::calculateNumberOfFrameSamples(int numBytes) {
     int frameSamples = (int)(numBytes * CALLBACK_ACCELERATOR_RATIO + 0.5f) / sizeof(int16_t);
     return frameSamples;
 }
+
+int Audio::getFramesAvailableInRingAndAudioOutputBuffers() const {
+    int framesInAudioOutputBuffer = (_audioOutput->bufferSize() - _audioOutput->bytesFree()) 
+                                    / (sizeof(int16_t) * _ringBuffer.getNumFrameSamples());
+    return _ringBuffer.framesAvailable() + framesInAudioOutputBuffer;
+}

interface/src/Audio.h

@@ -239,6 +239,8 @@ private:
     void renderGrid(const float* color, int x, int y, int width, int height, int rows, int cols);
     void renderLineStrip(const float* color, int x, int y, int n, int offset, const QByteArray* byteArray);
 
+    int getFramesAvailableInRingAndAudioOutputBuffers() const;
+
     // Audio scope data
     static const unsigned int NETWORK_SAMPLES_PER_FRAME = NETWORK_BUFFER_LENGTH_SAMPLES_PER_CHANNEL;
     static const unsigned int DEFAULT_FRAMES_PER_SCOPE = 5;
@@ -266,6 +268,7 @@ private:
     SequenceNumberStats _incomingMixedAudioSequenceNumberStats;
 
     MovingMinMaxAvg<quint64> _interframeTimeGapStats;
+    MovingMinMaxAvg<int> _framesAvailableStats;
 };
 
 

interface/src/Hair.h

@@ -25,9 +25,9 @@ const int HAIR_CONSTRAINTS = 2;
 
 const int DEFAULT_HAIR_STRANDS = 50;
 const int DEFAULT_HAIR_LINKS = 10;
-const float DEFAULT_HAIR_RADIUS = 0.15;
+const float DEFAULT_HAIR_RADIUS = 0.15f;
-const float DEFAULT_HAIR_LINK_LENGTH = 0.03;
+const float DEFAULT_HAIR_LINK_LENGTH = 0.03f;
-const float DEFAULT_HAIR_THICKNESS = 0.015;
+const float DEFAULT_HAIR_THICKNESS = 0.015f;
 
 class Hair {
 public:

interface/src/ui/ChatWindow.cpp

@@ -30,8 +30,8 @@
 
 const int NUM_MESSAGES_TO_TIME_STAMP = 20;
 
-const float OPACITY_ACTIVE = 1.0;
+const float OPACITY_ACTIVE = 1.0f;
-const float OPACITY_INACTIVE = 0.8;
+const float OPACITY_INACTIVE = 0.8f;
 
 const QRegularExpression regexLinks("((?:(?:ftp)|(?:https?)|(?:hifi))://\\S+)");
 const QRegularExpression regexHifiLinks("([#@]\\S+)");

interface/src/ui/Stats.cpp

@@ -315,10 +315,10 @@ void Stats::display(
             drawText(horizontalOffset, verticalOffset, scale, rotation, font, voxelMaxPing, color);
 
             char audioMixerStatsLabelString[] = "AudioMixer stats:";
-            char streamStatsFormatLabelString[] = "lost%/30s_lost%";
+            char streamStatsFormatLabelString[] = "lost%/lost_30s%";
-            char streamStatsFormatLabelString2[] = "avail/currJ/desiredJ";
+            char streamStatsFormatLabelString2[] = "desired/avail_avg_10s/avail";
             char streamStatsFormatLabelString3[] = "gaps: min/max/avg, starv/ovfl";
-            char streamStatsFormatLabelString4[] = "30s gaps: (same), notmix/sdrop";
+            char streamStatsFormatLabelString4[] = "gaps_30s: (same), notmix/sdrop";
             
             verticalOffset += STATS_PELS_PER_LINE;
             drawText(horizontalOffset, verticalOffset, scale, rotation, font, audioMixerStatsLabelString, color);
@@ -339,9 +339,10 @@ void Stats::display(
 
             AudioStreamStats downstreamAudioStreamStats = audio->getDownstreamAudioStreamStats();
 
-            sprintf(downstreamAudioStatsString, " mix: %.2f%%/%.2f%%, %u/?/%u", downstreamAudioStreamStats._packetStreamStats.getLostRate()*100.0f,
+            sprintf(downstreamAudioStatsString, " mix: %.2f%%/%.2f%%, %u/%u/%u", downstreamAudioStreamStats._packetStreamStats.getLostRate()*100.0f,
                 downstreamAudioStreamStats._packetStreamWindowStats.getLostRate() * 100.0f,
-                downstreamAudioStreamStats._ringBufferFramesAvailable, downstreamAudioStreamStats._ringBufferDesiredJitterBufferFrames);
+                downstreamAudioStreamStats._ringBufferDesiredJitterBufferFrames, downstreamAudioStreamStats._ringBufferFramesAvailableAverage,
+                downstreamAudioStreamStats._ringBufferFramesAvailable);
 
             verticalOffset += STATS_PELS_PER_LINE;
             drawText(horizontalOffset, verticalOffset, scale, rotation, font, downstreamAudioStatsString, color);
@@ -373,8 +374,8 @@ void Stats::display(
 
             sprintf(upstreamAudioStatsString, " mic: %.2f%%/%.2f%%, %u/%u/%u", audioMixerAvatarAudioStreamStats._packetStreamStats.getLostRate()*100.0f,
                 audioMixerAvatarAudioStreamStats._packetStreamWindowStats.getLostRate() * 100.0f,
-                audioMixerAvatarAudioStreamStats._ringBufferFramesAvailable, audioMixerAvatarAudioStreamStats._ringBufferCurrentJitterBufferFrames,
+                audioMixerAvatarAudioStreamStats._ringBufferDesiredJitterBufferFrames, audioMixerAvatarAudioStreamStats._ringBufferFramesAvailableAverage,
-                audioMixerAvatarAudioStreamStats._ringBufferDesiredJitterBufferFrames);
+                audioMixerAvatarAudioStreamStats._ringBufferFramesAvailable);
 
             verticalOffset += STATS_PELS_PER_LINE;
             drawText(horizontalOffset, verticalOffset, scale, rotation, font, upstreamAudioStatsString, color);
@@ -399,8 +400,8 @@ void Stats::display(
 
                 sprintf(upstreamAudioStatsString, " inj: %.2f%%/%.2f%%, %u/%u/%u", injectedStreamAudioStats._packetStreamStats.getLostRate()*100.0f,
                     injectedStreamAudioStats._packetStreamWindowStats.getLostRate() * 100.0f,
-                    injectedStreamAudioStats._ringBufferFramesAvailable, injectedStreamAudioStats._ringBufferCurrentJitterBufferFrames,
+                    injectedStreamAudioStats._ringBufferDesiredJitterBufferFrames, injectedStreamAudioStats._ringBufferFramesAvailableAverage,
-                    injectedStreamAudioStats._ringBufferDesiredJitterBufferFrames);
+                    injectedStreamAudioStats._ringBufferFramesAvailable);
 
                 verticalOffset += STATS_PELS_PER_LINE;
                 drawText(horizontalOffset, verticalOffset, scale, rotation, font, upstreamAudioStatsString, color);

libraries/audio/src/AudioStreamStats.h

@@ -27,7 +27,7 @@ public:
         _timeGapWindowMax(0),
         _timeGapWindowAverage(0.0f),
         _ringBufferFramesAvailable(0),
-        _ringBufferCurrentJitterBufferFrames(0),
+        _ringBufferFramesAvailableAverage(0),
         _ringBufferDesiredJitterBufferFrames(0),
         _ringBufferStarveCount(0),
         _ringBufferConsecutiveNotMixedCount(0),
@@ -48,7 +48,7 @@ public:
     float _timeGapWindowAverage;
 
     quint32 _ringBufferFramesAvailable;
-    quint16 _ringBufferCurrentJitterBufferFrames;
+    quint16 _ringBufferFramesAvailableAverage;
     quint16 _ringBufferDesiredJitterBufferFrames;
     quint32 _ringBufferStarveCount;
     quint32 _ringBufferConsecutiveNotMixedCount;

libraries/audio/src/InjectedAudioRingBuffer.cpp

@@ -30,9 +30,8 @@ InjectedAudioRingBuffer::InjectedAudioRingBuffer(const QUuid& streamIdentifier,
 
 const uchar MAX_INJECTOR_VOLUME = 255;
 
-int InjectedAudioRingBuffer::parseData(const QByteArray& packet) {
+int InjectedAudioRingBuffer::parseData(const QByteArray& packet, int packetsSkipped) {
-    timeGapStatsFrameReceived();
+    frameReceivedUpdateTimingStats();
-    updateDesiredJitterBufferFrames();
 
     // setup a data stream to read from this packet
     QDataStream packetStream(packet);
@@ -59,8 +58,14 @@ int InjectedAudioRingBuffer::parseData(const QByteArray& packet) {
     packetStream >> attenuationByte;
     _attenuationRatio = attenuationByte / (float) MAX_INJECTOR_VOLUME;
     
-    packetStream.skipRawData(writeData(packet.data() + packetStream.device()->pos(),
+    int numAudioBytes = packet.size() - packetStream.device()->pos();
-                                       packet.size() - packetStream.device()->pos()));
+    int numAudioSamples = numAudioBytes / sizeof(int16_t);
+
+    // add silent samples for the dropped packets.
+    // ASSUME that each dropped packet had same number of samples as this one
+    addDroppableSilentSamples(numAudioSamples * packetsSkipped);
+
+    packetStream.skipRawData(writeData(packet.data() + packetStream.device()->pos(), numAudioBytes));
     
     return packetStream.device()->pos();
 }

libraries/audio/src/InjectedAudioRingBuffer.h

@@ -20,7 +20,7 @@ class InjectedAudioRingBuffer : public PositionalAudioRingBuffer {
 public:
     InjectedAudioRingBuffer(const QUuid& streamIdentifier = QUuid(), bool dynamicJitterBuffer = false);
     
-    int parseData(const QByteArray& packet);
+    int parseData(const QByteArray& packet, int packetsSkipped = 0);
     
     const QUuid& getStreamIdentifier() const { return _streamIdentifier; }
     float getRadius() const { return _radius; }

libraries/audio/src/PositionalAudioRingBuffer.cpp

@@ -37,8 +37,8 @@ PositionalAudioRingBuffer::PositionalAudioRingBuffer(PositionalAudioRingBuffer::
     _lastFrameReceivedTime(0),
     _interframeTimeGapStatsForJitterCalc(TIME_GAPS_FOR_JITTER_CALC_INTERVAL_SAMPLES, TIME_GAPS_FOR_JITTER_CALC_WINDOW_INTERVALS),
     _interframeTimeGapStatsForStatsPacket(TIME_GAPS_FOR_STATS_PACKET_INTERVAL_SAMPLES, TIME_GAPS_FOR_STATS_PACKET_WINDOW_INTERVALS),
+    _framesAvailableStats(FRAMES_AVAILABLE_STATS_INTERVAL_SAMPLES, FRAMES_AVAILABLE_STATS_WINDOW_INTERVALS),
     _desiredJitterBufferFrames(1),
-    _currentJitterBufferFrames(-1),
     _dynamicJitterBuffers(dynamicJitterBuffers),
     _consecutiveNotMixedCount(0),
     _starveCount(0),
@@ -46,63 +46,6 @@ PositionalAudioRingBuffer::PositionalAudioRingBuffer(PositionalAudioRingBuffer::
 {
 }
 
-int PositionalAudioRingBuffer::parseData(const QByteArray& packet) {
-    
-    // skip the packet header (includes the source UUID)
-    int readBytes = numBytesForPacketHeader(packet);
-
-    // skip the sequence number
-    readBytes += sizeof(quint16);
-    
-    // hop over the channel flag that has already been read in AudioMixerClientData
-    readBytes += sizeof(quint8);
-    // read the positional data
-    readBytes += parsePositionalData(packet.mid(readBytes));
-   
-    if (packetTypeForPacket(packet) == PacketTypeSilentAudioFrame) {
-        // this source had no audio to send us, but this counts as a packet
-        // write silence equivalent to the number of silent samples they just sent us
-        int16_t numSilentSamples;
-        
-        memcpy(&numSilentSamples, packet.data() + readBytes, sizeof(int16_t));
-        
-        readBytes += sizeof(int16_t);
-        
-        // NOTE: fixes a bug in old clients that would send garbage for their number of silentSamples
-        numSilentSamples = getSamplesPerFrame();
-        
-        if (numSilentSamples > 0) {
-            if (_dynamicJitterBuffers && _currentJitterBufferFrames > _desiredJitterBufferFrames) {
-                // our current jitter buffer size exceeds its desired value, so ignore some silent
-                // frames to get that size as close to desired as possible
-                int samplesPerFrame = getSamplesPerFrame();
-                int numSilentFrames = numSilentSamples / samplesPerFrame;
-                int numFramesToDropDesired = _currentJitterBufferFrames - _desiredJitterBufferFrames;
-
-                if (numSilentFrames > numFramesToDropDesired) {
-                    // we have more than enough frames to drop to get the jitter buffer to its desired length
-                    int numSilentFramesToAdd = numSilentFrames - numFramesToDropDesired;
-                    addSilentFrame(numSilentFramesToAdd * samplesPerFrame);
-                    _currentJitterBufferFrames = _desiredJitterBufferFrames;
-
-                    _silentFramesDropped += numFramesToDropDesired;
-                } else {
-                    // we need to drop all frames to get the jitter buffer close as possible to its desired length
-                    _currentJitterBufferFrames -= numSilentFrames;
-
-                    _silentFramesDropped += numSilentFrames;
-                }
-            } else {
-                addSilentFrame(numSilentSamples);
-            }
-        }
-    } else {
-        // there is audio data to read
-        readBytes += writeData(packet.data() + readBytes, packet.size() - readBytes);
-    }
-    return readBytes;
-}
-
 int PositionalAudioRingBuffer::parsePositionalData(const QByteArray& positionalByteArray) {
     QDataStream packetStream(positionalByteArray);
     
@@ -147,10 +90,9 @@ void PositionalAudioRingBuffer::updateNextOutputTrailingLoudness() {
 }
 
 bool PositionalAudioRingBuffer::shouldBeAddedToMix() {
-    int samplesPerFrame = getSamplesPerFrame();
+    int desiredJitterBufferSamples = _desiredJitterBufferFrames * _numFrameSamples;
-    int desiredJitterBufferSamples = _desiredJitterBufferFrames * samplesPerFrame;
     
-    if (!isNotStarvedOrHasMinimumSamples(samplesPerFrame + desiredJitterBufferSamples)) {
+    if (!isNotStarvedOrHasMinimumSamples(_numFrameSamples + desiredJitterBufferSamples)) {
         // if the buffer was starved, allow it to accrue at least the desired number of
         // jitter buffer frames before we start taking frames from it for mixing
 
@@ -160,13 +102,12 @@ bool PositionalAudioRingBuffer::shouldBeAddedToMix() {
 
         _consecutiveNotMixedCount++;
         return  false;
-    } else if (samplesAvailable() < samplesPerFrame) { 
+    } else if (samplesAvailable() < _numFrameSamples) {
         // if the buffer doesn't have a full frame of samples to take for mixing, it is starved
         _isStarved = true;
         _starveCount++;
-        
+
-        // set to -1 to indicate the jitter buffer is starved
+        _framesAvailableStats.reset();
-        _currentJitterBufferFrames = -1;
         
         // reset our _shouldOutputStarveDebug to true so the next is printed
         _shouldOutputStarveDebug = true;
@@ -176,13 +117,14 @@ bool PositionalAudioRingBuffer::shouldBeAddedToMix() {
     }
     
     // good buffer, add this to the mix
-    if (_isStarved) {
-        // if this buffer has just finished replenishing after being starved, the number of frames in it now
-        // minus one (since a frame will be read immediately after this) is the length of the jitter buffer
-        _currentJitterBufferFrames = samplesAvailable() / samplesPerFrame - 1;
-        _isStarved = false;
-    }
 
+    // if we just finished refilling after a starve, we have a new jitter buffer length.
+    // reset the frames available stats.
+    
+    _isStarved = false;
+
+    _framesAvailableStats.update(framesAvailable());
+    
     // since we've read data from ring buffer at least once - we've started
     _hasStarted = true;
 
@@ -190,17 +132,18 @@ bool PositionalAudioRingBuffer::shouldBeAddedToMix() {
 }
 
 int PositionalAudioRingBuffer::getCalculatedDesiredJitterBufferFrames() const {
-    int calculatedDesiredJitterBufferFrames = 1;
     const float USECS_PER_FRAME = NETWORK_BUFFER_LENGTH_SAMPLES_PER_CHANNEL * USECS_PER_SECOND / (float)SAMPLE_RATE;
      
-    calculatedDesiredJitterBufferFrames = ceilf((float)_interframeTimeGapStatsForJitterCalc.getWindowMax() / USECS_PER_FRAME);
+    int calculatedDesiredJitterBufferFrames = ceilf((float)_interframeTimeGapStatsForJitterCalc.getWindowMax() / USECS_PER_FRAME);
     if (calculatedDesiredJitterBufferFrames < 1) {
         calculatedDesiredJitterBufferFrames = 1;
     }
     return calculatedDesiredJitterBufferFrames;
 }
 
-void PositionalAudioRingBuffer::timeGapStatsFrameReceived() {
+
+void PositionalAudioRingBuffer::frameReceivedUpdateTimingStats() {
+    // update the two time gap stats we're keeping
     quint64 now = usecTimestampNow();
     if (_lastFrameReceivedTime != 0) {
         quint64 gap = now - _lastFrameReceivedTime;
@@ -208,15 +151,14 @@ void PositionalAudioRingBuffer::timeGapStatsFrameReceived() {
         _interframeTimeGapStatsForStatsPacket.update(gap);
     }
     _lastFrameReceivedTime = now;
-}
 
-void PositionalAudioRingBuffer::updateDesiredJitterBufferFrames() {
+    // recalculate the _desiredJitterBufferFrames if _interframeTimeGapStatsForJitterCalc has updated stats for us
     if (_interframeTimeGapStatsForJitterCalc.getNewStatsAvailableFlag()) {
         if (!_dynamicJitterBuffers) {
             _desiredJitterBufferFrames = 1; // HACK to see if this fixes the audio silence
         } else {
             const float USECS_PER_FRAME = NETWORK_BUFFER_LENGTH_SAMPLES_PER_CHANNEL * USECS_PER_SECOND / (float)SAMPLE_RATE;
-            
+
             _desiredJitterBufferFrames = ceilf((float)_interframeTimeGapStatsForJitterCalc.getWindowMax() / USECS_PER_FRAME);
             if (_desiredJitterBufferFrames < 1) {
                 _desiredJitterBufferFrames = 1;
@@ -229,3 +171,38 @@ void PositionalAudioRingBuffer::updateDesiredJitterBufferFrames() {
         _interframeTimeGapStatsForJitterCalc.clearNewStatsAvailableFlag();
     }
 }
+
+void PositionalAudioRingBuffer::addDroppableSilentSamples(int numSilentSamples) {
+
+    // This adds some number of frames to the desired jitter buffer frames target we use.
+    // The larger this value is, the less aggressive we are about reducing the jitter buffer length.
+    // Setting this to 0 will try to get the jitter buffer to be exactly _desiredJitterBufferFrames long,
+    // which could lead immediately to a starve.
+    const int DESIRED_JITTER_BUFFER_FRAMES_PADDING = 1;
+
+    // calculate how many silent frames we should drop.  We only drop silent frames if
+    // the running avg num frames available has stabilized and it's more than
+    // our desired number of frames by the margin defined above.
+    int numSilentFramesToDrop = 0;
+    if (_framesAvailableStats.getNewStatsAvailableFlag() && _framesAvailableStats.isWindowFilled()
+        && numSilentSamples >= _numFrameSamples) {
+        _framesAvailableStats.clearNewStatsAvailableFlag();
+        int averageJitterBufferFrames = (int)_framesAvailableStats.getWindowAverage();
+        int desiredJitterBufferFramesPlusPadding = _desiredJitterBufferFrames + DESIRED_JITTER_BUFFER_FRAMES_PADDING;
+
+        if (averageJitterBufferFrames > desiredJitterBufferFramesPlusPadding) {
+            // our avg jitter buffer size exceeds its desired value, so ignore some silent
+            // frames to get that size as close to desired as possible
+            int numSilentFramesToDropDesired = averageJitterBufferFrames - desiredJitterBufferFramesPlusPadding;
+            int numSilentFramesReceived = numSilentSamples / _numFrameSamples;
+            numSilentFramesToDrop = std::min(numSilentFramesToDropDesired, numSilentFramesReceived);
+
+            // since we now have a new jitter buffer length, reset the frames available stats.
+            _framesAvailableStats.reset();
+
+            _silentFramesDropped += numSilentFramesToDrop;
+        }   
+    }
+
+    addSilentFrame(numSilentSamples - numSilentFramesToDrop * _numFrameSamples);
+}

libraries/audio/src/PositionalAudioRingBuffer.h

@@ -29,6 +29,11 @@ const int TIME_GAPS_FOR_JITTER_CALC_WINDOW_INTERVALS = 10;
 const int TIME_GAPS_FOR_STATS_PACKET_INTERVAL_SAMPLES = USECS_PER_SECOND / BUFFER_SEND_INTERVAL_USECS;
 const int TIME_GAPS_FOR_STATS_PACKET_WINDOW_INTERVALS = 30;
 
+// the stats for calculating the average frames available  will recalculate every ~1 second
+// and will include data for the past ~2 seconds 
+const int FRAMES_AVAILABLE_STATS_INTERVAL_SAMPLES = USECS_PER_SECOND / BUFFER_SEND_INTERVAL_USECS;
+const int FRAMES_AVAILABLE_STATS_WINDOW_INTERVALS = 2;
+
 const int AUDIOMIXER_INBOUND_RING_BUFFER_FRAME_CAPACITY = 100;
 
 class PositionalAudioRingBuffer : public AudioRingBuffer {
@@ -40,7 +45,8 @@ public:
     
     PositionalAudioRingBuffer(PositionalAudioRingBuffer::Type type, bool isStereo = false, bool dynamicJitterBuffers = false);
     
-    int parseData(const QByteArray& packet);
+    virtual int parseData(const QByteArray& packet, int packetsSkipped = 0) = 0;
+
     int parsePositionalData(const QByteArray& positionalByteArray);
     int parseListenModeData(const QByteArray& listenModeByteArray);
     
@@ -69,7 +75,7 @@ public:
 
     int getCalculatedDesiredJitterBufferFrames() const; /// returns what we would calculate our desired as if asked
     int getDesiredJitterBufferFrames() const { return _desiredJitterBufferFrames; }
-    int getCurrentJitterBufferFrames() const { return _currentJitterBufferFrames; }
+    double getFramesAvailableAverage() const { return _framesAvailableStats.getWindowAverage(); }
 
     int getConsecutiveNotMixedCount() const { return _consecutiveNotMixedCount; }
     int getStarveCount() const { return _starveCount; }
@@ -80,8 +86,8 @@ protected:
     PositionalAudioRingBuffer(const PositionalAudioRingBuffer&);
     PositionalAudioRingBuffer& operator= (const PositionalAudioRingBuffer&);
 
-    void timeGapStatsFrameReceived();
+    void frameReceivedUpdateTimingStats();
-    void updateDesiredJitterBufferFrames();
+    void addDroppableSilentSamples(int numSilentSamples);
     
     PositionalAudioRingBuffer::Type _type;
     glm::vec3 _position;
@@ -97,9 +103,9 @@ protected:
     quint64 _lastFrameReceivedTime;
     MovingMinMaxAvg<quint64> _interframeTimeGapStatsForJitterCalc;
     MovingMinMaxAvg<quint64> _interframeTimeGapStatsForStatsPacket;
+    MovingMinMaxAvg<int> _framesAvailableStats;
     
     int _desiredJitterBufferFrames;
-    int _currentJitterBufferFrames;
     bool _dynamicJitterBuffers;
 
     // extra stats

libraries/networking/src/SequenceNumberStats.cpp

@@ -30,7 +30,9 @@ void SequenceNumberStats::reset() {
 
 static const int UINT16_RANGE = std::numeric_limits<uint16_t>::max() + 1;
 
-void SequenceNumberStats::sequenceNumberReceived(quint16 incoming, QUuid senderUUID, const bool wantExtraDebugging) {
+SequenceNumberStats::ArrivalInfo SequenceNumberStats::sequenceNumberReceived(quint16 incoming, QUuid senderUUID, const bool wantExtraDebugging) {
+
+    SequenceNumberStats::ArrivalInfo arrivalInfo;
 
     // if the sender node has changed, reset all stats
     if (senderUUID != _lastSenderUUID) {
@@ -46,6 +48,7 @@ void SequenceNumberStats::sequenceNumberReceived(quint16 incoming, QUuid senderU
     _stats._numReceived++;
 
     if (incoming == expected) { // on time
+        arrivalInfo._status = OnTime;
         _lastReceived = incoming;
     } else { // out of order
 
@@ -67,21 +70,29 @@ void SequenceNumberStats::sequenceNumberReceived(quint16 incoming, QUuid senderU
                 expectedInt -= UINT16_RANGE;
             }
         } else if (absGap > MAX_REASONABLE_SEQUENCE_GAP) {
+            arrivalInfo._status = Unreasonable;
+
             // ignore packet if gap is unreasonable
             qDebug() << "ignoring unreasonable sequence number:" << incoming
                 << "previous:" << _lastReceived;
             _stats._numUnreasonable++;
-            return;
+
+            return arrivalInfo;
         }
 
-        // now that rollover has been corrected for (if it occurred), incoming and expected can be
+        // now that rollover has been corrected for (if it occurred), incomingInt and expectedInt can be
         // compared to each other directly, though one of them might be negative
+
+        arrivalInfo._seqDiffFromExpected = incomingInt - expectedInt;
+
         if (incomingInt > expectedInt) { // early
+            arrivalInfo._status = Early;
+
             if (wantExtraDebugging) {
                 qDebug() << "this packet is earlier than expected...";
                 qDebug() << ">>>>>>>> missing gap=" << (incomingInt - expectedInt);
             }
-
+            
             _stats._numEarly++;
             _stats._numLost += (incomingInt - expectedInt);
             _lastReceived = incoming;
@@ -100,18 +111,23 @@ void SequenceNumberStats::sequenceNumberReceived(quint16 incoming, QUuid senderU
             if (wantExtraDebugging) {
                 qDebug() << "this packet is later than expected...";
             }
+
             _stats._numLate++;
 
             // do not update _lastReceived; it shouldn't become smaller
 
             // remove this from missing sequence number if it's in there
             if (_missingSet.remove(incoming)) {
+                arrivalInfo._status = Late;
+
                 if (wantExtraDebugging) {
                     qDebug() << "found it in _missingSet";
                 }
                 _stats._numLost--;
                 _stats._numRecovered++;
             } else {
+                arrivalInfo._status = Duplicate;
+
                 if (wantExtraDebugging) {
                     qDebug() << "sequence:" << incoming << "was NOT found in _missingSet and is probably a duplicate";
                 }
@@ -119,6 +135,7 @@ void SequenceNumberStats::sequenceNumberReceived(quint16 incoming, QUuid senderU
             }
         }
     }
+    return arrivalInfo;
 }
 
 void SequenceNumberStats::pruneMissingSet(const bool wantExtraDebugging) {

libraries/networking/src/SequenceNumberStats.h

@@ -48,10 +48,25 @@ public:
 
 class SequenceNumberStats {
 public:
+    enum ArrivalStatus {
+        OnTime,
+        Unreasonable,
+        Early,
+        Late,   // recovered
+        Duplicate
+    };
+
+    class ArrivalInfo {
+    public:
+        ArrivalStatus _status;
+        int _seqDiffFromExpected;
+    };
+
+
     SequenceNumberStats(int statsHistoryLength = 0);
 
     void reset();
-    void sequenceNumberReceived(quint16 incoming, QUuid senderUUID = QUuid(), const bool wantExtraDebugging = false);
+    ArrivalInfo sequenceNumberReceived(quint16 incoming, QUuid senderUUID = QUuid(), const bool wantExtraDebugging = false);
     void pruneMissingSet(const bool wantExtraDebugging = false);
     void pushStatsToHistory() { _statsHistory.insert(_stats); }
 

libraries/shared/src/MovingMinMaxAvg.h

@@ -126,6 +126,8 @@ public:
     T getWindowMax() const { return _windowStats._max; }
     double getWindowAverage() const { return _windowStats._average; }
 
+    bool isWindowFilled() const { return _intervalStats.isFilled(); }
+
 private:
     int _intervalLength;
     int _windowIntervals;

libraries/shared/src/RingBufferHistory.h

@@ -73,6 +73,7 @@ public:
 
     int getCapacity() const { return _capacity; }
     int getNumEntries() const { return _numEntries; }
+    bool isFilled() const { return _numEntries == _capacity; }
 
 private:
     int _size;