Merge pull request #15530 from roxanneskelly/bugz85b

BUGZ-85 - extrapolate audio on network lag spike to reduce pop

assignment-client/src/audio/AvatarAudioStream.cpp

@@ -45,6 +45,7 @@ int AvatarAudioStream::parseStreamProperties(PacketType type, const QByteArray&
                                            : AudioConstants::NETWORK_FRAME_SAMPLES_PER_CHANNEL);
             // restart the codec
             if (_codec) {
+                QMutexLocker lock(&_decoderMutex);
                 if (_decoder) {
                     _codec->releaseDecoder(_decoder);
                 }

libraries/audio/src/InboundAudioStream.cpp

@@ -10,6 +10,7 @@
 //
 
 #include "InboundAudioStream.h"
+#include "TryLocker.h"
 
 #include <glm/glm.hpp>
 
@@ -215,7 +216,7 @@ int InboundAudioStream::parseData(ReceivedMessage& message) {
     if (framesAvailable > _desiredJitterBufferFrames + MAX_FRAMES_OVER_DESIRED) {
         int framesToDrop = framesAvailable - (_desiredJitterBufferFrames + DESIRED_JITTER_BUFFER_FRAMES_PADDING);
         _ringBuffer.shiftReadPosition(framesToDrop * _ringBuffer.getNumFrameSamples());
-        
+
         _framesAvailableStat.reset();
         _currentJitterBufferFrames = 0;
 
@@ -247,10 +248,18 @@ int InboundAudioStream::lostAudioData(int numPackets) {
     QByteArray decodedBuffer;
 
     while (numPackets--) {
+        MutexTryLocker lock(_decoderMutex);
+        if (!lock.isLocked()) {
+            // an incoming packet is being processed,
+            // and will likely be on the ring buffer shortly,
+            // so don't bother generating more data
+            qCInfo(audiostream, "Packet currently being unpacked or lost frame already being generated.  Not generating lost frame.");
+            return 0;
+        }
         if (_decoder) {
             _decoder->lostFrame(decodedBuffer);
         } else {
-            decodedBuffer.resize(AudioConstants::NETWORK_FRAME_BYTES_STEREO);
+            decodedBuffer.resize(AudioConstants::NETWORK_FRAME_BYTES_PER_CHANNEL * _numChannels);
             memset(decodedBuffer.data(), 0, decodedBuffer.size());
         }
         _ringBuffer.writeData(decodedBuffer.data(), decodedBuffer.size());
@@ -260,6 +269,12 @@ int InboundAudioStream::lostAudioData(int numPackets) {
 
 int InboundAudioStream::parseAudioData(PacketType type, const QByteArray& packetAfterStreamProperties) {
     QByteArray decodedBuffer;
+
+    // may block on the real-time thread, which is acceptible as 
+    // parseAudioData is only called by the packet processing
+    // thread which, while high performance, is not as sensitive to
+    // delays as the real-time thread.
+    QMutexLocker lock(&_decoderMutex);
     if (_decoder) {
         _decoder->decode(packetAfterStreamProperties, decodedBuffer);
     } else {
@@ -278,16 +293,23 @@ int InboundAudioStream::writeDroppableSilentFrames(int silentFrames) {
     // case we will call the decoder's lostFrame() method, which indicates
     // that it should interpolate from its last known state down toward 
     // silence.
-    if (_decoder) {
+    {
-        // FIXME - We could potentially use the output from the codec, in which 
+        // may block on the real-time thread, which is acceptible as 
-        // case we might get a cleaner fade toward silence. NOTE: The below logic 
+        // writeDroppableSilentFrames is only called by the packet processing
-        // attempts to catch up in the event that the jitter buffers have grown. 
+        // thread which, while high performance, is not as sensitive to
-        // The better long term fix is to use the output from the decode, detect
+        // delays as the real-time thread.
-        // when it actually reaches silence, and then delete the silent portions
+        QMutexLocker lock(&_decoderMutex);
-        // of the jitter buffers. Or petentially do a cross fade from the decode
+        if (_decoder) {
-        // output to silence.
+            // FIXME - We could potentially use the output from the codec, in which 
-        QByteArray decodedBuffer;
+            // case we might get a cleaner fade toward silence. NOTE: The below logic 
-        _decoder->lostFrame(decodedBuffer);
+            // attempts to catch up in the event that the jitter buffers have grown. 
+            // The better long term fix is to use the output from the decode, detect
+            // when it actually reaches silence, and then delete the silent portions
+            // of the jitter buffers. Or petentially do a cross fade from the decode
+            // output to silence.
+            QByteArray decodedBuffer;
+            _decoder->lostFrame(decodedBuffer);
+        }
     }
 
     // calculate how many silent frames we should drop.
@@ -338,10 +360,23 @@ int InboundAudioStream::popSamples(int maxSamples, bool allOrNothing) {
             popSamplesNoCheck(samplesAvailable);
             samplesPopped = samplesAvailable;
         } else {
-            // we can't pop any samples, set this stream to starved
+            // we can't pop any samples, set this stream to starved for jitter 
+            // buffer calculations.
             setToStarved();
             _consecutiveNotMixedCount++;
-            _lastPopSucceeded = false;
+            //Kick PLC to generate a filler frame, reducing 'click'
+            lostAudioData(allOrNothing ? (maxSamples - samplesAvailable) / _ringBuffer.getNumFrameSamples() : 1);
+            samplesPopped = _ringBuffer.samplesAvailable();
+            if (samplesPopped) {
+                popSamplesNoCheck(samplesPopped);
+            } else {
+                // No samples available means a packet is currently being
+                // processed, so we don't generate lost audio data, and instead
+                // just wait for the packet to come in.  This prevents locking
+                // the real-time audio thread at the cost of a potential (but rare)
+                // 'click'
+                _lastPopSucceeded = false;
+            }
         }
     }
     return samplesPopped;
@@ -528,6 +563,7 @@ void InboundAudioStream::setupCodec(CodecPluginPointer codec, const QString& cod
     _codec = codec;
     _selectedCodecName = codecName;
     if (_codec) {
+        QMutexLocker lock(&_decoderMutex);
         _decoder = codec->createDecoder(AudioConstants::SAMPLE_RATE, numChannels);
     }
 }
@@ -535,6 +571,7 @@ void InboundAudioStream::setupCodec(CodecPluginPointer codec, const QString& cod
 void InboundAudioStream::cleanupCodec() {
     // release any old codec encoder/decoder first...
     if (_codec) {
+        QMutexLocker lock(&_decoderMutex);
         if (_decoder) {
             _codec->releaseDecoder(_decoder);
             _decoder = nullptr;

libraries/audio/src/InboundAudioStream.h

@@ -187,6 +187,7 @@ protected:
 
     CodecPluginPointer _codec;
     QString _selectedCodecName;
+    QMutex _decoderMutex;
     Decoder* _decoder { nullptr };
     int _mismatchedAudioCodecCount { 0 };
 };

libraries/audio/src/MixedProcessedAudioStream.cpp

@@ -11,6 +11,7 @@
 
 #include "MixedProcessedAudioStream.h"
 #include "AudioLogging.h"
+#include "TryLocker.h"
 
 MixedProcessedAudioStream::MixedProcessedAudioStream(int numFramesCapacity, int numStaticJitterFrames)
     : InboundAudioStream(AudioConstants::STEREO, AudioConstants::NETWORK_FRAME_SAMPLES_PER_CHANNEL,
@@ -36,13 +37,20 @@ int MixedProcessedAudioStream::lostAudioData(int numPackets) {
     QByteArray outputBuffer;
 
     while (numPackets--) {
+        MutexTryLocker lock(_decoderMutex);
+        if (!lock.isLocked()) {
+            // an incoming packet is being processed,
+            // and will likely be on the ring buffer shortly,
+            // so don't bother generating more data
+            qCInfo(audiostream, "Packet currently being unpacked or lost frame already being generated.  Not generating lost frame.");
+            return 0;
+        }
         if (_decoder) {
             _decoder->lostFrame(decodedBuffer);
         } else {
             decodedBuffer.resize(AudioConstants::NETWORK_FRAME_BYTES_STEREO);
             memset(decodedBuffer.data(), 0, decodedBuffer.size());
         }
-
         emit addedStereoSamples(decodedBuffer);
 
         emit processSamples(decodedBuffer, outputBuffer);
@@ -55,6 +63,12 @@ int MixedProcessedAudioStream::lostAudioData(int numPackets) {
 
 int MixedProcessedAudioStream::parseAudioData(PacketType type, const QByteArray& packetAfterStreamProperties) {
     QByteArray decodedBuffer;
+
+    // may block on the real-time thread, which is acceptible as 
+    // parseAudioData is only called by the packet processing
+    // thread which, while high performance, is not as sensitive to
+    // delays as the real-time thread.
+    QMutexLocker lock(&_decoderMutex);
     if (_decoder) {
         _decoder->decode(packetAfterStreamProperties, decodedBuffer);
     } else {