Use interpolated audio (codec packet-loss concealment) or zero samples (if no codec) when audio packets are lost. This audio is still processed by the audio pipeline to avoid clicks/pops.

libraries/audio/src/InboundAudioStream.cpp

@@ -136,7 +136,8 @@ int InboundAudioStream::parseData(ReceivedMessage& message) {
             // NOTE: we assume that each dropped packet contains the same number of samples
             // as the packet we just received.
             int packetsDropped = arrivalInfo._seqDiffFromExpected;
-            writeFramesForDroppedPackets(packetsDropped * networkFrames);
+            //writeFramesForDroppedPackets(packetsDropped * networkFrames);
+            lostAudioData(packetsDropped);
 
             // fall through to OnTime case
         }
@@ -208,6 +209,21 @@ int InboundAudioStream::parseStreamProperties(PacketType type, const QByteArray&
     }
 }
 
+int InboundAudioStream::lostAudioData(int numPackets) {
+    QByteArray decodedBuffer;
+
+    while (numPackets--) {
+        if (_decoder) {
+            _decoder->lostFrame(decodedBuffer);
+        } else {
+            decodedBuffer.resize(AudioConstants::NETWORK_FRAME_BYTES_STEREO);
+            memset(decodedBuffer.data(), 0, decodedBuffer.size());
+        }
+        _ringBuffer.writeData(decodedBuffer.data(), decodedBuffer.size());
+    }
+    return 0;
+}
+
 int InboundAudioStream::parseAudioData(PacketType type, const QByteArray& packetAfterStreamProperties) {
     QByteArray decodedBuffer;
     if (_decoder) {
@@ -220,9 +236,6 @@ int InboundAudioStream::parseAudioData(PacketType type, const QByteArray& packet
 }
 
 int InboundAudioStream::writeDroppableSilentFrames(int silentFrames) {
-    if (_decoder) {
-        _decoder->trackLostFrames(silentFrames);
-    }
 
     // calculate how many silent frames we should drop.
     int silentSamples = silentFrames * _numChannels;

libraries/audio/src/InboundAudioStream.h

@@ -134,6 +134,9 @@ protected:
     /// default implementation assumes packet contains raw audio samples after stream properties
     virtual int parseAudioData(PacketType type, const QByteArray& packetAfterStreamProperties);
 
+    /// produces audio data for lost network packets.
+    virtual int lostAudioData(int numPackets);
+
     /// writes silent frames to the buffer that may be dropped to reduce latency caused by the buffer
     virtual int writeDroppableSilentFrames(int silentFrames);
 

libraries/audio/src/MixedProcessedAudioStream.cpp

@@ -38,6 +38,28 @@ int MixedProcessedAudioStream::writeLastFrameRepeatedWithFade(int frames) {
     return deviceFramesWritten;
 }
 
+int MixedProcessedAudioStream::lostAudioData(int numPackets) {
+    QByteArray decodedBuffer;
+    QByteArray outputBuffer;
+
+    while (numPackets--) {
+        if (_decoder) {
+            _decoder->lostFrame(decodedBuffer);
+        } else {
+            decodedBuffer.resize(AudioConstants::NETWORK_FRAME_BYTES_STEREO);
+            memset(decodedBuffer.data(), 0, decodedBuffer.size());
+        }
+
+        emit addedStereoSamples(decodedBuffer);
+
+        emit processSamples(decodedBuffer, outputBuffer);
+
+        _ringBuffer.writeData(outputBuffer.data(), outputBuffer.size());
+        qCDebug(audiostream, "Wrote %d samples to buffer (%d available)", outputBuffer.size() / (int)sizeof(int16_t), getSamplesAvailable());
+    }
+    return 0;
+}
+
 int MixedProcessedAudioStream::parseAudioData(PacketType type, const QByteArray& packetAfterStreamProperties) {
     QByteArray decodedBuffer;
     if (_decoder) {

libraries/audio/src/MixedProcessedAudioStream.h

@@ -36,6 +36,7 @@ protected:
     int writeDroppableSilentFrames(int silentFrames) override;
     int writeLastFrameRepeatedWithFade(int frames) override;
     int parseAudioData(PacketType type, const QByteArray& packetAfterStreamProperties) override;
+    int lostAudioData(int numPackets) override;
 
 private:
     int networkToDeviceFrames(int networkFrames);

libraries/plugins/src/plugins/CodecPlugin.h

@@ -23,8 +23,7 @@ public:
     virtual ~Decoder() { }
     virtual void decode(const QByteArray& encodedBuffer, QByteArray& decodedBuffer) = 0;
 
-    // numFrames - number of samples (mono) or sample-pairs (stereo)
+    virtual void lostFrame(QByteArray& decodedBuffer) = 0;
-    virtual void trackLostFrames(int numFrames) = 0;
 };
 
 class CodecPlugin : public Plugin {

plugins/hifiCodec/src/HiFiCodec.cpp

@@ -65,12 +65,10 @@ public:
         AudioDecoder::process((const int16_t*)encodedBuffer.constData(), (int16_t*)decodedBuffer.data(), AudioConstants::NETWORK_FRAME_SAMPLES_PER_CHANNEL, true);
     }
 
-    virtual void trackLostFrames(int numFrames)  override { 
+    virtual void lostFrame(QByteArray& decodedBuffer) override {
-        QByteArray encodedBuffer;
-        QByteArray decodedBuffer;
         decodedBuffer.resize(_decodedSize);
-        // NOTE: we don't actually use the results of this decode, we just do it to keep the state of the codec clean
+        // this performs packet loss interpolation
-        AudioDecoder::process((const int16_t*)encodedBuffer.constData(), (int16_t*)decodedBuffer.data(), AudioConstants::NETWORK_FRAME_SAMPLES_PER_CHANNEL, false);
+        AudioDecoder::process(nullptr, (int16_t*)decodedBuffer.data(), AudioConstants::NETWORK_FRAME_SAMPLES_PER_CHANNEL, false);
     }
 private:
     int _decodedSize;

plugins/pcmCodec/src/PCMCodecManager.h

@@ -38,11 +38,14 @@ public:
     virtual void encode(const QByteArray& decodedBuffer, QByteArray& encodedBuffer) override {
         encodedBuffer = decodedBuffer;
     }
+
     virtual void decode(const QByteArray& encodedBuffer, QByteArray& decodedBuffer) override {
         decodedBuffer = encodedBuffer;
     }
 
-    virtual void trackLostFrames(int numFrames)  override { }
+    virtual void lostFrame(QByteArray& decodedBuffer) override {
+        memset(decodedBuffer.data(), 0, decodedBuffer.size());
+    }
 
 private:
     static const char* NAME;
@@ -77,7 +80,9 @@ public:
         decodedBuffer = qUncompress(encodedBuffer);
     }
 
-    virtual void trackLostFrames(int numFrames)  override { }
+    virtual void lostFrame(QByteArray& decodedBuffer) override {
+        memset(decodedBuffer.data(), 0, decodedBuffer.size());
+    }
 
 private:
     static const char* NAME;