Merge pull request #9615 from kencooke/audio-packetloss-fix

Reduce audio clicks/pops caused by packet loss

libraries/audio/src/InboundAudioStream.cpp

@@ -131,12 +131,16 @@ int InboundAudioStream::parseData(ReceivedMessage& message) {
 
     // handle this packet based on its arrival status.
     switch (arrivalInfo._status) {
+        case SequenceNumberStats::Unreasonable: {
+            lostAudioData(1);
+            break;
+        }
         case SequenceNumberStats::Early: {
             // Packet is early; write droppable silent samples for each of the skipped packets.
             // NOTE: we assume that each dropped packet contains the same number of samples
             // as the packet we just received.
             int packetsDropped = arrivalInfo._seqDiffFromExpected;
-            writeFramesForDroppedPackets(packetsDropped * networkFrames);
+            lostAudioData(packetsDropped);
 
             // fall through to OnTime case
         }
@@ -208,6 +212,21 @@ int InboundAudioStream::parseStreamProperties(PacketType type, const QByteArray&
     }
 }
 
+int InboundAudioStream::lostAudioData(int numPackets) {
+    QByteArray decodedBuffer;
+
+    while (numPackets--) {
+        if (_decoder) {
+            _decoder->lostFrame(decodedBuffer);
+        } else {
+            decodedBuffer.resize(AudioConstants::NETWORK_FRAME_BYTES_STEREO);
+            memset(decodedBuffer.data(), 0, decodedBuffer.size());
+        }
+        _ringBuffer.writeData(decodedBuffer.data(), decodedBuffer.size());
+    }
+    return 0;
+}
+
 int InboundAudioStream::parseAudioData(PacketType type, const QByteArray& packetAfterStreamProperties) {
     QByteArray decodedBuffer;
     if (_decoder) {
@@ -220,9 +239,6 @@ int InboundAudioStream::parseAudioData(PacketType type, const QByteArray& packet
 }
 
 int InboundAudioStream::writeDroppableSilentFrames(int silentFrames) {
-    if (_decoder) {
-        _decoder->trackLostFrames(silentFrames);
-    }
 
     // calculate how many silent frames we should drop.
     int silentSamples = silentFrames * _numChannels;
@@ -416,29 +432,6 @@ void InboundAudioStream::packetReceivedUpdateTimingStats() {
     _lastPacketReceivedTime = now;
 }
 
-int InboundAudioStream::writeFramesForDroppedPackets(int networkFrames) {
-    return writeLastFrameRepeatedWithFade(networkFrames);
-}
-
-int InboundAudioStream::writeLastFrameRepeatedWithFade(int frames) {
-    AudioRingBuffer::ConstIterator frameToRepeat = _ringBuffer.lastFrameWritten();
-    int frameSize = _ringBuffer.getNumFrameSamples();
-    int samplesToWrite = frames * _numChannels;
-    int indexOfRepeat = 0;
-    do {
-        int samplesToWriteThisIteration = std::min(samplesToWrite, frameSize);
-        float fade = calculateRepeatedFrameFadeFactor(indexOfRepeat);
-        if (fade == 1.0f) {
-            samplesToWrite -= _ringBuffer.writeSamples(frameToRepeat, samplesToWriteThisIteration);
-        } else {
-            samplesToWrite -= _ringBuffer.writeSamplesWithFade(frameToRepeat, samplesToWriteThisIteration, fade);
-        }
-        indexOfRepeat++;
-    } while (samplesToWrite > 0);
-
-    return frames;
-}
-
 AudioStreamStats InboundAudioStream::getAudioStreamStats() const {
     AudioStreamStats streamStats;
 

libraries/audio/src/InboundAudioStream.h

@@ -115,8 +115,6 @@ public slots:
 private:
     void packetReceivedUpdateTimingStats();
 
-    int writeFramesForDroppedPackets(int networkFrames);
-
     void popSamplesNoCheck(int samples);
     void framesAvailableChanged();
 
@@ -134,12 +132,11 @@ protected:
     /// default implementation assumes packet contains raw audio samples after stream properties
     virtual int parseAudioData(PacketType type, const QByteArray& packetAfterStreamProperties);
 
+    /// produces audio data for lost network packets.
+    virtual int lostAudioData(int numPackets);
+
     /// writes silent frames to the buffer that may be dropped to reduce latency caused by the buffer
     virtual int writeDroppableSilentFrames(int silentFrames);
-
-    /// writes the last written frame repeatedly, gradually fading to silence.
-    /// used for writing samples for dropped packets.
-    virtual int writeLastFrameRepeatedWithFade(int frames);
     
 protected:
 

libraries/audio/src/MixedProcessedAudioStream.cpp

@@ -31,11 +31,26 @@ int MixedProcessedAudioStream::writeDroppableSilentFrames(int silentFrames) {
     return deviceSilentFramesWritten;
 }
 
-int MixedProcessedAudioStream::writeLastFrameRepeatedWithFade(int frames) {
+int MixedProcessedAudioStream::lostAudioData(int numPackets) {
-    int deviceFrames = networkToDeviceFrames(frames);
+    QByteArray decodedBuffer;
-    int deviceFramesWritten = InboundAudioStream::writeLastFrameRepeatedWithFade(deviceFrames);
+    QByteArray outputBuffer;
-    emit addedLastFrameRepeatedWithFade(deviceToNetworkFrames(deviceFramesWritten));
+
-    return deviceFramesWritten;
+    while (numPackets--) {
+        if (_decoder) {
+            _decoder->lostFrame(decodedBuffer);
+        } else {
+            decodedBuffer.resize(AudioConstants::NETWORK_FRAME_BYTES_STEREO);
+            memset(decodedBuffer.data(), 0, decodedBuffer.size());
+        }
+
+        emit addedStereoSamples(decodedBuffer);
+
+        emit processSamples(decodedBuffer, outputBuffer);
+
+        _ringBuffer.writeData(outputBuffer.data(), outputBuffer.size());
+        qCDebug(audiostream, "Wrote %d samples to buffer (%d available)", outputBuffer.size() / (int)sizeof(int16_t), getSamplesAvailable());
+    }
+    return 0;
 }
 
 int MixedProcessedAudioStream::parseAudioData(PacketType type, const QByteArray& packetAfterStreamProperties) {

libraries/audio/src/MixedProcessedAudioStream.h

@@ -34,8 +34,8 @@ public:
 
 protected:
     int writeDroppableSilentFrames(int silentFrames) override;
-    int writeLastFrameRepeatedWithFade(int frames) override;
     int parseAudioData(PacketType type, const QByteArray& packetAfterStreamProperties) override;
+    int lostAudioData(int numPackets) override;
 
 private:
     int networkToDeviceFrames(int networkFrames);

libraries/plugins/src/plugins/CodecPlugin.h

@@ -23,8 +23,7 @@ public:
     virtual ~Decoder() { }
     virtual void decode(const QByteArray& encodedBuffer, QByteArray& decodedBuffer) = 0;
 
-    // numFrames - number of samples (mono) or sample-pairs (stereo)
+    virtual void lostFrame(QByteArray& decodedBuffer) = 0;
-    virtual void trackLostFrames(int numFrames) = 0;
 };
 
 class CodecPlugin : public Plugin {

plugins/hifiCodec/src/HiFiCodec.cpp

@@ -65,12 +65,10 @@ public:
         AudioDecoder::process((const int16_t*)encodedBuffer.constData(), (int16_t*)decodedBuffer.data(), AudioConstants::NETWORK_FRAME_SAMPLES_PER_CHANNEL, true);
     }
 
-    virtual void trackLostFrames(int numFrames)  override { 
+    virtual void lostFrame(QByteArray& decodedBuffer) override {
-        QByteArray encodedBuffer;
-        QByteArray decodedBuffer;
         decodedBuffer.resize(_decodedSize);
-        // NOTE: we don't actually use the results of this decode, we just do it to keep the state of the codec clean
+        // this performs packet loss interpolation
-        AudioDecoder::process((const int16_t*)encodedBuffer.constData(), (int16_t*)decodedBuffer.data(), AudioConstants::NETWORK_FRAME_SAMPLES_PER_CHANNEL, false);
+        AudioDecoder::process(nullptr, (int16_t*)decodedBuffer.data(), AudioConstants::NETWORK_FRAME_SAMPLES_PER_CHANNEL, false);
     }
 private:
     int _decodedSize;

plugins/pcmCodec/src/PCMCodecManager.h

@@ -38,11 +38,14 @@ public:
     virtual void encode(const QByteArray& decodedBuffer, QByteArray& encodedBuffer) override {
         encodedBuffer = decodedBuffer;
     }
+
     virtual void decode(const QByteArray& encodedBuffer, QByteArray& decodedBuffer) override {
         decodedBuffer = encodedBuffer;
     }
 
-    virtual void trackLostFrames(int numFrames)  override { }
+    virtual void lostFrame(QByteArray& decodedBuffer) override {
+        memset(decodedBuffer.data(), 0, decodedBuffer.size());
+    }
 
 private:
     static const char* NAME;
@@ -77,7 +80,9 @@ public:
         decodedBuffer = qUncompress(encodedBuffer);
     }
 
-    virtual void trackLostFrames(int numFrames)  override { }
+    virtual void lostFrame(QByteArray& decodedBuffer) override {
+        memset(decodedBuffer.data(), 0, decodedBuffer.size());
+    }
 
 private:
     static const char* NAME;