From 4acb99cd4fdb4310dd539bfaf7cc6aa89017807b Mon Sep 17 00:00:00 2001 From: Ken Cooke Date: Mon, 6 Feb 2017 09:29:39 -0800 Subject: [PATCH] Use interpolated audio (codec packet-loss concealment) or zero samples (if no codec) when audio packets are lost. This audio is still processed by the audio pipeline to avoid clicks/pops. --- libraries/audio/src/InboundAudioStream.cpp | 21 ++++++++++++++---- libraries/audio/src/InboundAudioStream.h | 3 +++ .../audio/src/MixedProcessedAudioStream.cpp | 22 +++++++++++++++++++ .../audio/src/MixedProcessedAudioStream.h | 1 + libraries/plugins/src/plugins/CodecPlugin.h | 3 +-- plugins/hifiCodec/src/HiFiCodec.cpp | 8 +++---- plugins/pcmCodec/src/PCMCodecManager.h | 9 ++++++-- 7 files changed, 54 insertions(+), 13 deletions(-) diff --git a/libraries/audio/src/InboundAudioStream.cpp b/libraries/audio/src/InboundAudioStream.cpp index 36a6079546..6f61c59cbd 100644 --- a/libraries/audio/src/InboundAudioStream.cpp +++ b/libraries/audio/src/InboundAudioStream.cpp @@ -136,7 +136,8 @@ int InboundAudioStream::parseData(ReceivedMessage& message) { // NOTE: we assume that each dropped packet contains the same number of samples // as the packet we just received. int packetsDropped = arrivalInfo._seqDiffFromExpected; - writeFramesForDroppedPackets(packetsDropped * networkFrames); + //writeFramesForDroppedPackets(packetsDropped * networkFrames); + lostAudioData(packetsDropped); // fall through to OnTime case } @@ -208,6 +209,21 @@ int InboundAudioStream::parseStreamProperties(PacketType type, const QByteArray& } } +int InboundAudioStream::lostAudioData(int numPackets) { + QByteArray decodedBuffer; + + while (numPackets--) { + if (_decoder) { + _decoder->lostFrame(decodedBuffer); + } else { + decodedBuffer.resize(AudioConstants::NETWORK_FRAME_BYTES_STEREO); + memset(decodedBuffer.data(), 0, decodedBuffer.size()); + } + _ringBuffer.writeData(decodedBuffer.data(), decodedBuffer.size()); + } + return 0; +} + int InboundAudioStream::parseAudioData(PacketType type, const QByteArray& packetAfterStreamProperties) { QByteArray decodedBuffer; if (_decoder) { @@ -220,9 +236,6 @@ int InboundAudioStream::parseAudioData(PacketType type, const QByteArray& packet } int InboundAudioStream::writeDroppableSilentFrames(int silentFrames) { - if (_decoder) { - _decoder->trackLostFrames(silentFrames); - } // calculate how many silent frames we should drop. int silentSamples = silentFrames * _numChannels; diff --git a/libraries/audio/src/InboundAudioStream.h b/libraries/audio/src/InboundAudioStream.h index f7b79ab136..b349016452 100644 --- a/libraries/audio/src/InboundAudioStream.h +++ b/libraries/audio/src/InboundAudioStream.h @@ -134,6 +134,9 @@ protected: /// default implementation assumes packet contains raw audio samples after stream properties virtual int parseAudioData(PacketType type, const QByteArray& packetAfterStreamProperties); + /// produces audio data for lost network packets. + virtual int lostAudioData(int numPackets); + /// writes silent frames to the buffer that may be dropped to reduce latency caused by the buffer virtual int writeDroppableSilentFrames(int silentFrames); diff --git a/libraries/audio/src/MixedProcessedAudioStream.cpp b/libraries/audio/src/MixedProcessedAudioStream.cpp index 671d3a9d60..c3170c3259 100644 --- a/libraries/audio/src/MixedProcessedAudioStream.cpp +++ b/libraries/audio/src/MixedProcessedAudioStream.cpp @@ -38,6 +38,28 @@ int MixedProcessedAudioStream::writeLastFrameRepeatedWithFade(int frames) { return deviceFramesWritten; } +int MixedProcessedAudioStream::lostAudioData(int numPackets) { + QByteArray decodedBuffer; + QByteArray outputBuffer; + + while (numPackets--) { + if (_decoder) { + _decoder->lostFrame(decodedBuffer); + } else { + decodedBuffer.resize(AudioConstants::NETWORK_FRAME_BYTES_STEREO); + memset(decodedBuffer.data(), 0, decodedBuffer.size()); + } + + emit addedStereoSamples(decodedBuffer); + + emit processSamples(decodedBuffer, outputBuffer); + + _ringBuffer.writeData(outputBuffer.data(), outputBuffer.size()); + qCDebug(audiostream, "Wrote %d samples to buffer (%d available)", outputBuffer.size() / (int)sizeof(int16_t), getSamplesAvailable()); + } + return 0; +} + int MixedProcessedAudioStream::parseAudioData(PacketType type, const QByteArray& packetAfterStreamProperties) { QByteArray decodedBuffer; if (_decoder) { diff --git a/libraries/audio/src/MixedProcessedAudioStream.h b/libraries/audio/src/MixedProcessedAudioStream.h index d536163d2d..83c4cea635 100644 --- a/libraries/audio/src/MixedProcessedAudioStream.h +++ b/libraries/audio/src/MixedProcessedAudioStream.h @@ -36,6 +36,7 @@ protected: int writeDroppableSilentFrames(int silentFrames) override; int writeLastFrameRepeatedWithFade(int frames) override; int parseAudioData(PacketType type, const QByteArray& packetAfterStreamProperties) override; + int lostAudioData(int numPackets) override; private: int networkToDeviceFrames(int networkFrames); diff --git a/libraries/plugins/src/plugins/CodecPlugin.h b/libraries/plugins/src/plugins/CodecPlugin.h index 404f05e860..cb5b857be8 100644 --- a/libraries/plugins/src/plugins/CodecPlugin.h +++ b/libraries/plugins/src/plugins/CodecPlugin.h @@ -23,8 +23,7 @@ public: virtual ~Decoder() { } virtual void decode(const QByteArray& encodedBuffer, QByteArray& decodedBuffer) = 0; - // numFrames - number of samples (mono) or sample-pairs (stereo) - virtual void trackLostFrames(int numFrames) = 0; + virtual void lostFrame(QByteArray& decodedBuffer) = 0; }; class CodecPlugin : public Plugin { diff --git a/plugins/hifiCodec/src/HiFiCodec.cpp b/plugins/hifiCodec/src/HiFiCodec.cpp index 77c369dcae..2c7151fe59 100644 --- a/plugins/hifiCodec/src/HiFiCodec.cpp +++ b/plugins/hifiCodec/src/HiFiCodec.cpp @@ -65,12 +65,10 @@ public: AudioDecoder::process((const int16_t*)encodedBuffer.constData(), (int16_t*)decodedBuffer.data(), AudioConstants::NETWORK_FRAME_SAMPLES_PER_CHANNEL, true); } - virtual void trackLostFrames(int numFrames) override { - QByteArray encodedBuffer; - QByteArray decodedBuffer; + virtual void lostFrame(QByteArray& decodedBuffer) override { decodedBuffer.resize(_decodedSize); - // NOTE: we don't actually use the results of this decode, we just do it to keep the state of the codec clean - AudioDecoder::process((const int16_t*)encodedBuffer.constData(), (int16_t*)decodedBuffer.data(), AudioConstants::NETWORK_FRAME_SAMPLES_PER_CHANNEL, false); + // this performs packet loss interpolation + AudioDecoder::process(nullptr, (int16_t*)decodedBuffer.data(), AudioConstants::NETWORK_FRAME_SAMPLES_PER_CHANNEL, false); } private: int _decodedSize; diff --git a/plugins/pcmCodec/src/PCMCodecManager.h b/plugins/pcmCodec/src/PCMCodecManager.h index d58a219fef..608e9a1556 100644 --- a/plugins/pcmCodec/src/PCMCodecManager.h +++ b/plugins/pcmCodec/src/PCMCodecManager.h @@ -38,11 +38,14 @@ public: virtual void encode(const QByteArray& decodedBuffer, QByteArray& encodedBuffer) override { encodedBuffer = decodedBuffer; } + virtual void decode(const QByteArray& encodedBuffer, QByteArray& decodedBuffer) override { decodedBuffer = encodedBuffer; } - virtual void trackLostFrames(int numFrames) override { } + virtual void lostFrame(QByteArray& decodedBuffer) override { + memset(decodedBuffer.data(), 0, decodedBuffer.size()); + } private: static const char* NAME; @@ -77,7 +80,9 @@ public: decodedBuffer = qUncompress(encodedBuffer); } - virtual void trackLostFrames(int numFrames) override { } + virtual void lostFrame(QByteArray& decodedBuffer) override { + memset(decodedBuffer.data(), 0, decodedBuffer.size()); + } private: static const char* NAME;