From 4acb99cd4fdb4310dd539bfaf7cc6aa89017807b Mon Sep 17 00:00:00 2001 From: Ken Cooke Date: Mon, 6 Feb 2017 09:29:39 -0800 Subject: [PATCH 1/5] Use interpolated audio (codec packet-loss concealment) or zero samples (if no codec) when audio packets are lost. This audio is still processed by the audio pipeline to avoid clicks/pops. --- libraries/audio/src/InboundAudioStream.cpp | 21 ++++++++++++++---- libraries/audio/src/InboundAudioStream.h | 3 +++ .../audio/src/MixedProcessedAudioStream.cpp | 22 +++++++++++++++++++ .../audio/src/MixedProcessedAudioStream.h | 1 + libraries/plugins/src/plugins/CodecPlugin.h | 3 +-- plugins/hifiCodec/src/HiFiCodec.cpp | 8 +++---- plugins/pcmCodec/src/PCMCodecManager.h | 9 ++++++-- 7 files changed, 54 insertions(+), 13 deletions(-) diff --git a/libraries/audio/src/InboundAudioStream.cpp b/libraries/audio/src/InboundAudioStream.cpp index 36a6079546..6f61c59cbd 100644 --- a/libraries/audio/src/InboundAudioStream.cpp +++ b/libraries/audio/src/InboundAudioStream.cpp @@ -136,7 +136,8 @@ int InboundAudioStream::parseData(ReceivedMessage& message) { // NOTE: we assume that each dropped packet contains the same number of samples // as the packet we just received. int packetsDropped = arrivalInfo._seqDiffFromExpected; - writeFramesForDroppedPackets(packetsDropped * networkFrames); + //writeFramesForDroppedPackets(packetsDropped * networkFrames); + lostAudioData(packetsDropped); // fall through to OnTime case } @@ -208,6 +209,21 @@ int InboundAudioStream::parseStreamProperties(PacketType type, const QByteArray& } } +int InboundAudioStream::lostAudioData(int numPackets) { + QByteArray decodedBuffer; + + while (numPackets--) { + if (_decoder) { + _decoder->lostFrame(decodedBuffer); + } else { + decodedBuffer.resize(AudioConstants::NETWORK_FRAME_BYTES_STEREO); + memset(decodedBuffer.data(), 0, decodedBuffer.size()); + } + _ringBuffer.writeData(decodedBuffer.data(), decodedBuffer.size()); + } + return 0; +} + int InboundAudioStream::parseAudioData(PacketType type, const QByteArray& packetAfterStreamProperties) { QByteArray decodedBuffer; if (_decoder) { @@ -220,9 +236,6 @@ int InboundAudioStream::parseAudioData(PacketType type, const QByteArray& packet } int InboundAudioStream::writeDroppableSilentFrames(int silentFrames) { - if (_decoder) { - _decoder->trackLostFrames(silentFrames); - } // calculate how many silent frames we should drop. int silentSamples = silentFrames * _numChannels; diff --git a/libraries/audio/src/InboundAudioStream.h b/libraries/audio/src/InboundAudioStream.h index f7b79ab136..b349016452 100644 --- a/libraries/audio/src/InboundAudioStream.h +++ b/libraries/audio/src/InboundAudioStream.h @@ -134,6 +134,9 @@ protected: /// default implementation assumes packet contains raw audio samples after stream properties virtual int parseAudioData(PacketType type, const QByteArray& packetAfterStreamProperties); + /// produces audio data for lost network packets. + virtual int lostAudioData(int numPackets); + /// writes silent frames to the buffer that may be dropped to reduce latency caused by the buffer virtual int writeDroppableSilentFrames(int silentFrames); diff --git a/libraries/audio/src/MixedProcessedAudioStream.cpp b/libraries/audio/src/MixedProcessedAudioStream.cpp index 671d3a9d60..c3170c3259 100644 --- a/libraries/audio/src/MixedProcessedAudioStream.cpp +++ b/libraries/audio/src/MixedProcessedAudioStream.cpp @@ -38,6 +38,28 @@ int MixedProcessedAudioStream::writeLastFrameRepeatedWithFade(int frames) { return deviceFramesWritten; } +int MixedProcessedAudioStream::lostAudioData(int numPackets) { + QByteArray decodedBuffer; + QByteArray outputBuffer; + + while (numPackets--) { + if (_decoder) { + _decoder->lostFrame(decodedBuffer); + } else { + decodedBuffer.resize(AudioConstants::NETWORK_FRAME_BYTES_STEREO); + memset(decodedBuffer.data(), 0, decodedBuffer.size()); + } + + emit addedStereoSamples(decodedBuffer); + + emit processSamples(decodedBuffer, outputBuffer); + + _ringBuffer.writeData(outputBuffer.data(), outputBuffer.size()); + qCDebug(audiostream, "Wrote %d samples to buffer (%d available)", outputBuffer.size() / (int)sizeof(int16_t), getSamplesAvailable()); + } + return 0; +} + int MixedProcessedAudioStream::parseAudioData(PacketType type, const QByteArray& packetAfterStreamProperties) { QByteArray decodedBuffer; if (_decoder) { diff --git a/libraries/audio/src/MixedProcessedAudioStream.h b/libraries/audio/src/MixedProcessedAudioStream.h index d536163d2d..83c4cea635 100644 --- a/libraries/audio/src/MixedProcessedAudioStream.h +++ b/libraries/audio/src/MixedProcessedAudioStream.h @@ -36,6 +36,7 @@ protected: int writeDroppableSilentFrames(int silentFrames) override; int writeLastFrameRepeatedWithFade(int frames) override; int parseAudioData(PacketType type, const QByteArray& packetAfterStreamProperties) override; + int lostAudioData(int numPackets) override; private: int networkToDeviceFrames(int networkFrames); diff --git a/libraries/plugins/src/plugins/CodecPlugin.h b/libraries/plugins/src/plugins/CodecPlugin.h index 404f05e860..cb5b857be8 100644 --- a/libraries/plugins/src/plugins/CodecPlugin.h +++ b/libraries/plugins/src/plugins/CodecPlugin.h @@ -23,8 +23,7 @@ public: virtual ~Decoder() { } virtual void decode(const QByteArray& encodedBuffer, QByteArray& decodedBuffer) = 0; - // numFrames - number of samples (mono) or sample-pairs (stereo) - virtual void trackLostFrames(int numFrames) = 0; + virtual void lostFrame(QByteArray& decodedBuffer) = 0; }; class CodecPlugin : public Plugin { diff --git a/plugins/hifiCodec/src/HiFiCodec.cpp b/plugins/hifiCodec/src/HiFiCodec.cpp index 77c369dcae..2c7151fe59 100644 --- a/plugins/hifiCodec/src/HiFiCodec.cpp +++ b/plugins/hifiCodec/src/HiFiCodec.cpp @@ -65,12 +65,10 @@ public: AudioDecoder::process((const int16_t*)encodedBuffer.constData(), (int16_t*)decodedBuffer.data(), AudioConstants::NETWORK_FRAME_SAMPLES_PER_CHANNEL, true); } - virtual void trackLostFrames(int numFrames) override { - QByteArray encodedBuffer; - QByteArray decodedBuffer; + virtual void lostFrame(QByteArray& decodedBuffer) override { decodedBuffer.resize(_decodedSize); - // NOTE: we don't actually use the results of this decode, we just do it to keep the state of the codec clean - AudioDecoder::process((const int16_t*)encodedBuffer.constData(), (int16_t*)decodedBuffer.data(), AudioConstants::NETWORK_FRAME_SAMPLES_PER_CHANNEL, false); + // this performs packet loss interpolation + AudioDecoder::process(nullptr, (int16_t*)decodedBuffer.data(), AudioConstants::NETWORK_FRAME_SAMPLES_PER_CHANNEL, false); } private: int _decodedSize; diff --git a/plugins/pcmCodec/src/PCMCodecManager.h b/plugins/pcmCodec/src/PCMCodecManager.h index d58a219fef..608e9a1556 100644 --- a/plugins/pcmCodec/src/PCMCodecManager.h +++ b/plugins/pcmCodec/src/PCMCodecManager.h @@ -38,11 +38,14 @@ public: virtual void encode(const QByteArray& decodedBuffer, QByteArray& encodedBuffer) override { encodedBuffer = decodedBuffer; } + virtual void decode(const QByteArray& encodedBuffer, QByteArray& decodedBuffer) override { decodedBuffer = encodedBuffer; } - virtual void trackLostFrames(int numFrames) override { } + virtual void lostFrame(QByteArray& decodedBuffer) override { + memset(decodedBuffer.data(), 0, decodedBuffer.size()); + } private: static const char* NAME; @@ -77,7 +80,9 @@ public: decodedBuffer = qUncompress(encodedBuffer); } - virtual void trackLostFrames(int numFrames) override { } + virtual void lostFrame(QByteArray& decodedBuffer) override { + memset(decodedBuffer.data(), 0, decodedBuffer.size()); + } private: static const char* NAME; From 8033f932a65734dc76a5cadbf0cd7f6c00ce7030 Mon Sep 17 00:00:00 2001 From: Ken Cooke Date: Mon, 6 Feb 2017 10:16:25 -0800 Subject: [PATCH 2/5] Simulate 1% random packet-loss, for debug/test --- libraries/audio/src/InboundAudioStream.cpp | 11 ++++++++++- 1 file changed, 10 insertions(+), 1 deletion(-) diff --git a/libraries/audio/src/InboundAudioStream.cpp b/libraries/audio/src/InboundAudioStream.cpp index 6f61c59cbd..57e3f4eb36 100644 --- a/libraries/audio/src/InboundAudioStream.cpp +++ b/libraries/audio/src/InboundAudioStream.cpp @@ -129,14 +129,23 @@ int InboundAudioStream::parseData(ReceivedMessage& message) { int propertyBytes = parseStreamProperties(message.getType(), message.readWithoutCopy(message.getBytesLeftToRead()), networkFrames); message.seek(prePropertyPosition + propertyBytes); + // simulate 1% packetloss + if (rand() < (RAND_MAX/100)) { + arrivalInfo._status = SequenceNumberStats::Unreasonable; + qDebug(audio) << "Simulated a lost packet..."; + } + // handle this packet based on its arrival status. switch (arrivalInfo._status) { + case SequenceNumberStats::Unreasonable: { + lostAudioData(1); + break; + } case SequenceNumberStats::Early: { // Packet is early; write droppable silent samples for each of the skipped packets. // NOTE: we assume that each dropped packet contains the same number of samples // as the packet we just received. int packetsDropped = arrivalInfo._seqDiffFromExpected; - //writeFramesForDroppedPackets(packetsDropped * networkFrames); lostAudioData(packetsDropped); // fall through to OnTime case From 49605d52e058249e7ccd2483ec70683045f044a8 Mon Sep 17 00:00:00 2001 From: Ken Cooke Date: Mon, 6 Feb 2017 10:42:32 -0800 Subject: [PATCH 3/5] Remove debug code --- libraries/audio/src/InboundAudioStream.cpp | 6 ------ 1 file changed, 6 deletions(-) diff --git a/libraries/audio/src/InboundAudioStream.cpp b/libraries/audio/src/InboundAudioStream.cpp index 57e3f4eb36..d23579430e 100644 --- a/libraries/audio/src/InboundAudioStream.cpp +++ b/libraries/audio/src/InboundAudioStream.cpp @@ -129,12 +129,6 @@ int InboundAudioStream::parseData(ReceivedMessage& message) { int propertyBytes = parseStreamProperties(message.getType(), message.readWithoutCopy(message.getBytesLeftToRead()), networkFrames); message.seek(prePropertyPosition + propertyBytes); - // simulate 1% packetloss - if (rand() < (RAND_MAX/100)) { - arrivalInfo._status = SequenceNumberStats::Unreasonable; - qDebug(audio) << "Simulated a lost packet..."; - } - // handle this packet based on its arrival status. switch (arrivalInfo._status) { case SequenceNumberStats::Unreasonable: { From a77e4262e3f1112ee77f715a50ce27b668a29d1a Mon Sep 17 00:00:00 2001 From: Ken Cooke Date: Mon, 6 Feb 2017 12:47:28 -0800 Subject: [PATCH 4/5] Remove obsoleted code --- libraries/audio/src/InboundAudioStream.cpp | 4 ---- libraries/audio/src/InboundAudioStream.h | 2 -- 2 files changed, 6 deletions(-) diff --git a/libraries/audio/src/InboundAudioStream.cpp b/libraries/audio/src/InboundAudioStream.cpp index d23579430e..c90032436b 100644 --- a/libraries/audio/src/InboundAudioStream.cpp +++ b/libraries/audio/src/InboundAudioStream.cpp @@ -432,10 +432,6 @@ void InboundAudioStream::packetReceivedUpdateTimingStats() { _lastPacketReceivedTime = now; } -int InboundAudioStream::writeFramesForDroppedPackets(int networkFrames) { - return writeLastFrameRepeatedWithFade(networkFrames); -} - int InboundAudioStream::writeLastFrameRepeatedWithFade(int frames) { AudioRingBuffer::ConstIterator frameToRepeat = _ringBuffer.lastFrameWritten(); int frameSize = _ringBuffer.getNumFrameSamples(); diff --git a/libraries/audio/src/InboundAudioStream.h b/libraries/audio/src/InboundAudioStream.h index b349016452..f96ef539c7 100644 --- a/libraries/audio/src/InboundAudioStream.h +++ b/libraries/audio/src/InboundAudioStream.h @@ -115,8 +115,6 @@ public slots: private: void packetReceivedUpdateTimingStats(); - int writeFramesForDroppedPackets(int networkFrames); - void popSamplesNoCheck(int samples); void framesAvailableChanged(); From 63c2aa1dfcd302576a2a541fbb0fede4a3f71f64 Mon Sep 17 00:00:00 2001 From: Ken Cooke Date: Mon, 6 Feb 2017 12:55:03 -0800 Subject: [PATCH 5/5] Remove obsoleted code --- libraries/audio/src/InboundAudioStream.cpp | 19 ------------------- libraries/audio/src/InboundAudioStream.h | 4 ---- .../audio/src/MixedProcessedAudioStream.cpp | 7 ------- .../audio/src/MixedProcessedAudioStream.h | 1 - 4 files changed, 31 deletions(-) diff --git a/libraries/audio/src/InboundAudioStream.cpp b/libraries/audio/src/InboundAudioStream.cpp index c90032436b..57c344adaf 100644 --- a/libraries/audio/src/InboundAudioStream.cpp +++ b/libraries/audio/src/InboundAudioStream.cpp @@ -432,25 +432,6 @@ void InboundAudioStream::packetReceivedUpdateTimingStats() { _lastPacketReceivedTime = now; } -int InboundAudioStream::writeLastFrameRepeatedWithFade(int frames) { - AudioRingBuffer::ConstIterator frameToRepeat = _ringBuffer.lastFrameWritten(); - int frameSize = _ringBuffer.getNumFrameSamples(); - int samplesToWrite = frames * _numChannels; - int indexOfRepeat = 0; - do { - int samplesToWriteThisIteration = std::min(samplesToWrite, frameSize); - float fade = calculateRepeatedFrameFadeFactor(indexOfRepeat); - if (fade == 1.0f) { - samplesToWrite -= _ringBuffer.writeSamples(frameToRepeat, samplesToWriteThisIteration); - } else { - samplesToWrite -= _ringBuffer.writeSamplesWithFade(frameToRepeat, samplesToWriteThisIteration, fade); - } - indexOfRepeat++; - } while (samplesToWrite > 0); - - return frames; -} - AudioStreamStats InboundAudioStream::getAudioStreamStats() const { AudioStreamStats streamStats; diff --git a/libraries/audio/src/InboundAudioStream.h b/libraries/audio/src/InboundAudioStream.h index f96ef539c7..9494b2f204 100644 --- a/libraries/audio/src/InboundAudioStream.h +++ b/libraries/audio/src/InboundAudioStream.h @@ -137,10 +137,6 @@ protected: /// writes silent frames to the buffer that may be dropped to reduce latency caused by the buffer virtual int writeDroppableSilentFrames(int silentFrames); - - /// writes the last written frame repeatedly, gradually fading to silence. - /// used for writing samples for dropped packets. - virtual int writeLastFrameRepeatedWithFade(int frames); protected: diff --git a/libraries/audio/src/MixedProcessedAudioStream.cpp b/libraries/audio/src/MixedProcessedAudioStream.cpp index c3170c3259..082977246b 100644 --- a/libraries/audio/src/MixedProcessedAudioStream.cpp +++ b/libraries/audio/src/MixedProcessedAudioStream.cpp @@ -31,13 +31,6 @@ int MixedProcessedAudioStream::writeDroppableSilentFrames(int silentFrames) { return deviceSilentFramesWritten; } -int MixedProcessedAudioStream::writeLastFrameRepeatedWithFade(int frames) { - int deviceFrames = networkToDeviceFrames(frames); - int deviceFramesWritten = InboundAudioStream::writeLastFrameRepeatedWithFade(deviceFrames); - emit addedLastFrameRepeatedWithFade(deviceToNetworkFrames(deviceFramesWritten)); - return deviceFramesWritten; -} - int MixedProcessedAudioStream::lostAudioData(int numPackets) { QByteArray decodedBuffer; QByteArray outputBuffer; diff --git a/libraries/audio/src/MixedProcessedAudioStream.h b/libraries/audio/src/MixedProcessedAudioStream.h index 83c4cea635..14da1d45af 100644 --- a/libraries/audio/src/MixedProcessedAudioStream.h +++ b/libraries/audio/src/MixedProcessedAudioStream.h @@ -34,7 +34,6 @@ public: protected: int writeDroppableSilentFrames(int silentFrames) override; - int writeLastFrameRepeatedWithFade(int frames) override; int parseAudioData(PacketType type, const QByteArray& packetAfterStreamProperties) override; int lostAudioData(int numPackets) override;