diff --git a/libraries/audio/src/InboundAudioStream.cpp b/libraries/audio/src/InboundAudioStream.cpp index 36a6079546..57c344adaf 100644 --- a/libraries/audio/src/InboundAudioStream.cpp +++ b/libraries/audio/src/InboundAudioStream.cpp @@ -131,12 +131,16 @@ int InboundAudioStream::parseData(ReceivedMessage& message) { // handle this packet based on its arrival status. switch (arrivalInfo._status) { + case SequenceNumberStats::Unreasonable: { + lostAudioData(1); + break; + } case SequenceNumberStats::Early: { // Packet is early; write droppable silent samples for each of the skipped packets. // NOTE: we assume that each dropped packet contains the same number of samples // as the packet we just received. int packetsDropped = arrivalInfo._seqDiffFromExpected; - writeFramesForDroppedPackets(packetsDropped * networkFrames); + lostAudioData(packetsDropped); // fall through to OnTime case } @@ -208,6 +212,21 @@ int InboundAudioStream::parseStreamProperties(PacketType type, const QByteArray& } } +int InboundAudioStream::lostAudioData(int numPackets) { + QByteArray decodedBuffer; + + while (numPackets--) { + if (_decoder) { + _decoder->lostFrame(decodedBuffer); + } else { + decodedBuffer.resize(AudioConstants::NETWORK_FRAME_BYTES_STEREO); + memset(decodedBuffer.data(), 0, decodedBuffer.size()); + } + _ringBuffer.writeData(decodedBuffer.data(), decodedBuffer.size()); + } + return 0; +} + int InboundAudioStream::parseAudioData(PacketType type, const QByteArray& packetAfterStreamProperties) { QByteArray decodedBuffer; if (_decoder) { @@ -220,9 +239,6 @@ int InboundAudioStream::parseAudioData(PacketType type, const QByteArray& packet } int InboundAudioStream::writeDroppableSilentFrames(int silentFrames) { - if (_decoder) { - _decoder->trackLostFrames(silentFrames); - } // calculate how many silent frames we should drop. int silentSamples = silentFrames * _numChannels; @@ -416,29 +432,6 @@ void InboundAudioStream::packetReceivedUpdateTimingStats() { _lastPacketReceivedTime = now; } -int InboundAudioStream::writeFramesForDroppedPackets(int networkFrames) { - return writeLastFrameRepeatedWithFade(networkFrames); -} - -int InboundAudioStream::writeLastFrameRepeatedWithFade(int frames) { - AudioRingBuffer::ConstIterator frameToRepeat = _ringBuffer.lastFrameWritten(); - int frameSize = _ringBuffer.getNumFrameSamples(); - int samplesToWrite = frames * _numChannels; - int indexOfRepeat = 0; - do { - int samplesToWriteThisIteration = std::min(samplesToWrite, frameSize); - float fade = calculateRepeatedFrameFadeFactor(indexOfRepeat); - if (fade == 1.0f) { - samplesToWrite -= _ringBuffer.writeSamples(frameToRepeat, samplesToWriteThisIteration); - } else { - samplesToWrite -= _ringBuffer.writeSamplesWithFade(frameToRepeat, samplesToWriteThisIteration, fade); - } - indexOfRepeat++; - } while (samplesToWrite > 0); - - return frames; -} - AudioStreamStats InboundAudioStream::getAudioStreamStats() const { AudioStreamStats streamStats; diff --git a/libraries/audio/src/InboundAudioStream.h b/libraries/audio/src/InboundAudioStream.h index f7b79ab136..9494b2f204 100644 --- a/libraries/audio/src/InboundAudioStream.h +++ b/libraries/audio/src/InboundAudioStream.h @@ -115,8 +115,6 @@ public slots: private: void packetReceivedUpdateTimingStats(); - int writeFramesForDroppedPackets(int networkFrames); - void popSamplesNoCheck(int samples); void framesAvailableChanged(); @@ -134,12 +132,11 @@ protected: /// default implementation assumes packet contains raw audio samples after stream properties virtual int parseAudioData(PacketType type, const QByteArray& packetAfterStreamProperties); + /// produces audio data for lost network packets. + virtual int lostAudioData(int numPackets); + /// writes silent frames to the buffer that may be dropped to reduce latency caused by the buffer virtual int writeDroppableSilentFrames(int silentFrames); - - /// writes the last written frame repeatedly, gradually fading to silence. - /// used for writing samples for dropped packets. - virtual int writeLastFrameRepeatedWithFade(int frames); protected: diff --git a/libraries/audio/src/MixedProcessedAudioStream.cpp b/libraries/audio/src/MixedProcessedAudioStream.cpp index 671d3a9d60..082977246b 100644 --- a/libraries/audio/src/MixedProcessedAudioStream.cpp +++ b/libraries/audio/src/MixedProcessedAudioStream.cpp @@ -31,11 +31,26 @@ int MixedProcessedAudioStream::writeDroppableSilentFrames(int silentFrames) { return deviceSilentFramesWritten; } -int MixedProcessedAudioStream::writeLastFrameRepeatedWithFade(int frames) { - int deviceFrames = networkToDeviceFrames(frames); - int deviceFramesWritten = InboundAudioStream::writeLastFrameRepeatedWithFade(deviceFrames); - emit addedLastFrameRepeatedWithFade(deviceToNetworkFrames(deviceFramesWritten)); - return deviceFramesWritten; +int MixedProcessedAudioStream::lostAudioData(int numPackets) { + QByteArray decodedBuffer; + QByteArray outputBuffer; + + while (numPackets--) { + if (_decoder) { + _decoder->lostFrame(decodedBuffer); + } else { + decodedBuffer.resize(AudioConstants::NETWORK_FRAME_BYTES_STEREO); + memset(decodedBuffer.data(), 0, decodedBuffer.size()); + } + + emit addedStereoSamples(decodedBuffer); + + emit processSamples(decodedBuffer, outputBuffer); + + _ringBuffer.writeData(outputBuffer.data(), outputBuffer.size()); + qCDebug(audiostream, "Wrote %d samples to buffer (%d available)", outputBuffer.size() / (int)sizeof(int16_t), getSamplesAvailable()); + } + return 0; } int MixedProcessedAudioStream::parseAudioData(PacketType type, const QByteArray& packetAfterStreamProperties) { diff --git a/libraries/audio/src/MixedProcessedAudioStream.h b/libraries/audio/src/MixedProcessedAudioStream.h index d536163d2d..14da1d45af 100644 --- a/libraries/audio/src/MixedProcessedAudioStream.h +++ b/libraries/audio/src/MixedProcessedAudioStream.h @@ -34,8 +34,8 @@ public: protected: int writeDroppableSilentFrames(int silentFrames) override; - int writeLastFrameRepeatedWithFade(int frames) override; int parseAudioData(PacketType type, const QByteArray& packetAfterStreamProperties) override; + int lostAudioData(int numPackets) override; private: int networkToDeviceFrames(int networkFrames); diff --git a/libraries/plugins/src/plugins/CodecPlugin.h b/libraries/plugins/src/plugins/CodecPlugin.h index 404f05e860..cb5b857be8 100644 --- a/libraries/plugins/src/plugins/CodecPlugin.h +++ b/libraries/plugins/src/plugins/CodecPlugin.h @@ -23,8 +23,7 @@ public: virtual ~Decoder() { } virtual void decode(const QByteArray& encodedBuffer, QByteArray& decodedBuffer) = 0; - // numFrames - number of samples (mono) or sample-pairs (stereo) - virtual void trackLostFrames(int numFrames) = 0; + virtual void lostFrame(QByteArray& decodedBuffer) = 0; }; class CodecPlugin : public Plugin { diff --git a/plugins/hifiCodec/src/HiFiCodec.cpp b/plugins/hifiCodec/src/HiFiCodec.cpp index 77c369dcae..2c7151fe59 100644 --- a/plugins/hifiCodec/src/HiFiCodec.cpp +++ b/plugins/hifiCodec/src/HiFiCodec.cpp @@ -65,12 +65,10 @@ public: AudioDecoder::process((const int16_t*)encodedBuffer.constData(), (int16_t*)decodedBuffer.data(), AudioConstants::NETWORK_FRAME_SAMPLES_PER_CHANNEL, true); } - virtual void trackLostFrames(int numFrames) override { - QByteArray encodedBuffer; - QByteArray decodedBuffer; + virtual void lostFrame(QByteArray& decodedBuffer) override { decodedBuffer.resize(_decodedSize); - // NOTE: we don't actually use the results of this decode, we just do it to keep the state of the codec clean - AudioDecoder::process((const int16_t*)encodedBuffer.constData(), (int16_t*)decodedBuffer.data(), AudioConstants::NETWORK_FRAME_SAMPLES_PER_CHANNEL, false); + // this performs packet loss interpolation + AudioDecoder::process(nullptr, (int16_t*)decodedBuffer.data(), AudioConstants::NETWORK_FRAME_SAMPLES_PER_CHANNEL, false); } private: int _decodedSize; diff --git a/plugins/pcmCodec/src/PCMCodecManager.h b/plugins/pcmCodec/src/PCMCodecManager.h index d58a219fef..608e9a1556 100644 --- a/plugins/pcmCodec/src/PCMCodecManager.h +++ b/plugins/pcmCodec/src/PCMCodecManager.h @@ -38,11 +38,14 @@ public: virtual void encode(const QByteArray& decodedBuffer, QByteArray& encodedBuffer) override { encodedBuffer = decodedBuffer; } + virtual void decode(const QByteArray& encodedBuffer, QByteArray& decodedBuffer) override { decodedBuffer = encodedBuffer; } - virtual void trackLostFrames(int numFrames) override { } + virtual void lostFrame(QByteArray& decodedBuffer) override { + memset(decodedBuffer.data(), 0, decodedBuffer.size()); + } private: static const char* NAME; @@ -77,7 +80,9 @@ public: decodedBuffer = qUncompress(encodedBuffer); } - virtual void trackLostFrames(int numFrames) override { } + virtual void lostFrame(QByteArray& decodedBuffer) override { + memset(decodedBuffer.data(), 0, decodedBuffer.size()); + } private: static const char* NAME;