Mirrors/overte-Armored-Dragon
3
0
Fork 0
mirror of https://github.com/Armored-Dragon/overte.git synced 2025-03-11 16:13:16 +01:00
Code Issues Projects Releases Packages Wiki Activity Actions

Merge pull request #15530 from roxanneskelly/bugz85b

Browse source 
BUGZ-85 - extrapolate audio on network lag spike to reduce pop
This commit is contained in:
Roxanne Skelly 2019-05-10 13:51:30 -07:00 committed by GitHub
commit 7dfbbf55e9
No known key found for this signature in database
GPG key ID: 4AEE18F83AFDEB23
4 changed files with 68 additions and 15 deletions
Show stats Download patch file Download diff file Expand all files Collapse all files

1
assignment-client/src/audio/AvatarAudioStream.cpp
View file

@ -45,6 +45,7 @@ int AvatarAudioStream::parseStreamProperties(PacketType type, const QByteArray&
: AudioConstants::NETWORK_FRAME_SAMPLES_PER_CHANNEL);
// restart the codec
if (_codec) {
QMutexLocker lock(&_decoderMutex);
if (_decoder) {
_codec->releaseDecoder(_decoder);
}

65
libraries/audio/src/InboundAudioStream.cpp
View file

@ -10,6 +10,7 @@
//
#include "InboundAudioStream.h"
#include "TryLocker.h"
#include <glm/glm.hpp>
@ -215,7 +216,7 @@ int InboundAudioStream::parseData(ReceivedMessage& message) {
if (framesAvailable > _desiredJitterBufferFrames + MAX_FRAMES_OVER_DESIRED) {
int framesToDrop = framesAvailable - (_desiredJitterBufferFrames + DESIRED_JITTER_BUFFER_FRAMES_PADDING);
_ringBuffer.shiftReadPosition(framesToDrop * _ringBuffer.getNumFrameSamples());
_framesAvailableStat.reset();
_currentJitterBufferFrames = 0;
@ -247,10 +248,18 @@ int InboundAudioStream::lostAudioData(int numPackets) {
QByteArray decodedBuffer;
while (numPackets--) {
MutexTryLocker lock(_decoderMutex);
if (!lock.isLocked()) {
// an incoming packet is being processed,
// and will likely be on the ring buffer shortly,
// so don't bother generating more data
qCInfo(audiostream, "Packet currently being unpacked or lost frame already being generated. Not generating lost frame.");
return 0;
}
if (_decoder) {
_decoder->lostFrame(decodedBuffer);
} else {
decodedBuffer.resize(AudioConstants::NETWORK_FRAME_BYTES_STEREO);
decodedBuffer.resize(AudioConstants::NETWORK_FRAME_BYTES_PER_CHANNEL * _numChannels);
memset(decodedBuffer.data(), 0, decodedBuffer.size());
}
_ringBuffer.writeData(decodedBuffer.data(), decodedBuffer.size());
@ -260,6 +269,12 @@ int InboundAudioStream::lostAudioData(int numPackets) {
int InboundAudioStream::parseAudioData(PacketType type, const QByteArray& packetAfterStreamProperties) {
QByteArray decodedBuffer;
// may block on the real-time thread, which is acceptible as
// parseAudioData is only called by the packet processing
// thread which, while high performance, is not as sensitive to
// delays as the real-time thread.
QMutexLocker lock(&_decoderMutex);
if (_decoder) {
_decoder->decode(packetAfterStreamProperties, decodedBuffer);
} else {
@ -278,16 +293,23 @@ int InboundAudioStream::writeDroppableSilentFrames(int silentFrames) {
// case we will call the decoder's lostFrame() method, which indicates
// that it should interpolate from its last known state down toward
// silence.
if (_decoder) {
// FIXME - We could potentially use the output from the codec, in which
// case we might get a cleaner fade toward silence. NOTE: The below logic
// attempts to catch up in the event that the jitter buffers have grown.
// The better long term fix is to use the output from the decode, detect
// when it actually reaches silence, and then delete the silent portions
// of the jitter buffers. Or petentially do a cross fade from the decode
// output to silence.
QByteArray decodedBuffer;
_decoder->lostFrame(decodedBuffer);
{
// may block on the real-time thread, which is acceptible as
// writeDroppableSilentFrames is only called by the packet processing
// thread which, while high performance, is not as sensitive to
// delays as the real-time thread.
QMutexLocker lock(&_decoderMutex);
if (_decoder) {
// FIXME - We could potentially use the output from the codec, in which
// case we might get a cleaner fade toward silence. NOTE: The below logic
// attempts to catch up in the event that the jitter buffers have grown.
// The better long term fix is to use the output from the decode, detect
// when it actually reaches silence, and then delete the silent portions
// of the jitter buffers. Or petentially do a cross fade from the decode
// output to silence.
QByteArray decodedBuffer;
_decoder->lostFrame(decodedBuffer);
}
}
// calculate how many silent frames we should drop.
@ -338,10 +360,23 @@ int InboundAudioStream::popSamples(int maxSamples, bool allOrNothing) {
popSamplesNoCheck(samplesAvailable);
samplesPopped = samplesAvailable;
} else {
// we can't pop any samples, set this stream to starved
// we can't pop any samples, set this stream to starved for jitter
// buffer calculations.
setToStarved();
_consecutiveNotMixedCount++;
_lastPopSucceeded = false;
//Kick PLC to generate a filler frame, reducing 'click'
lostAudioData(allOrNothing ? (maxSamples - samplesAvailable) / _ringBuffer.getNumFrameSamples() : 1);
samplesPopped = _ringBuffer.samplesAvailable();
if (samplesPopped) {
popSamplesNoCheck(samplesPopped);
} else {
// No samples available means a packet is currently being
// processed, so we don't generate lost audio data, and instead
// just wait for the packet to come in. This prevents locking
// the real-time audio thread at the cost of a potential (but rare)
// 'click'
_lastPopSucceeded = false;
}
}
}
return samplesPopped;
@ -528,6 +563,7 @@ void InboundAudioStream::setupCodec(CodecPluginPointer codec, const QString& cod
_codec = codec;
_selectedCodecName = codecName;
if (_codec) {
QMutexLocker lock(&_decoderMutex);
_decoder = codec->createDecoder(AudioConstants::SAMPLE_RATE, numChannels);
}
}
@ -535,6 +571,7 @@ void InboundAudioStream::setupCodec(CodecPluginPointer codec, const QString& cod
void InboundAudioStream::cleanupCodec() {
// release any old codec encoder/decoder first...
if (_codec) {
QMutexLocker lock(&_decoderMutex);
if (_decoder) {
_codec->releaseDecoder(_decoder);
_decoder = nullptr;

1
libraries/audio/src/InboundAudioStream.h
View file

@ -187,6 +187,7 @@ protected:
CodecPluginPointer _codec;
QString _selectedCodecName;
QMutex _decoderMutex;
Decoder* _decoder { nullptr };
int _mismatchedAudioCodecCount { 0 };
};

16
libraries/audio/src/MixedProcessedAudioStream.cpp
View file

@ -11,6 +11,7 @@
#include "MixedProcessedAudioStream.h"
#include "AudioLogging.h"
#include "TryLocker.h"
MixedProcessedAudioStream::MixedProcessedAudioStream(int numFramesCapacity, int numStaticJitterFrames)
: InboundAudioStream(AudioConstants::STEREO, AudioConstants::NETWORK_FRAME_SAMPLES_PER_CHANNEL,
@ -36,13 +37,20 @@ int MixedProcessedAudioStream::lostAudioData(int numPackets) {
QByteArray outputBuffer;
while (numPackets--) {
MutexTryLocker lock(_decoderMutex);
if (!lock.isLocked()) {
// an incoming packet is being processed,
// and will likely be on the ring buffer shortly,
// so don't bother generating more data
qCInfo(audiostream, "Packet currently being unpacked or lost frame already being generated. Not generating lost frame.");
return 0;
}
if (_decoder) {
_decoder->lostFrame(decodedBuffer);
} else {
decodedBuffer.resize(AudioConstants::NETWORK_FRAME_BYTES_STEREO);
memset(decodedBuffer.data(), 0, decodedBuffer.size());
}
emit addedStereoSamples(decodedBuffer);
emit processSamples(decodedBuffer, outputBuffer);
@ -55,6 +63,12 @@ int MixedProcessedAudioStream::lostAudioData(int numPackets) {
int MixedProcessedAudioStream::parseAudioData(PacketType type, const QByteArray& packetAfterStreamProperties) {
QByteArray decodedBuffer;
// may block on the real-time thread, which is acceptible as
// parseAudioData is only called by the packet processing
// thread which, while high performance, is not as sensitive to
// delays as the real-time thread.
QMutexLocker lock(&_decoderMutex);
if (_decoder) {
_decoder->decode(packetAfterStreamProperties, decodedBuffer);
} else {