Mirrors/overte-JulianGro
3
0
Fork 0
mirror of https://github.com/JulianGro/overte.git synced 2025-04-25 17:14:59 +02:00
Code Issues Projects Releases Packages Wiki Activity Actions

Merge pull request #3179 from wangyix/quick_audio_PR

Browse source 
Lost audio packets are replaced with silent frames in AudioMixer; AudioMixer silent-packet-dropping now enabled no matter what; Moving avg of audio frames available replaces _currentJitterBufferFrames
This commit is contained in:
Brad Hefta-Gaub 2014-07-17 11:44:05 -07:00
commit 437e3efbc2
17 changed files with 245 additions and 126 deletions
Show stats Download patch file Download diff file Expand all files Collapse all files

53
assignment-client/src/audio/AudioMixerClientData.cpp
View file

@ -61,7 +61,7 @@ int AudioMixerClientData::parseData(const QByteArray& packet) {
|| packetType == PacketTypeMicrophoneAudioNoEcho
|| packetType == PacketTypeSilentAudioFrame) {
_incomingAvatarAudioSequenceNumberStats.sequenceNumberReceived(sequence);
SequenceNumberStats::ArrivalInfo packetArrivalInfo = _incomingAvatarAudioSequenceNumberStats.sequenceNumberReceived(sequence);
// grab the AvatarAudioRingBuffer from the vector (or create it if it doesn't exist)
AvatarAudioRingBuffer* avatarRingBuffer = getAvatarAudioRingBuffer();
@ -84,8 +84,24 @@ int AudioMixerClientData::parseData(const QByteArray& packet) {
_ringBuffers.push_back(avatarRingBuffer);
}
// ask the AvatarAudioRingBuffer instance to parse the data
avatarRingBuffer->parseData(packet);
// for now, late packets are simply discarded. In the future, it may be good to insert them into their correct place
// in the ring buffer (if that frame hasn't been mixed yet)
switch (packetArrivalInfo._status) {
case SequenceNumberStats::Early: {
int packetsLost = packetArrivalInfo._seqDiffFromExpected;
avatarRingBuffer->parseData(packet, packetsLost);
break;
}
case SequenceNumberStats::OnTime: {
// ask the AvatarAudioRingBuffer instance to parse the data
avatarRingBuffer->parseData(packet);
break;
}
default: {
break;
}
}
} else if (packetType == PacketTypeInjectAudio) {
// this is injected audio
@ -95,7 +111,8 @@ int AudioMixerClientData::parseData(const QByteArray& packet) {
if (!_incomingInjectedAudioSequenceNumberStatsMap.contains(streamIdentifier)) {
_incomingInjectedAudioSequenceNumberStatsMap.insert(streamIdentifier, SequenceNumberStats(INCOMING_SEQ_STATS_HISTORY_LENGTH));
}
_incomingInjectedAudioSequenceNumberStatsMap[streamIdentifier].sequenceNumberReceived(sequence);
SequenceNumberStats::ArrivalInfo packetArrivalInfo =
_incomingInjectedAudioSequenceNumberStatsMap[streamIdentifier].sequenceNumberReceived(sequence);
InjectedAudioRingBuffer* matchingInjectedRingBuffer = NULL;
@ -112,7 +129,23 @@ int AudioMixerClientData::parseData(const QByteArray& packet) {
_ringBuffers.push_back(matchingInjectedRingBuffer);
}
matchingInjectedRingBuffer->parseData(packet);
// for now, late packets are simply discarded. In the future, it may be good to insert them into their correct place
// in the ring buffer (if that frame hasn't been mixed yet)
switch (packetArrivalInfo._status) {
case SequenceNumberStats::Early: {
int packetsLost = packetArrivalInfo._seqDiffFromExpected;
matchingInjectedRingBuffer->parseData(packet, packetsLost);
break;
}
case SequenceNumberStats::OnTime: {
// ask the AvatarAudioRingBuffer instance to parse the data
matchingInjectedRingBuffer->parseData(packet);
break;
}
default: {
break;
}
}
} else if (packetType == PacketTypeAudioStreamStats) {
const char* dataAt = packet.data();
@ -198,7 +231,7 @@ AudioStreamStats AudioMixerClientData::getAudioStreamStatsOfStream(const Positio
streamStats._timeGapWindowAverage = timeGapStats.getWindowAverage();
streamStats._ringBufferFramesAvailable = ringBuffer->framesAvailable();
streamStats._ringBufferCurrentJitterBufferFrames = ringBuffer->getCurrentJitterBufferFrames();
streamStats._ringBufferFramesAvailableAverage = ringBuffer->getFramesAvailableAverage();
streamStats._ringBufferDesiredJitterBufferFrames = ringBuffer->getDesiredJitterBufferFrames();
streamStats._ringBufferStarveCount = ringBuffer->getStarveCount();
streamStats._ringBufferConsecutiveNotMixedCount = ringBuffer->getConsecutiveNotMixedCount();
@ -272,7 +305,7 @@ QString AudioMixerClientData::getAudioStreamStatsString() const {
QString result;
AudioStreamStats streamStats = _downstreamAudioStreamStats;
result += "DOWNSTREAM.desired:" + QString::number(streamStats._ringBufferDesiredJitterBufferFrames)
+ " current: ?"
+ " available_avg_10s:" + QString::number(streamStats._ringBufferFramesAvailableAverage)
+ " available:" + QString::number(streamStats._ringBufferFramesAvailable)
+ " starves:" + QString::number(streamStats._ringBufferStarveCount)
+ " not_mixed:" + QString::number(streamStats._ringBufferConsecutiveNotMixedCount)
@ -291,7 +324,8 @@ QString AudioMixerClientData::getAudioStreamStatsString() const {
if (avatarRingBuffer) {
AudioStreamStats streamStats = getAudioStreamStatsOfStream(avatarRingBuffer);
result += " UPSTREAM.mic.desired:" + QString::number(streamStats._ringBufferDesiredJitterBufferFrames)
+ " current:" + QString::number(streamStats._ringBufferCurrentJitterBufferFrames)
+ " desired_calc:" + QString::number(avatarRingBuffer->getCalculatedDesiredJitterBufferFrames())
+ " available_avg_10s:" + QString::number(streamStats._ringBufferFramesAvailableAverage)
+ " available:" + QString::number(streamStats._ringBufferFramesAvailable)
+ " starves:" + QString::number(streamStats._ringBufferStarveCount)
+ " not_mixed:" + QString::number(streamStats._ringBufferConsecutiveNotMixedCount)
@ -313,7 +347,8 @@ QString AudioMixerClientData::getAudioStreamStatsString() const {
if (_ringBuffers[i]->getType() == PositionalAudioRingBuffer::Injector) {
AudioStreamStats streamStats = getAudioStreamStatsOfStream(_ringBuffers[i]);
result += " UPSTREAM.inj.desired:" + QString::number(streamStats._ringBufferDesiredJitterBufferFrames)
+ " current:" + QString::number(streamStats._ringBufferCurrentJitterBufferFrames)
+ " desired_calc:" + QString::number(_ringBuffers[i]->getCalculatedDesiredJitterBufferFrames())
+ " available_avg_10s:" + QString::number(streamStats._ringBufferFramesAvailableAverage)
+ " available:" + QString::number(streamStats._ringBufferFramesAvailable)
+ " starves:" + QString::number(streamStats._ringBufferStarveCount)
+ " not_mixed:" + QString::number(streamStats._ringBufferConsecutiveNotMixedCount)

51
assignment-client/src/audio/AvatarAudioRingBuffer.cpp
View file

@ -18,10 +18,53 @@ AvatarAudioRingBuffer::AvatarAudioRingBuffer(bool isStereo, bool dynamicJitterBu
}
int AvatarAudioRingBuffer::parseData(const QByteArray& packet) {
timeGapStatsFrameReceived();
updateDesiredJitterBufferFrames();
int AvatarAudioRingBuffer::parseData(const QByteArray& packet, int packetsSkipped) {
frameReceivedUpdateTimingStats();
_shouldLoopbackForNode = (packetTypeForPacket(packet) == PacketTypeMicrophoneAudioWithEcho);
return PositionalAudioRingBuffer::parseData(packet);
// skip the packet header (includes the source UUID)
int readBytes = numBytesForPacketHeader(packet);
// skip the sequence number
readBytes += sizeof(quint16);
// hop over the channel flag that has already been read in AudioMixerClientData
readBytes += sizeof(quint8);
// read the positional data
readBytes += parsePositionalData(packet.mid(readBytes));
if (packetTypeForPacket(packet) == PacketTypeSilentAudioFrame) {
// this source had no audio to send us, but this counts as a packet
// write silence equivalent to the number of silent samples they just sent us
int16_t numSilentSamples;
memcpy(&numSilentSamples, packet.data() + readBytes, sizeof(int16_t));
readBytes += sizeof(int16_t);
// add silent samples for the dropped packets as well.
// ASSUME that each dropped packet had same number of silent samples as this one
numSilentSamples *= (packetsSkipped + 1);
// NOTE: fixes a bug in old clients that would send garbage for their number of silentSamples
// CAN'T DO THIS because ScriptEngine.cpp sends frames of different size due to having a different sending interval
// (every 16.667ms) than Audio.cpp (every 10.667ms)
//numSilentSamples = getSamplesPerFrame();
addDroppableSilentSamples(numSilentSamples);
} else {
int numAudioBytes = packet.size() - readBytes;
int numAudioSamples = numAudioBytes / sizeof(int16_t);
// add silent samples for the dropped packets.
// ASSUME that each dropped packet had same number of samples as this one
if (packetsSkipped > 0) {
addDroppableSilentSamples(packetsSkipped * numAudioSamples);
}
// there is audio data to read
readBytes += writeData(packet.data() + readBytes, numAudioBytes);
}
return readBytes;
}

2
assignment-client/src/audio/AvatarAudioRingBuffer.h
View file

@ -20,7 +20,7 @@ class AvatarAudioRingBuffer : public PositionalAudioRingBuffer {
public:
AvatarAudioRingBuffer(bool isStereo = false, bool dynamicJitterBuffer = false);
int parseData(const QByteArray& packet);
int parseData(const QByteArray& packet, int packetsSkipped = 0);
private:
// disallow copying of AvatarAudioRingBuffer objects
AvatarAudioRingBuffer(const AvatarAudioRingBuffer&);

20
interface/src/Audio.cpp
View file

@ -56,6 +56,10 @@ static const int TIME_GAP_STATS_WINDOW_INTERVALS = 30;
static const int INCOMING_SEQ_STATS_HISTORY_LENGTH = INCOMING_SEQ_STATS_HISTORY_LENGTH_SECONDS /
(TOO_LONG_SINCE_LAST_SEND_DOWNSTREAM_AUDIO_STATS / USECS_PER_SECOND);
// the stats for the total frames available in the ring buffer and the audio output buffer
// will sample every second, update every second, and have a moving window covering 10 seconds
static const int FRAMES_AVAILABLE_STATS_WINDOW_SECONDS = 10;
// Mute icon configration
static const int MUTE_ICON_SIZE = 24;
@ -116,7 +120,8 @@ Audio::Audio(int16_t initialJitterBufferSamples, QObject* parent) :
_consecutiveNotMixedCount(0),
_outgoingAvatarAudioSequenceNumber(0),
_incomingMixedAudioSequenceNumberStats(INCOMING_SEQ_STATS_HISTORY_LENGTH),
_interframeTimeGapStats(TIME_GAPS_STATS_INTERVAL_SAMPLES, TIME_GAP_STATS_WINDOW_INTERVALS)
_interframeTimeGapStats(TIME_GAPS_STATS_INTERVAL_SAMPLES, TIME_GAP_STATS_WINDOW_INTERVALS),
_framesAvailableStats(1, FRAMES_AVAILABLE_STATS_WINDOW_SECONDS)
{
// clear the array of locally injected samples
memset(_localProceduralSamples, 0, NETWORK_BUFFER_LENGTH_BYTES_PER_CHANNEL);
@ -779,8 +784,8 @@ AudioStreamStats Audio::getDownstreamAudioStreamStats() const {
stats._timeGapWindowMax = _interframeTimeGapStats.getWindowMax();
stats._timeGapWindowAverage = _interframeTimeGapStats.getWindowAverage();
stats._ringBufferFramesAvailable = _ringBuffer.framesAvailable();
stats._ringBufferCurrentJitterBufferFrames = 0;
stats._ringBufferFramesAvailable = getFramesAvailableInRingAndAudioOutputBuffers();
stats._ringBufferFramesAvailableAverage = _framesAvailableStats.getWindowAverage();
stats._ringBufferDesiredJitterBufferFrames = getDesiredJitterBufferFrames();
stats._ringBufferStarveCount = _starveCount;
stats._ringBufferConsecutiveNotMixedCount = _consecutiveNotMixedCount;
@ -795,6 +800,9 @@ AudioStreamStats Audio::getDownstreamAudioStreamStats() const {
void Audio::sendDownstreamAudioStatsPacket() {
// since this function is called every second, we'll sample the number of audio frames available here.
_framesAvailableStats.update(getFramesAvailableInRingAndAudioOutputBuffers());
// push the current seq number stats into history, which moves the history window forward 1s
// (since that's how often pushStatsToHistory() is called)
_incomingMixedAudioSequenceNumberStats.pushStatsToHistory();
@ -1598,3 +1606,9 @@ int Audio::calculateNumberOfFrameSamples(int numBytes) {
int frameSamples = (int)(numBytes * CALLBACK_ACCELERATOR_RATIO + 0.5f) / sizeof(int16_t);
return frameSamples;
}
int Audio::getFramesAvailableInRingAndAudioOutputBuffers() const {
int framesInAudioOutputBuffer = (_audioOutput->bufferSize() - _audioOutput->bytesFree())
/ (sizeof(int16_t) * _ringBuffer.getNumFrameSamples());
return _ringBuffer.framesAvailable() + framesInAudioOutputBuffer;
}

3
interface/src/Audio.h
View file

@ -239,6 +239,8 @@ private:
void renderGrid(const float* color, int x, int y, int width, int height, int rows, int cols);
void renderLineStrip(const float* color, int x, int y, int n, int offset, const QByteArray* byteArray);
int getFramesAvailableInRingAndAudioOutputBuffers() const;
// Audio scope data
static const unsigned int NETWORK_SAMPLES_PER_FRAME = NETWORK_BUFFER_LENGTH_SAMPLES_PER_CHANNEL;
static const unsigned int DEFAULT_FRAMES_PER_SCOPE = 5;
@ -266,6 +268,7 @@ private:
SequenceNumberStats _incomingMixedAudioSequenceNumberStats;
MovingMinMaxAvg<quint64> _interframeTimeGapStats;
MovingMinMaxAvg<int> _framesAvailableStats;
};

6
interface/src/Hair.h
View file

@ -25,9 +25,9 @@ const int HAIR_CONSTRAINTS = 2;
const int DEFAULT_HAIR_STRANDS = 50;
const int DEFAULT_HAIR_LINKS = 10;
const float DEFAULT_HAIR_RADIUS = 0.15;
const float DEFAULT_HAIR_LINK_LENGTH = 0.03;
const float DEFAULT_HAIR_THICKNESS = 0.015;
const float DEFAULT_HAIR_RADIUS = 0.15f;
const float DEFAULT_HAIR_LINK_LENGTH = 0.03f;
const float DEFAULT_HAIR_THICKNESS = 0.015f;
class Hair {
public:

4
interface/src/ui/ChatWindow.cpp
View file

@ -30,8 +30,8 @@
const int NUM_MESSAGES_TO_TIME_STAMP = 20;
const float OPACITY_ACTIVE = 1.0;
const float OPACITY_INACTIVE = 0.8;
const float OPACITY_ACTIVE = 1.0f;
const float OPACITY_INACTIVE = 0.8f;
const QRegularExpression regexLinks("((?:(?:ftp)|(?:https?)|(?:hifi))://\\S+)");
const QRegularExpression regexHifiLinks("([#@]\\S+)");

19
interface/src/ui/Stats.cpp
View file

@ -315,10 +315,10 @@ void Stats::display(
drawText(horizontalOffset, verticalOffset, scale, rotation, font, voxelMaxPing, color);
char audioMixerStatsLabelString[] = "AudioMixer stats:";
char streamStatsFormatLabelString[] = "lost%/30s_lost%";
char streamStatsFormatLabelString2[] = "avail/currJ/desiredJ";
char streamStatsFormatLabelString[] = "lost%/lost_30s%";
char streamStatsFormatLabelString2[] = "desired/avail_avg_10s/avail";
char streamStatsFormatLabelString3[] = "gaps: min/max/avg, starv/ovfl";
char streamStatsFormatLabelString4[] = "30s gaps: (same), notmix/sdrop";
char streamStatsFormatLabelString4[] = "gaps_30s: (same), notmix/sdrop";
verticalOffset += STATS_PELS_PER_LINE;
drawText(horizontalOffset, verticalOffset, scale, rotation, font, audioMixerStatsLabelString, color);
@ -339,9 +339,10 @@ void Stats::display(
AudioStreamStats downstreamAudioStreamStats = audio->getDownstreamAudioStreamStats();
sprintf(downstreamAudioStatsString, " mix: %.2f%%/%.2f%%, %u/?/%u", downstreamAudioStreamStats._packetStreamStats.getLostRate()*100.0f,
sprintf(downstreamAudioStatsString, " mix: %.2f%%/%.2f%%, %u/%u/%u", downstreamAudioStreamStats._packetStreamStats.getLostRate()*100.0f,
downstreamAudioStreamStats._packetStreamWindowStats.getLostRate() * 100.0f,
downstreamAudioStreamStats._ringBufferFramesAvailable, downstreamAudioStreamStats._ringBufferDesiredJitterBufferFrames);
downstreamAudioStreamStats._ringBufferDesiredJitterBufferFrames, downstreamAudioStreamStats._ringBufferFramesAvailableAverage,
downstreamAudioStreamStats._ringBufferFramesAvailable);
verticalOffset += STATS_PELS_PER_LINE;
drawText(horizontalOffset, verticalOffset, scale, rotation, font, downstreamAudioStatsString, color);
@ -373,8 +374,8 @@ void Stats::display(
sprintf(upstreamAudioStatsString, " mic: %.2f%%/%.2f%%, %u/%u/%u", audioMixerAvatarAudioStreamStats._packetStreamStats.getLostRate()*100.0f,
audioMixerAvatarAudioStreamStats._packetStreamWindowStats.getLostRate() * 100.0f,
audioMixerAvatarAudioStreamStats._ringBufferFramesAvailable, audioMixerAvatarAudioStreamStats._ringBufferCurrentJitterBufferFrames,
audioMixerAvatarAudioStreamStats._ringBufferDesiredJitterBufferFrames);
audioMixerAvatarAudioStreamStats._ringBufferDesiredJitterBufferFrames, audioMixerAvatarAudioStreamStats._ringBufferFramesAvailableAverage,
audioMixerAvatarAudioStreamStats._ringBufferFramesAvailable);
verticalOffset += STATS_PELS_PER_LINE;
drawText(horizontalOffset, verticalOffset, scale, rotation, font, upstreamAudioStatsString, color);
@ -399,8 +400,8 @@ void Stats::display(
sprintf(upstreamAudioStatsString, " inj: %.2f%%/%.2f%%, %u/%u/%u", injectedStreamAudioStats._packetStreamStats.getLostRate()*100.0f,
injectedStreamAudioStats._packetStreamWindowStats.getLostRate() * 100.0f,
injectedStreamAudioStats._ringBufferFramesAvailable, injectedStreamAudioStats._ringBufferCurrentJitterBufferFrames,
injectedStreamAudioStats._ringBufferDesiredJitterBufferFrames);
injectedStreamAudioStats._ringBufferDesiredJitterBufferFrames, injectedStreamAudioStats._ringBufferFramesAvailableAverage,
injectedStreamAudioStats._ringBufferFramesAvailable);
verticalOffset += STATS_PELS_PER_LINE;
drawText(horizontalOffset, verticalOffset, scale, rotation, font, upstreamAudioStatsString, color);

4
libraries/audio/src/AudioStreamStats.h
View file

@ -27,7 +27,7 @@ public:
_timeGapWindowMax(0),
_timeGapWindowAverage(0.0f),
_ringBufferFramesAvailable(0),
_ringBufferCurrentJitterBufferFrames(0),
_ringBufferFramesAvailableAverage(0),
_ringBufferDesiredJitterBufferFrames(0),
_ringBufferStarveCount(0),
_ringBufferConsecutiveNotMixedCount(0),
@ -48,7 +48,7 @@ public:
float _timeGapWindowAverage;
quint32 _ringBufferFramesAvailable;
quint16 _ringBufferCurrentJitterBufferFrames;
quint16 _ringBufferFramesAvailableAverage;
quint16 _ringBufferDesiredJitterBufferFrames;
quint32 _ringBufferStarveCount;
quint32 _ringBufferConsecutiveNotMixedCount;

15
libraries/audio/src/InjectedAudioRingBuffer.cpp
View file

@ -30,9 +30,8 @@ InjectedAudioRingBuffer::InjectedAudioRingBuffer(const QUuid& streamIdentifier,
const uchar MAX_INJECTOR_VOLUME = 255;
int InjectedAudioRingBuffer::parseData(const QByteArray& packet) {
timeGapStatsFrameReceived();
updateDesiredJitterBufferFrames();
int InjectedAudioRingBuffer::parseData(const QByteArray& packet, int packetsSkipped) {
frameReceivedUpdateTimingStats();
// setup a data stream to read from this packet
QDataStream packetStream(packet);
@ -59,8 +58,14 @@ int InjectedAudioRingBuffer::parseData(const QByteArray& packet) {
packetStream >> attenuationByte;
_attenuationRatio = attenuationByte / (float) MAX_INJECTOR_VOLUME;
packetStream.skipRawData(writeData(packet.data() + packetStream.device()->pos(),
packet.size() - packetStream.device()->pos()));
int numAudioBytes = packet.size() - packetStream.device()->pos();
int numAudioSamples = numAudioBytes / sizeof(int16_t);
// add silent samples for the dropped packets.
// ASSUME that each dropped packet had same number of samples as this one
addDroppableSilentSamples(numAudioSamples * packetsSkipped);
packetStream.skipRawData(writeData(packet.data() + packetStream.device()->pos(), numAudioBytes));
return packetStream.device()->pos();
}

2
libraries/audio/src/InjectedAudioRingBuffer.h
View file

@ -20,7 +20,7 @@ class InjectedAudioRingBuffer : public PositionalAudioRingBuffer {
public:
InjectedAudioRingBuffer(const QUuid& streamIdentifier = QUuid(), bool dynamicJitterBuffer = false);
int parseData(const QByteArray& packet);
int parseData(const QByteArray& packet, int packetsSkipped = 0);
const QUuid& getStreamIdentifier() const { return _streamIdentifier; }
float getRadius() const { return _radius; }

131
libraries/audio/src/PositionalAudioRingBuffer.cpp
View file

@ -37,8 +37,8 @@ PositionalAudioRingBuffer::PositionalAudioRingBuffer(PositionalAudioRingBuffer::
_lastFrameReceivedTime(0),
_interframeTimeGapStatsForJitterCalc(TIME_GAPS_FOR_JITTER_CALC_INTERVAL_SAMPLES, TIME_GAPS_FOR_JITTER_CALC_WINDOW_INTERVALS),
_interframeTimeGapStatsForStatsPacket(TIME_GAPS_FOR_STATS_PACKET_INTERVAL_SAMPLES, TIME_GAPS_FOR_STATS_PACKET_WINDOW_INTERVALS),
_framesAvailableStats(FRAMES_AVAILABLE_STATS_INTERVAL_SAMPLES, FRAMES_AVAILABLE_STATS_WINDOW_INTERVALS),
_desiredJitterBufferFrames(1),
_currentJitterBufferFrames(-1),
_dynamicJitterBuffers(dynamicJitterBuffers),
_consecutiveNotMixedCount(0),
_starveCount(0),
@ -46,63 +46,6 @@ PositionalAudioRingBuffer::PositionalAudioRingBuffer(PositionalAudioRingBuffer::
{
}
int PositionalAudioRingBuffer::parseData(const QByteArray& packet) {
// skip the packet header (includes the source UUID)
int readBytes = numBytesForPacketHeader(packet);
// skip the sequence number
readBytes += sizeof(quint16);
// hop over the channel flag that has already been read in AudioMixerClientData
readBytes += sizeof(quint8);
// read the positional data
readBytes += parsePositionalData(packet.mid(readBytes));
if (packetTypeForPacket(packet) == PacketTypeSilentAudioFrame) {
// this source had no audio to send us, but this counts as a packet
// write silence equivalent to the number of silent samples they just sent us
int16_t numSilentSamples;
memcpy(&numSilentSamples, packet.data() + readBytes, sizeof(int16_t));
readBytes += sizeof(int16_t);
// NOTE: fixes a bug in old clients that would send garbage for their number of silentSamples
numSilentSamples = getSamplesPerFrame();
if (numSilentSamples > 0) {
if (_dynamicJitterBuffers && _currentJitterBufferFrames > _desiredJitterBufferFrames) {
// our current jitter buffer size exceeds its desired value, so ignore some silent
// frames to get that size as close to desired as possible
int samplesPerFrame = getSamplesPerFrame();
int numSilentFrames = numSilentSamples / samplesPerFrame;
int numFramesToDropDesired = _currentJitterBufferFrames - _desiredJitterBufferFrames;
if (numSilentFrames > numFramesToDropDesired) {
// we have more than enough frames to drop to get the jitter buffer to its desired length
int numSilentFramesToAdd = numSilentFrames - numFramesToDropDesired;
addSilentFrame(numSilentFramesToAdd * samplesPerFrame);
_currentJitterBufferFrames = _desiredJitterBufferFrames;
_silentFramesDropped += numFramesToDropDesired;
} else {
// we need to drop all frames to get the jitter buffer close as possible to its desired length
_currentJitterBufferFrames -= numSilentFrames;
_silentFramesDropped += numSilentFrames;
}
} else {
addSilentFrame(numSilentSamples);
}
}
} else {
// there is audio data to read
readBytes += writeData(packet.data() + readBytes, packet.size() - readBytes);
}
return readBytes;
}
int PositionalAudioRingBuffer::parsePositionalData(const QByteArray& positionalByteArray) {
QDataStream packetStream(positionalByteArray);
@ -147,10 +90,9 @@ void PositionalAudioRingBuffer::updateNextOutputTrailingLoudness() {
}
bool PositionalAudioRingBuffer::shouldBeAddedToMix() {
int samplesPerFrame = getSamplesPerFrame();
int desiredJitterBufferSamples = _desiredJitterBufferFrames * samplesPerFrame;
int desiredJitterBufferSamples = _desiredJitterBufferFrames * _numFrameSamples;
if (!isNotStarvedOrHasMinimumSamples(samplesPerFrame + desiredJitterBufferSamples)) {
if (!isNotStarvedOrHasMinimumSamples(_numFrameSamples + desiredJitterBufferSamples)) {
// if the buffer was starved, allow it to accrue at least the desired number of
// jitter buffer frames before we start taking frames from it for mixing
@ -160,13 +102,12 @@ bool PositionalAudioRingBuffer::shouldBeAddedToMix() {
_consecutiveNotMixedCount++;
return false;
} else if (samplesAvailable() < samplesPerFrame) {
} else if (samplesAvailable() < _numFrameSamples) {
// if the buffer doesn't have a full frame of samples to take for mixing, it is starved
_isStarved = true;
_starveCount++;
// set to -1 to indicate the jitter buffer is starved
_currentJitterBufferFrames = -1;
_framesAvailableStats.reset();
// reset our _shouldOutputStarveDebug to true so the next is printed
_shouldOutputStarveDebug = true;
@ -176,13 +117,14 @@ bool PositionalAudioRingBuffer::shouldBeAddedToMix() {
}
// good buffer, add this to the mix
if (_isStarved) {
// if this buffer has just finished replenishing after being starved, the number of frames in it now
// minus one (since a frame will be read immediately after this) is the length of the jitter buffer
_currentJitterBufferFrames = samplesAvailable() / samplesPerFrame - 1;
_isStarved = false;
}
// if we just finished refilling after a starve, we have a new jitter buffer length.
// reset the frames available stats.
_isStarved = false;
_framesAvailableStats.update(framesAvailable());
// since we've read data from ring buffer at least once - we've started
_hasStarted = true;
@ -190,17 +132,18 @@ bool PositionalAudioRingBuffer::shouldBeAddedToMix() {
}
int PositionalAudioRingBuffer::getCalculatedDesiredJitterBufferFrames() const {
int calculatedDesiredJitterBufferFrames = 1;
const float USECS_PER_FRAME = NETWORK_BUFFER_LENGTH_SAMPLES_PER_CHANNEL * USECS_PER_SECOND / (float)SAMPLE_RATE;
calculatedDesiredJitterBufferFrames = ceilf((float)_interframeTimeGapStatsForJitterCalc.getWindowMax() / USECS_PER_FRAME);
int calculatedDesiredJitterBufferFrames = ceilf((float)_interframeTimeGapStatsForJitterCalc.getWindowMax() / USECS_PER_FRAME);
if (calculatedDesiredJitterBufferFrames < 1) {
calculatedDesiredJitterBufferFrames = 1;
}
return calculatedDesiredJitterBufferFrames;
}
void PositionalAudioRingBuffer::timeGapStatsFrameReceived() {
void PositionalAudioRingBuffer::frameReceivedUpdateTimingStats() {
// update the two time gap stats we're keeping
quint64 now = usecTimestampNow();
if (_lastFrameReceivedTime != 0) {
quint64 gap = now - _lastFrameReceivedTime;
@ -208,15 +151,14 @@ void PositionalAudioRingBuffer::timeGapStatsFrameReceived() {
_interframeTimeGapStatsForStatsPacket.update(gap);
}
_lastFrameReceivedTime = now;
}
void PositionalAudioRingBuffer::updateDesiredJitterBufferFrames() {
// recalculate the _desiredJitterBufferFrames if _interframeTimeGapStatsForJitterCalc has updated stats for us
if (_interframeTimeGapStatsForJitterCalc.getNewStatsAvailableFlag()) {
if (!_dynamicJitterBuffers) {
_desiredJitterBufferFrames = 1; // HACK to see if this fixes the audio silence
} else {
const float USECS_PER_FRAME = NETWORK_BUFFER_LENGTH_SAMPLES_PER_CHANNEL * USECS_PER_SECOND / (float)SAMPLE_RATE;
_desiredJitterBufferFrames = ceilf((float)_interframeTimeGapStatsForJitterCalc.getWindowMax() / USECS_PER_FRAME);
if (_desiredJitterBufferFrames < 1) {
_desiredJitterBufferFrames = 1;
@ -229,3 +171,38 @@ void PositionalAudioRingBuffer::updateDesiredJitterBufferFrames() {
_interframeTimeGapStatsForJitterCalc.clearNewStatsAvailableFlag();
}
}
void PositionalAudioRingBuffer::addDroppableSilentSamples(int numSilentSamples) {
// This adds some number of frames to the desired jitter buffer frames target we use.
// The larger this value is, the less aggressive we are about reducing the jitter buffer length.
// Setting this to 0 will try to get the jitter buffer to be exactly _desiredJitterBufferFrames long,
// which could lead immediately to a starve.
const int DESIRED_JITTER_BUFFER_FRAMES_PADDING = 1;
// calculate how many silent frames we should drop. We only drop silent frames if
// the running avg num frames available has stabilized and it's more than
// our desired number of frames by the margin defined above.
int numSilentFramesToDrop = 0;
if (_framesAvailableStats.getNewStatsAvailableFlag() && _framesAvailableStats.isWindowFilled()
&& numSilentSamples >= _numFrameSamples) {
_framesAvailableStats.clearNewStatsAvailableFlag();
int averageJitterBufferFrames = (int)_framesAvailableStats.getWindowAverage();
int desiredJitterBufferFramesPlusPadding = _desiredJitterBufferFrames + DESIRED_JITTER_BUFFER_FRAMES_PADDING;
if (averageJitterBufferFrames > desiredJitterBufferFramesPlusPadding) {
// our avg jitter buffer size exceeds its desired value, so ignore some silent
// frames to get that size as close to desired as possible
int numSilentFramesToDropDesired = averageJitterBufferFrames - desiredJitterBufferFramesPlusPadding;
int numSilentFramesReceived = numSilentSamples / _numFrameSamples;
numSilentFramesToDrop = std::min(numSilentFramesToDropDesired, numSilentFramesReceived);
// since we now have a new jitter buffer length, reset the frames available stats.
_framesAvailableStats.reset();
_silentFramesDropped += numSilentFramesToDrop;
}
}
addSilentFrame(numSilentSamples - numSilentFramesToDrop * _numFrameSamples);
}

16
libraries/audio/src/PositionalAudioRingBuffer.h
View file

@ -29,6 +29,11 @@ const int TIME_GAPS_FOR_JITTER_CALC_WINDOW_INTERVALS = 10;
const int TIME_GAPS_FOR_STATS_PACKET_INTERVAL_SAMPLES = USECS_PER_SECOND / BUFFER_SEND_INTERVAL_USECS;
const int TIME_GAPS_FOR_STATS_PACKET_WINDOW_INTERVALS = 30;
// the stats for calculating the average frames available will recalculate every ~1 second
// and will include data for the past ~2 seconds
const int FRAMES_AVAILABLE_STATS_INTERVAL_SAMPLES = USECS_PER_SECOND / BUFFER_SEND_INTERVAL_USECS;
const int FRAMES_AVAILABLE_STATS_WINDOW_INTERVALS = 2;
const int AUDIOMIXER_INBOUND_RING_BUFFER_FRAME_CAPACITY = 100;
class PositionalAudioRingBuffer : public AudioRingBuffer {
@ -40,7 +45,8 @@ public:
PositionalAudioRingBuffer(PositionalAudioRingBuffer::Type type, bool isStereo = false, bool dynamicJitterBuffers = false);
int parseData(const QByteArray& packet);
virtual int parseData(const QByteArray& packet, int packetsSkipped = 0) = 0;
int parsePositionalData(const QByteArray& positionalByteArray);
int parseListenModeData(const QByteArray& listenModeByteArray);
@ -69,7 +75,7 @@ public:
int getCalculatedDesiredJitterBufferFrames() const; /// returns what we would calculate our desired as if asked
int getDesiredJitterBufferFrames() const { return _desiredJitterBufferFrames; }
int getCurrentJitterBufferFrames() const { return _currentJitterBufferFrames; }
double getFramesAvailableAverage() const { return _framesAvailableStats.getWindowAverage(); }
int getConsecutiveNotMixedCount() const { return _consecutiveNotMixedCount; }
int getStarveCount() const { return _starveCount; }
@ -80,8 +86,8 @@ protected:
PositionalAudioRingBuffer(const PositionalAudioRingBuffer&);
PositionalAudioRingBuffer& operator= (const PositionalAudioRingBuffer&);
void timeGapStatsFrameReceived();
void updateDesiredJitterBufferFrames();
void frameReceivedUpdateTimingStats();
void addDroppableSilentSamples(int numSilentSamples);
PositionalAudioRingBuffer::Type _type;
glm::vec3 _position;
@ -97,9 +103,9 @@ protected:
quint64 _lastFrameReceivedTime;
MovingMinMaxAvg<quint64> _interframeTimeGapStatsForJitterCalc;
MovingMinMaxAvg<quint64> _interframeTimeGapStatsForStatsPacket;
MovingMinMaxAvg<int> _framesAvailableStats;
int _desiredJitterBufferFrames;
int _currentJitterBufferFrames;
bool _dynamicJitterBuffers;
// extra stats

25
libraries/networking/src/SequenceNumberStats.cpp
View file

@ -30,7 +30,9 @@ void SequenceNumberStats::reset() {
static const int UINT16_RANGE = std::numeric_limits<uint16_t>::max() + 1;
void SequenceNumberStats::sequenceNumberReceived(quint16 incoming, QUuid senderUUID, const bool wantExtraDebugging) {
SequenceNumberStats::ArrivalInfo SequenceNumberStats::sequenceNumberReceived(quint16 incoming, QUuid senderUUID, const bool wantExtraDebugging) {
SequenceNumberStats::ArrivalInfo arrivalInfo;
// if the sender node has changed, reset all stats
if (senderUUID != _lastSenderUUID) {
@ -46,6 +48,7 @@ void SequenceNumberStats::sequenceNumberReceived(quint16 incoming, QUuid senderU
_stats._numReceived++;
if (incoming == expected) { // on time
arrivalInfo._status = OnTime;
_lastReceived = incoming;
} else { // out of order
@ -67,21 +70,29 @@ void SequenceNumberStats::sequenceNumberReceived(quint16 incoming, QUuid senderU
expectedInt -= UINT16_RANGE;
}
} else if (absGap > MAX_REASONABLE_SEQUENCE_GAP) {
arrivalInfo._status = Unreasonable;
// ignore packet if gap is unreasonable
qDebug() << "ignoring unreasonable sequence number:" << incoming
<< "previous:" << _lastReceived;
_stats._numUnreasonable++;
return;
return arrivalInfo;
}
// now that rollover has been corrected for (if it occurred), incoming and expected can be
// now that rollover has been corrected for (if it occurred), incomingInt and expectedInt can be
// compared to each other directly, though one of them might be negative
arrivalInfo._seqDiffFromExpected = incomingInt - expectedInt;
if (incomingInt > expectedInt) { // early
arrivalInfo._status = Early;
if (wantExtraDebugging) {
qDebug() << "this packet is earlier than expected...";
qDebug() << ">>>>>>>> missing gap=" << (incomingInt - expectedInt);
}
_stats._numEarly++;
_stats._numLost += (incomingInt - expectedInt);
_lastReceived = incoming;
@ -100,18 +111,23 @@ void SequenceNumberStats::sequenceNumberReceived(quint16 incoming, QUuid senderU
if (wantExtraDebugging) {
qDebug() << "this packet is later than expected...";
}
_stats._numLate++;
// do not update _lastReceived; it shouldn't become smaller
// remove this from missing sequence number if it's in there
if (_missingSet.remove(incoming)) {
arrivalInfo._status = Late;
if (wantExtraDebugging) {
qDebug() << "found it in _missingSet";
}
_stats._numLost--;
_stats._numRecovered++;
} else {
arrivalInfo._status = Duplicate;
if (wantExtraDebugging) {
qDebug() << "sequence:" << incoming << "was NOT found in _missingSet and is probably a duplicate";
}
@ -119,6 +135,7 @@ void SequenceNumberStats::sequenceNumberReceived(quint16 incoming, QUuid senderU
}
}
}
return arrivalInfo;
}
void SequenceNumberStats::pruneMissingSet(const bool wantExtraDebugging) {

17
libraries/networking/src/SequenceNumberStats.h
View file

@ -48,10 +48,25 @@ public:
class SequenceNumberStats {
public:
enum ArrivalStatus {
OnTime,
Unreasonable,
Early,
Late, // recovered
Duplicate
};
class ArrivalInfo {
public:
ArrivalStatus _status;
int _seqDiffFromExpected;
};
SequenceNumberStats(int statsHistoryLength = 0);
void reset();
void sequenceNumberReceived(quint16 incoming, QUuid senderUUID = QUuid(), const bool wantExtraDebugging = false);
ArrivalInfo sequenceNumberReceived(quint16 incoming, QUuid senderUUID = QUuid(), const bool wantExtraDebugging = false);
void pruneMissingSet(const bool wantExtraDebugging = false);
void pushStatsToHistory() { _statsHistory.insert(_stats); }

2
libraries/shared/src/MovingMinMaxAvg.h
View file

@ -126,6 +126,8 @@ public:
T getWindowMax() const { return _windowStats._max; }
double getWindowAverage() const { return _windowStats._average; }
bool isWindowFilled() const { return _intervalStats.isFilled(); }
private:
int _intervalLength;
int _windowIntervals;

1
libraries/shared/src/RingBufferHistory.h
View file

@ -73,6 +73,7 @@ public:
int getCapacity() const { return _capacity; }
int getNumEntries() const { return _numEntries; }
bool isFilled() const { return _numEntries == _capacity; }
private:
int _size;