diff --git a/assignment-client/src/audio/AudioMixerClientData.cpp b/assignment-client/src/audio/AudioMixerClientData.cpp index 6559b57959..b42a690d97 100644 --- a/assignment-client/src/audio/AudioMixerClientData.cpp +++ b/assignment-client/src/audio/AudioMixerClientData.cpp @@ -61,7 +61,7 @@ int AudioMixerClientData::parseData(const QByteArray& packet) { || packetType == PacketTypeMicrophoneAudioNoEcho || packetType == PacketTypeSilentAudioFrame) { - _incomingAvatarAudioSequenceNumberStats.sequenceNumberReceived(sequence); + SequenceNumberStats::ArrivalInfo packetArrivalInfo = _incomingAvatarAudioSequenceNumberStats.sequenceNumberReceived(sequence); // grab the AvatarAudioRingBuffer from the vector (or create it if it doesn't exist) AvatarAudioRingBuffer* avatarRingBuffer = getAvatarAudioRingBuffer(); @@ -84,8 +84,24 @@ int AudioMixerClientData::parseData(const QByteArray& packet) { _ringBuffers.push_back(avatarRingBuffer); } - // ask the AvatarAudioRingBuffer instance to parse the data - avatarRingBuffer->parseData(packet); + + // for now, late packets are simply discarded. In the future, it may be good to insert them into their correct place + // in the ring buffer (if that frame hasn't been mixed yet) + switch (packetArrivalInfo._status) { + case SequenceNumberStats::Early: { + int packetsLost = packetArrivalInfo._seqDiffFromExpected; + avatarRingBuffer->parseData(packet, packetsLost); + break; + } + case SequenceNumberStats::OnTime: { + // ask the AvatarAudioRingBuffer instance to parse the data + avatarRingBuffer->parseData(packet); + break; + } + default: { + break; + } + } } else if (packetType == PacketTypeInjectAudio) { // this is injected audio @@ -95,7 +111,8 @@ int AudioMixerClientData::parseData(const QByteArray& packet) { if (!_incomingInjectedAudioSequenceNumberStatsMap.contains(streamIdentifier)) { _incomingInjectedAudioSequenceNumberStatsMap.insert(streamIdentifier, SequenceNumberStats(INCOMING_SEQ_STATS_HISTORY_LENGTH)); } - _incomingInjectedAudioSequenceNumberStatsMap[streamIdentifier].sequenceNumberReceived(sequence); + SequenceNumberStats::ArrivalInfo packetArrivalInfo = + _incomingInjectedAudioSequenceNumberStatsMap[streamIdentifier].sequenceNumberReceived(sequence); InjectedAudioRingBuffer* matchingInjectedRingBuffer = NULL; @@ -112,7 +129,23 @@ int AudioMixerClientData::parseData(const QByteArray& packet) { _ringBuffers.push_back(matchingInjectedRingBuffer); } - matchingInjectedRingBuffer->parseData(packet); + // for now, late packets are simply discarded. In the future, it may be good to insert them into their correct place + // in the ring buffer (if that frame hasn't been mixed yet) + switch (packetArrivalInfo._status) { + case SequenceNumberStats::Early: { + int packetsLost = packetArrivalInfo._seqDiffFromExpected; + matchingInjectedRingBuffer->parseData(packet, packetsLost); + break; + } + case SequenceNumberStats::OnTime: { + // ask the AvatarAudioRingBuffer instance to parse the data + matchingInjectedRingBuffer->parseData(packet); + break; + } + default: { + break; + } + } } else if (packetType == PacketTypeAudioStreamStats) { const char* dataAt = packet.data(); @@ -198,7 +231,7 @@ AudioStreamStats AudioMixerClientData::getAudioStreamStatsOfStream(const Positio streamStats._timeGapWindowAverage = timeGapStats.getWindowAverage(); streamStats._ringBufferFramesAvailable = ringBuffer->framesAvailable(); - streamStats._ringBufferCurrentJitterBufferFrames = ringBuffer->getCurrentJitterBufferFrames(); + streamStats._ringBufferFramesAvailableAverage = ringBuffer->getFramesAvailableAverage(); streamStats._ringBufferDesiredJitterBufferFrames = ringBuffer->getDesiredJitterBufferFrames(); streamStats._ringBufferStarveCount = ringBuffer->getStarveCount(); streamStats._ringBufferConsecutiveNotMixedCount = ringBuffer->getConsecutiveNotMixedCount(); @@ -272,7 +305,7 @@ QString AudioMixerClientData::getAudioStreamStatsString() const { QString result; AudioStreamStats streamStats = _downstreamAudioStreamStats; result += "DOWNSTREAM.desired:" + QString::number(streamStats._ringBufferDesiredJitterBufferFrames) - + " current: ?" + + " available_avg_10s:" + QString::number(streamStats._ringBufferFramesAvailableAverage) + " available:" + QString::number(streamStats._ringBufferFramesAvailable) + " starves:" + QString::number(streamStats._ringBufferStarveCount) + " not_mixed:" + QString::number(streamStats._ringBufferConsecutiveNotMixedCount) @@ -291,7 +324,8 @@ QString AudioMixerClientData::getAudioStreamStatsString() const { if (avatarRingBuffer) { AudioStreamStats streamStats = getAudioStreamStatsOfStream(avatarRingBuffer); result += " UPSTREAM.mic.desired:" + QString::number(streamStats._ringBufferDesiredJitterBufferFrames) - + " current:" + QString::number(streamStats._ringBufferCurrentJitterBufferFrames) + + " desired_calc:" + QString::number(avatarRingBuffer->getCalculatedDesiredJitterBufferFrames()) + + " available_avg_10s:" + QString::number(streamStats._ringBufferFramesAvailableAverage) + " available:" + QString::number(streamStats._ringBufferFramesAvailable) + " starves:" + QString::number(streamStats._ringBufferStarveCount) + " not_mixed:" + QString::number(streamStats._ringBufferConsecutiveNotMixedCount) @@ -313,7 +347,8 @@ QString AudioMixerClientData::getAudioStreamStatsString() const { if (_ringBuffers[i]->getType() == PositionalAudioRingBuffer::Injector) { AudioStreamStats streamStats = getAudioStreamStatsOfStream(_ringBuffers[i]); result += " UPSTREAM.inj.desired:" + QString::number(streamStats._ringBufferDesiredJitterBufferFrames) - + " current:" + QString::number(streamStats._ringBufferCurrentJitterBufferFrames) + + " desired_calc:" + QString::number(_ringBuffers[i]->getCalculatedDesiredJitterBufferFrames()) + + " available_avg_10s:" + QString::number(streamStats._ringBufferFramesAvailableAverage) + " available:" + QString::number(streamStats._ringBufferFramesAvailable) + " starves:" + QString::number(streamStats._ringBufferStarveCount) + " not_mixed:" + QString::number(streamStats._ringBufferConsecutiveNotMixedCount) diff --git a/assignment-client/src/audio/AvatarAudioRingBuffer.cpp b/assignment-client/src/audio/AvatarAudioRingBuffer.cpp index 3fa9f64cff..f6edde7ac4 100644 --- a/assignment-client/src/audio/AvatarAudioRingBuffer.cpp +++ b/assignment-client/src/audio/AvatarAudioRingBuffer.cpp @@ -18,10 +18,53 @@ AvatarAudioRingBuffer::AvatarAudioRingBuffer(bool isStereo, bool dynamicJitterBu } -int AvatarAudioRingBuffer::parseData(const QByteArray& packet) { - timeGapStatsFrameReceived(); - updateDesiredJitterBufferFrames(); +int AvatarAudioRingBuffer::parseData(const QByteArray& packet, int packetsSkipped) { + frameReceivedUpdateTimingStats(); _shouldLoopbackForNode = (packetTypeForPacket(packet) == PacketTypeMicrophoneAudioWithEcho); - return PositionalAudioRingBuffer::parseData(packet); + + // skip the packet header (includes the source UUID) + int readBytes = numBytesForPacketHeader(packet); + + // skip the sequence number + readBytes += sizeof(quint16); + + // hop over the channel flag that has already been read in AudioMixerClientData + readBytes += sizeof(quint8); + // read the positional data + readBytes += parsePositionalData(packet.mid(readBytes)); + + if (packetTypeForPacket(packet) == PacketTypeSilentAudioFrame) { + // this source had no audio to send us, but this counts as a packet + // write silence equivalent to the number of silent samples they just sent us + int16_t numSilentSamples; + + memcpy(&numSilentSamples, packet.data() + readBytes, sizeof(int16_t)); + readBytes += sizeof(int16_t); + + // add silent samples for the dropped packets as well. + // ASSUME that each dropped packet had same number of silent samples as this one + numSilentSamples *= (packetsSkipped + 1); + + // NOTE: fixes a bug in old clients that would send garbage for their number of silentSamples + // CAN'T DO THIS because ScriptEngine.cpp sends frames of different size due to having a different sending interval + // (every 16.667ms) than Audio.cpp (every 10.667ms) + //numSilentSamples = getSamplesPerFrame(); + + addDroppableSilentSamples(numSilentSamples); + + } else { + int numAudioBytes = packet.size() - readBytes; + int numAudioSamples = numAudioBytes / sizeof(int16_t); + + // add silent samples for the dropped packets. + // ASSUME that each dropped packet had same number of samples as this one + if (packetsSkipped > 0) { + addDroppableSilentSamples(packetsSkipped * numAudioSamples); + } + + // there is audio data to read + readBytes += writeData(packet.data() + readBytes, numAudioBytes); + } + return readBytes; } diff --git a/assignment-client/src/audio/AvatarAudioRingBuffer.h b/assignment-client/src/audio/AvatarAudioRingBuffer.h index e227e70958..df9dc3787a 100644 --- a/assignment-client/src/audio/AvatarAudioRingBuffer.h +++ b/assignment-client/src/audio/AvatarAudioRingBuffer.h @@ -20,7 +20,7 @@ class AvatarAudioRingBuffer : public PositionalAudioRingBuffer { public: AvatarAudioRingBuffer(bool isStereo = false, bool dynamicJitterBuffer = false); - int parseData(const QByteArray& packet); + int parseData(const QByteArray& packet, int packetsSkipped = 0); private: // disallow copying of AvatarAudioRingBuffer objects AvatarAudioRingBuffer(const AvatarAudioRingBuffer&); diff --git a/interface/src/Audio.cpp b/interface/src/Audio.cpp index 6a26563e36..b728b740b7 100644 --- a/interface/src/Audio.cpp +++ b/interface/src/Audio.cpp @@ -56,6 +56,10 @@ static const int TIME_GAP_STATS_WINDOW_INTERVALS = 30; static const int INCOMING_SEQ_STATS_HISTORY_LENGTH = INCOMING_SEQ_STATS_HISTORY_LENGTH_SECONDS / (TOO_LONG_SINCE_LAST_SEND_DOWNSTREAM_AUDIO_STATS / USECS_PER_SECOND); +// the stats for the total frames available in the ring buffer and the audio output buffer +// will sample every second, update every second, and have a moving window covering 10 seconds +static const int FRAMES_AVAILABLE_STATS_WINDOW_SECONDS = 10; + // Mute icon configration static const int MUTE_ICON_SIZE = 24; @@ -116,7 +120,8 @@ Audio::Audio(int16_t initialJitterBufferSamples, QObject* parent) : _consecutiveNotMixedCount(0), _outgoingAvatarAudioSequenceNumber(0), _incomingMixedAudioSequenceNumberStats(INCOMING_SEQ_STATS_HISTORY_LENGTH), - _interframeTimeGapStats(TIME_GAPS_STATS_INTERVAL_SAMPLES, TIME_GAP_STATS_WINDOW_INTERVALS) + _interframeTimeGapStats(TIME_GAPS_STATS_INTERVAL_SAMPLES, TIME_GAP_STATS_WINDOW_INTERVALS), + _framesAvailableStats(1, FRAMES_AVAILABLE_STATS_WINDOW_SECONDS) { // clear the array of locally injected samples memset(_localProceduralSamples, 0, NETWORK_BUFFER_LENGTH_BYTES_PER_CHANNEL); @@ -779,8 +784,8 @@ AudioStreamStats Audio::getDownstreamAudioStreamStats() const { stats._timeGapWindowMax = _interframeTimeGapStats.getWindowMax(); stats._timeGapWindowAverage = _interframeTimeGapStats.getWindowAverage(); - stats._ringBufferFramesAvailable = _ringBuffer.framesAvailable(); - stats._ringBufferCurrentJitterBufferFrames = 0; + stats._ringBufferFramesAvailable = getFramesAvailableInRingAndAudioOutputBuffers(); + stats._ringBufferFramesAvailableAverage = _framesAvailableStats.getWindowAverage(); stats._ringBufferDesiredJitterBufferFrames = getDesiredJitterBufferFrames(); stats._ringBufferStarveCount = _starveCount; stats._ringBufferConsecutiveNotMixedCount = _consecutiveNotMixedCount; @@ -795,6 +800,9 @@ AudioStreamStats Audio::getDownstreamAudioStreamStats() const { void Audio::sendDownstreamAudioStatsPacket() { + // since this function is called every second, we'll sample the number of audio frames available here. + _framesAvailableStats.update(getFramesAvailableInRingAndAudioOutputBuffers()); + // push the current seq number stats into history, which moves the history window forward 1s // (since that's how often pushStatsToHistory() is called) _incomingMixedAudioSequenceNumberStats.pushStatsToHistory(); @@ -1598,3 +1606,9 @@ int Audio::calculateNumberOfFrameSamples(int numBytes) { int frameSamples = (int)(numBytes * CALLBACK_ACCELERATOR_RATIO + 0.5f) / sizeof(int16_t); return frameSamples; } + +int Audio::getFramesAvailableInRingAndAudioOutputBuffers() const { + int framesInAudioOutputBuffer = (_audioOutput->bufferSize() - _audioOutput->bytesFree()) + / (sizeof(int16_t) * _ringBuffer.getNumFrameSamples()); + return _ringBuffer.framesAvailable() + framesInAudioOutputBuffer; +} diff --git a/interface/src/Audio.h b/interface/src/Audio.h index b40355e714..0747bdf501 100644 --- a/interface/src/Audio.h +++ b/interface/src/Audio.h @@ -239,6 +239,8 @@ private: void renderGrid(const float* color, int x, int y, int width, int height, int rows, int cols); void renderLineStrip(const float* color, int x, int y, int n, int offset, const QByteArray* byteArray); + int getFramesAvailableInRingAndAudioOutputBuffers() const; + // Audio scope data static const unsigned int NETWORK_SAMPLES_PER_FRAME = NETWORK_BUFFER_LENGTH_SAMPLES_PER_CHANNEL; static const unsigned int DEFAULT_FRAMES_PER_SCOPE = 5; @@ -266,6 +268,7 @@ private: SequenceNumberStats _incomingMixedAudioSequenceNumberStats; MovingMinMaxAvg _interframeTimeGapStats; + MovingMinMaxAvg _framesAvailableStats; }; diff --git a/interface/src/Hair.h b/interface/src/Hair.h index 0ca507ee23..a54dd5166c 100644 --- a/interface/src/Hair.h +++ b/interface/src/Hair.h @@ -25,9 +25,9 @@ const int HAIR_CONSTRAINTS = 2; const int DEFAULT_HAIR_STRANDS = 50; const int DEFAULT_HAIR_LINKS = 10; -const float DEFAULT_HAIR_RADIUS = 0.15; -const float DEFAULT_HAIR_LINK_LENGTH = 0.03; -const float DEFAULT_HAIR_THICKNESS = 0.015; +const float DEFAULT_HAIR_RADIUS = 0.15f; +const float DEFAULT_HAIR_LINK_LENGTH = 0.03f; +const float DEFAULT_HAIR_THICKNESS = 0.015f; class Hair { public: diff --git a/interface/src/ui/ChatWindow.cpp b/interface/src/ui/ChatWindow.cpp index 23bf2eafc4..8874f18dba 100644 --- a/interface/src/ui/ChatWindow.cpp +++ b/interface/src/ui/ChatWindow.cpp @@ -30,8 +30,8 @@ const int NUM_MESSAGES_TO_TIME_STAMP = 20; -const float OPACITY_ACTIVE = 1.0; -const float OPACITY_INACTIVE = 0.8; +const float OPACITY_ACTIVE = 1.0f; +const float OPACITY_INACTIVE = 0.8f; const QRegularExpression regexLinks("((?:(?:ftp)|(?:https?)|(?:hifi))://\\S+)"); const QRegularExpression regexHifiLinks("([#@]\\S+)"); diff --git a/interface/src/ui/Stats.cpp b/interface/src/ui/Stats.cpp index 8789a332c9..a751f43059 100644 --- a/interface/src/ui/Stats.cpp +++ b/interface/src/ui/Stats.cpp @@ -315,10 +315,10 @@ void Stats::display( drawText(horizontalOffset, verticalOffset, scale, rotation, font, voxelMaxPing, color); char audioMixerStatsLabelString[] = "AudioMixer stats:"; - char streamStatsFormatLabelString[] = "lost%/30s_lost%"; - char streamStatsFormatLabelString2[] = "avail/currJ/desiredJ"; + char streamStatsFormatLabelString[] = "lost%/lost_30s%"; + char streamStatsFormatLabelString2[] = "desired/avail_avg_10s/avail"; char streamStatsFormatLabelString3[] = "gaps: min/max/avg, starv/ovfl"; - char streamStatsFormatLabelString4[] = "30s gaps: (same), notmix/sdrop"; + char streamStatsFormatLabelString4[] = "gaps_30s: (same), notmix/sdrop"; verticalOffset += STATS_PELS_PER_LINE; drawText(horizontalOffset, verticalOffset, scale, rotation, font, audioMixerStatsLabelString, color); @@ -339,9 +339,10 @@ void Stats::display( AudioStreamStats downstreamAudioStreamStats = audio->getDownstreamAudioStreamStats(); - sprintf(downstreamAudioStatsString, " mix: %.2f%%/%.2f%%, %u/?/%u", downstreamAudioStreamStats._packetStreamStats.getLostRate()*100.0f, + sprintf(downstreamAudioStatsString, " mix: %.2f%%/%.2f%%, %u/%u/%u", downstreamAudioStreamStats._packetStreamStats.getLostRate()*100.0f, downstreamAudioStreamStats._packetStreamWindowStats.getLostRate() * 100.0f, - downstreamAudioStreamStats._ringBufferFramesAvailable, downstreamAudioStreamStats._ringBufferDesiredJitterBufferFrames); + downstreamAudioStreamStats._ringBufferDesiredJitterBufferFrames, downstreamAudioStreamStats._ringBufferFramesAvailableAverage, + downstreamAudioStreamStats._ringBufferFramesAvailable); verticalOffset += STATS_PELS_PER_LINE; drawText(horizontalOffset, verticalOffset, scale, rotation, font, downstreamAudioStatsString, color); @@ -373,8 +374,8 @@ void Stats::display( sprintf(upstreamAudioStatsString, " mic: %.2f%%/%.2f%%, %u/%u/%u", audioMixerAvatarAudioStreamStats._packetStreamStats.getLostRate()*100.0f, audioMixerAvatarAudioStreamStats._packetStreamWindowStats.getLostRate() * 100.0f, - audioMixerAvatarAudioStreamStats._ringBufferFramesAvailable, audioMixerAvatarAudioStreamStats._ringBufferCurrentJitterBufferFrames, - audioMixerAvatarAudioStreamStats._ringBufferDesiredJitterBufferFrames); + audioMixerAvatarAudioStreamStats._ringBufferDesiredJitterBufferFrames, audioMixerAvatarAudioStreamStats._ringBufferFramesAvailableAverage, + audioMixerAvatarAudioStreamStats._ringBufferFramesAvailable); verticalOffset += STATS_PELS_PER_LINE; drawText(horizontalOffset, verticalOffset, scale, rotation, font, upstreamAudioStatsString, color); @@ -399,8 +400,8 @@ void Stats::display( sprintf(upstreamAudioStatsString, " inj: %.2f%%/%.2f%%, %u/%u/%u", injectedStreamAudioStats._packetStreamStats.getLostRate()*100.0f, injectedStreamAudioStats._packetStreamWindowStats.getLostRate() * 100.0f, - injectedStreamAudioStats._ringBufferFramesAvailable, injectedStreamAudioStats._ringBufferCurrentJitterBufferFrames, - injectedStreamAudioStats._ringBufferDesiredJitterBufferFrames); + injectedStreamAudioStats._ringBufferDesiredJitterBufferFrames, injectedStreamAudioStats._ringBufferFramesAvailableAverage, + injectedStreamAudioStats._ringBufferFramesAvailable); verticalOffset += STATS_PELS_PER_LINE; drawText(horizontalOffset, verticalOffset, scale, rotation, font, upstreamAudioStatsString, color); diff --git a/libraries/audio/src/AudioStreamStats.h b/libraries/audio/src/AudioStreamStats.h index f26a3847c7..f73cbd3e54 100644 --- a/libraries/audio/src/AudioStreamStats.h +++ b/libraries/audio/src/AudioStreamStats.h @@ -27,7 +27,7 @@ public: _timeGapWindowMax(0), _timeGapWindowAverage(0.0f), _ringBufferFramesAvailable(0), - _ringBufferCurrentJitterBufferFrames(0), + _ringBufferFramesAvailableAverage(0), _ringBufferDesiredJitterBufferFrames(0), _ringBufferStarveCount(0), _ringBufferConsecutiveNotMixedCount(0), @@ -48,7 +48,7 @@ public: float _timeGapWindowAverage; quint32 _ringBufferFramesAvailable; - quint16 _ringBufferCurrentJitterBufferFrames; + quint16 _ringBufferFramesAvailableAverage; quint16 _ringBufferDesiredJitterBufferFrames; quint32 _ringBufferStarveCount; quint32 _ringBufferConsecutiveNotMixedCount; diff --git a/libraries/audio/src/InjectedAudioRingBuffer.cpp b/libraries/audio/src/InjectedAudioRingBuffer.cpp index bdad7b2a7a..da2d8336de 100644 --- a/libraries/audio/src/InjectedAudioRingBuffer.cpp +++ b/libraries/audio/src/InjectedAudioRingBuffer.cpp @@ -30,9 +30,8 @@ InjectedAudioRingBuffer::InjectedAudioRingBuffer(const QUuid& streamIdentifier, const uchar MAX_INJECTOR_VOLUME = 255; -int InjectedAudioRingBuffer::parseData(const QByteArray& packet) { - timeGapStatsFrameReceived(); - updateDesiredJitterBufferFrames(); +int InjectedAudioRingBuffer::parseData(const QByteArray& packet, int packetsSkipped) { + frameReceivedUpdateTimingStats(); // setup a data stream to read from this packet QDataStream packetStream(packet); @@ -59,8 +58,14 @@ int InjectedAudioRingBuffer::parseData(const QByteArray& packet) { packetStream >> attenuationByte; _attenuationRatio = attenuationByte / (float) MAX_INJECTOR_VOLUME; - packetStream.skipRawData(writeData(packet.data() + packetStream.device()->pos(), - packet.size() - packetStream.device()->pos())); + int numAudioBytes = packet.size() - packetStream.device()->pos(); + int numAudioSamples = numAudioBytes / sizeof(int16_t); + + // add silent samples for the dropped packets. + // ASSUME that each dropped packet had same number of samples as this one + addDroppableSilentSamples(numAudioSamples * packetsSkipped); + + packetStream.skipRawData(writeData(packet.data() + packetStream.device()->pos(), numAudioBytes)); return packetStream.device()->pos(); } diff --git a/libraries/audio/src/InjectedAudioRingBuffer.h b/libraries/audio/src/InjectedAudioRingBuffer.h index 4e3fea672b..f226d99b12 100644 --- a/libraries/audio/src/InjectedAudioRingBuffer.h +++ b/libraries/audio/src/InjectedAudioRingBuffer.h @@ -20,7 +20,7 @@ class InjectedAudioRingBuffer : public PositionalAudioRingBuffer { public: InjectedAudioRingBuffer(const QUuid& streamIdentifier = QUuid(), bool dynamicJitterBuffer = false); - int parseData(const QByteArray& packet); + int parseData(const QByteArray& packet, int packetsSkipped = 0); const QUuid& getStreamIdentifier() const { return _streamIdentifier; } float getRadius() const { return _radius; } diff --git a/libraries/audio/src/PositionalAudioRingBuffer.cpp b/libraries/audio/src/PositionalAudioRingBuffer.cpp index 6c7ecd1cad..14af8c6a4a 100644 --- a/libraries/audio/src/PositionalAudioRingBuffer.cpp +++ b/libraries/audio/src/PositionalAudioRingBuffer.cpp @@ -37,8 +37,8 @@ PositionalAudioRingBuffer::PositionalAudioRingBuffer(PositionalAudioRingBuffer:: _lastFrameReceivedTime(0), _interframeTimeGapStatsForJitterCalc(TIME_GAPS_FOR_JITTER_CALC_INTERVAL_SAMPLES, TIME_GAPS_FOR_JITTER_CALC_WINDOW_INTERVALS), _interframeTimeGapStatsForStatsPacket(TIME_GAPS_FOR_STATS_PACKET_INTERVAL_SAMPLES, TIME_GAPS_FOR_STATS_PACKET_WINDOW_INTERVALS), + _framesAvailableStats(FRAMES_AVAILABLE_STATS_INTERVAL_SAMPLES, FRAMES_AVAILABLE_STATS_WINDOW_INTERVALS), _desiredJitterBufferFrames(1), - _currentJitterBufferFrames(-1), _dynamicJitterBuffers(dynamicJitterBuffers), _consecutiveNotMixedCount(0), _starveCount(0), @@ -46,63 +46,6 @@ PositionalAudioRingBuffer::PositionalAudioRingBuffer(PositionalAudioRingBuffer:: { } -int PositionalAudioRingBuffer::parseData(const QByteArray& packet) { - - // skip the packet header (includes the source UUID) - int readBytes = numBytesForPacketHeader(packet); - - // skip the sequence number - readBytes += sizeof(quint16); - - // hop over the channel flag that has already been read in AudioMixerClientData - readBytes += sizeof(quint8); - // read the positional data - readBytes += parsePositionalData(packet.mid(readBytes)); - - if (packetTypeForPacket(packet) == PacketTypeSilentAudioFrame) { - // this source had no audio to send us, but this counts as a packet - // write silence equivalent to the number of silent samples they just sent us - int16_t numSilentSamples; - - memcpy(&numSilentSamples, packet.data() + readBytes, sizeof(int16_t)); - - readBytes += sizeof(int16_t); - - // NOTE: fixes a bug in old clients that would send garbage for their number of silentSamples - numSilentSamples = getSamplesPerFrame(); - - if (numSilentSamples > 0) { - if (_dynamicJitterBuffers && _currentJitterBufferFrames > _desiredJitterBufferFrames) { - // our current jitter buffer size exceeds its desired value, so ignore some silent - // frames to get that size as close to desired as possible - int samplesPerFrame = getSamplesPerFrame(); - int numSilentFrames = numSilentSamples / samplesPerFrame; - int numFramesToDropDesired = _currentJitterBufferFrames - _desiredJitterBufferFrames; - - if (numSilentFrames > numFramesToDropDesired) { - // we have more than enough frames to drop to get the jitter buffer to its desired length - int numSilentFramesToAdd = numSilentFrames - numFramesToDropDesired; - addSilentFrame(numSilentFramesToAdd * samplesPerFrame); - _currentJitterBufferFrames = _desiredJitterBufferFrames; - - _silentFramesDropped += numFramesToDropDesired; - } else { - // we need to drop all frames to get the jitter buffer close as possible to its desired length - _currentJitterBufferFrames -= numSilentFrames; - - _silentFramesDropped += numSilentFrames; - } - } else { - addSilentFrame(numSilentSamples); - } - } - } else { - // there is audio data to read - readBytes += writeData(packet.data() + readBytes, packet.size() - readBytes); - } - return readBytes; -} - int PositionalAudioRingBuffer::parsePositionalData(const QByteArray& positionalByteArray) { QDataStream packetStream(positionalByteArray); @@ -147,10 +90,9 @@ void PositionalAudioRingBuffer::updateNextOutputTrailingLoudness() { } bool PositionalAudioRingBuffer::shouldBeAddedToMix() { - int samplesPerFrame = getSamplesPerFrame(); - int desiredJitterBufferSamples = _desiredJitterBufferFrames * samplesPerFrame; + int desiredJitterBufferSamples = _desiredJitterBufferFrames * _numFrameSamples; - if (!isNotStarvedOrHasMinimumSamples(samplesPerFrame + desiredJitterBufferSamples)) { + if (!isNotStarvedOrHasMinimumSamples(_numFrameSamples + desiredJitterBufferSamples)) { // if the buffer was starved, allow it to accrue at least the desired number of // jitter buffer frames before we start taking frames from it for mixing @@ -160,13 +102,12 @@ bool PositionalAudioRingBuffer::shouldBeAddedToMix() { _consecutiveNotMixedCount++; return false; - } else if (samplesAvailable() < samplesPerFrame) { + } else if (samplesAvailable() < _numFrameSamples) { // if the buffer doesn't have a full frame of samples to take for mixing, it is starved _isStarved = true; _starveCount++; - - // set to -1 to indicate the jitter buffer is starved - _currentJitterBufferFrames = -1; + + _framesAvailableStats.reset(); // reset our _shouldOutputStarveDebug to true so the next is printed _shouldOutputStarveDebug = true; @@ -176,13 +117,14 @@ bool PositionalAudioRingBuffer::shouldBeAddedToMix() { } // good buffer, add this to the mix - if (_isStarved) { - // if this buffer has just finished replenishing after being starved, the number of frames in it now - // minus one (since a frame will be read immediately after this) is the length of the jitter buffer - _currentJitterBufferFrames = samplesAvailable() / samplesPerFrame - 1; - _isStarved = false; - } + // if we just finished refilling after a starve, we have a new jitter buffer length. + // reset the frames available stats. + + _isStarved = false; + + _framesAvailableStats.update(framesAvailable()); + // since we've read data from ring buffer at least once - we've started _hasStarted = true; @@ -190,17 +132,18 @@ bool PositionalAudioRingBuffer::shouldBeAddedToMix() { } int PositionalAudioRingBuffer::getCalculatedDesiredJitterBufferFrames() const { - int calculatedDesiredJitterBufferFrames = 1; const float USECS_PER_FRAME = NETWORK_BUFFER_LENGTH_SAMPLES_PER_CHANNEL * USECS_PER_SECOND / (float)SAMPLE_RATE; - calculatedDesiredJitterBufferFrames = ceilf((float)_interframeTimeGapStatsForJitterCalc.getWindowMax() / USECS_PER_FRAME); + int calculatedDesiredJitterBufferFrames = ceilf((float)_interframeTimeGapStatsForJitterCalc.getWindowMax() / USECS_PER_FRAME); if (calculatedDesiredJitterBufferFrames < 1) { calculatedDesiredJitterBufferFrames = 1; } return calculatedDesiredJitterBufferFrames; } -void PositionalAudioRingBuffer::timeGapStatsFrameReceived() { + +void PositionalAudioRingBuffer::frameReceivedUpdateTimingStats() { + // update the two time gap stats we're keeping quint64 now = usecTimestampNow(); if (_lastFrameReceivedTime != 0) { quint64 gap = now - _lastFrameReceivedTime; @@ -208,15 +151,14 @@ void PositionalAudioRingBuffer::timeGapStatsFrameReceived() { _interframeTimeGapStatsForStatsPacket.update(gap); } _lastFrameReceivedTime = now; -} -void PositionalAudioRingBuffer::updateDesiredJitterBufferFrames() { + // recalculate the _desiredJitterBufferFrames if _interframeTimeGapStatsForJitterCalc has updated stats for us if (_interframeTimeGapStatsForJitterCalc.getNewStatsAvailableFlag()) { if (!_dynamicJitterBuffers) { _desiredJitterBufferFrames = 1; // HACK to see if this fixes the audio silence } else { const float USECS_PER_FRAME = NETWORK_BUFFER_LENGTH_SAMPLES_PER_CHANNEL * USECS_PER_SECOND / (float)SAMPLE_RATE; - + _desiredJitterBufferFrames = ceilf((float)_interframeTimeGapStatsForJitterCalc.getWindowMax() / USECS_PER_FRAME); if (_desiredJitterBufferFrames < 1) { _desiredJitterBufferFrames = 1; @@ -229,3 +171,38 @@ void PositionalAudioRingBuffer::updateDesiredJitterBufferFrames() { _interframeTimeGapStatsForJitterCalc.clearNewStatsAvailableFlag(); } } + +void PositionalAudioRingBuffer::addDroppableSilentSamples(int numSilentSamples) { + + // This adds some number of frames to the desired jitter buffer frames target we use. + // The larger this value is, the less aggressive we are about reducing the jitter buffer length. + // Setting this to 0 will try to get the jitter buffer to be exactly _desiredJitterBufferFrames long, + // which could lead immediately to a starve. + const int DESIRED_JITTER_BUFFER_FRAMES_PADDING = 1; + + // calculate how many silent frames we should drop. We only drop silent frames if + // the running avg num frames available has stabilized and it's more than + // our desired number of frames by the margin defined above. + int numSilentFramesToDrop = 0; + if (_framesAvailableStats.getNewStatsAvailableFlag() && _framesAvailableStats.isWindowFilled() + && numSilentSamples >= _numFrameSamples) { + _framesAvailableStats.clearNewStatsAvailableFlag(); + int averageJitterBufferFrames = (int)_framesAvailableStats.getWindowAverage(); + int desiredJitterBufferFramesPlusPadding = _desiredJitterBufferFrames + DESIRED_JITTER_BUFFER_FRAMES_PADDING; + + if (averageJitterBufferFrames > desiredJitterBufferFramesPlusPadding) { + // our avg jitter buffer size exceeds its desired value, so ignore some silent + // frames to get that size as close to desired as possible + int numSilentFramesToDropDesired = averageJitterBufferFrames - desiredJitterBufferFramesPlusPadding; + int numSilentFramesReceived = numSilentSamples / _numFrameSamples; + numSilentFramesToDrop = std::min(numSilentFramesToDropDesired, numSilentFramesReceived); + + // since we now have a new jitter buffer length, reset the frames available stats. + _framesAvailableStats.reset(); + + _silentFramesDropped += numSilentFramesToDrop; + } + } + + addSilentFrame(numSilentSamples - numSilentFramesToDrop * _numFrameSamples); +} diff --git a/libraries/audio/src/PositionalAudioRingBuffer.h b/libraries/audio/src/PositionalAudioRingBuffer.h index 3a9e7ed124..c4645c362c 100644 --- a/libraries/audio/src/PositionalAudioRingBuffer.h +++ b/libraries/audio/src/PositionalAudioRingBuffer.h @@ -29,6 +29,11 @@ const int TIME_GAPS_FOR_JITTER_CALC_WINDOW_INTERVALS = 10; const int TIME_GAPS_FOR_STATS_PACKET_INTERVAL_SAMPLES = USECS_PER_SECOND / BUFFER_SEND_INTERVAL_USECS; const int TIME_GAPS_FOR_STATS_PACKET_WINDOW_INTERVALS = 30; +// the stats for calculating the average frames available will recalculate every ~1 second +// and will include data for the past ~2 seconds +const int FRAMES_AVAILABLE_STATS_INTERVAL_SAMPLES = USECS_PER_SECOND / BUFFER_SEND_INTERVAL_USECS; +const int FRAMES_AVAILABLE_STATS_WINDOW_INTERVALS = 2; + const int AUDIOMIXER_INBOUND_RING_BUFFER_FRAME_CAPACITY = 100; class PositionalAudioRingBuffer : public AudioRingBuffer { @@ -40,7 +45,8 @@ public: PositionalAudioRingBuffer(PositionalAudioRingBuffer::Type type, bool isStereo = false, bool dynamicJitterBuffers = false); - int parseData(const QByteArray& packet); + virtual int parseData(const QByteArray& packet, int packetsSkipped = 0) = 0; + int parsePositionalData(const QByteArray& positionalByteArray); int parseListenModeData(const QByteArray& listenModeByteArray); @@ -69,7 +75,7 @@ public: int getCalculatedDesiredJitterBufferFrames() const; /// returns what we would calculate our desired as if asked int getDesiredJitterBufferFrames() const { return _desiredJitterBufferFrames; } - int getCurrentJitterBufferFrames() const { return _currentJitterBufferFrames; } + double getFramesAvailableAverage() const { return _framesAvailableStats.getWindowAverage(); } int getConsecutiveNotMixedCount() const { return _consecutiveNotMixedCount; } int getStarveCount() const { return _starveCount; } @@ -80,8 +86,8 @@ protected: PositionalAudioRingBuffer(const PositionalAudioRingBuffer&); PositionalAudioRingBuffer& operator= (const PositionalAudioRingBuffer&); - void timeGapStatsFrameReceived(); - void updateDesiredJitterBufferFrames(); + void frameReceivedUpdateTimingStats(); + void addDroppableSilentSamples(int numSilentSamples); PositionalAudioRingBuffer::Type _type; glm::vec3 _position; @@ -97,9 +103,9 @@ protected: quint64 _lastFrameReceivedTime; MovingMinMaxAvg _interframeTimeGapStatsForJitterCalc; MovingMinMaxAvg _interframeTimeGapStatsForStatsPacket; + MovingMinMaxAvg _framesAvailableStats; int _desiredJitterBufferFrames; - int _currentJitterBufferFrames; bool _dynamicJitterBuffers; // extra stats diff --git a/libraries/networking/src/SequenceNumberStats.cpp b/libraries/networking/src/SequenceNumberStats.cpp index 28dea7de63..421d8ddc9a 100644 --- a/libraries/networking/src/SequenceNumberStats.cpp +++ b/libraries/networking/src/SequenceNumberStats.cpp @@ -30,7 +30,9 @@ void SequenceNumberStats::reset() { static const int UINT16_RANGE = std::numeric_limits::max() + 1; -void SequenceNumberStats::sequenceNumberReceived(quint16 incoming, QUuid senderUUID, const bool wantExtraDebugging) { +SequenceNumberStats::ArrivalInfo SequenceNumberStats::sequenceNumberReceived(quint16 incoming, QUuid senderUUID, const bool wantExtraDebugging) { + + SequenceNumberStats::ArrivalInfo arrivalInfo; // if the sender node has changed, reset all stats if (senderUUID != _lastSenderUUID) { @@ -46,6 +48,7 @@ void SequenceNumberStats::sequenceNumberReceived(quint16 incoming, QUuid senderU _stats._numReceived++; if (incoming == expected) { // on time + arrivalInfo._status = OnTime; _lastReceived = incoming; } else { // out of order @@ -67,21 +70,29 @@ void SequenceNumberStats::sequenceNumberReceived(quint16 incoming, QUuid senderU expectedInt -= UINT16_RANGE; } } else if (absGap > MAX_REASONABLE_SEQUENCE_GAP) { + arrivalInfo._status = Unreasonable; + // ignore packet if gap is unreasonable qDebug() << "ignoring unreasonable sequence number:" << incoming << "previous:" << _lastReceived; _stats._numUnreasonable++; - return; + + return arrivalInfo; } - // now that rollover has been corrected for (if it occurred), incoming and expected can be + // now that rollover has been corrected for (if it occurred), incomingInt and expectedInt can be // compared to each other directly, though one of them might be negative + + arrivalInfo._seqDiffFromExpected = incomingInt - expectedInt; + if (incomingInt > expectedInt) { // early + arrivalInfo._status = Early; + if (wantExtraDebugging) { qDebug() << "this packet is earlier than expected..."; qDebug() << ">>>>>>>> missing gap=" << (incomingInt - expectedInt); } - + _stats._numEarly++; _stats._numLost += (incomingInt - expectedInt); _lastReceived = incoming; @@ -100,18 +111,23 @@ void SequenceNumberStats::sequenceNumberReceived(quint16 incoming, QUuid senderU if (wantExtraDebugging) { qDebug() << "this packet is later than expected..."; } + _stats._numLate++; // do not update _lastReceived; it shouldn't become smaller // remove this from missing sequence number if it's in there if (_missingSet.remove(incoming)) { + arrivalInfo._status = Late; + if (wantExtraDebugging) { qDebug() << "found it in _missingSet"; } _stats._numLost--; _stats._numRecovered++; } else { + arrivalInfo._status = Duplicate; + if (wantExtraDebugging) { qDebug() << "sequence:" << incoming << "was NOT found in _missingSet and is probably a duplicate"; } @@ -119,6 +135,7 @@ void SequenceNumberStats::sequenceNumberReceived(quint16 incoming, QUuid senderU } } } + return arrivalInfo; } void SequenceNumberStats::pruneMissingSet(const bool wantExtraDebugging) { diff --git a/libraries/networking/src/SequenceNumberStats.h b/libraries/networking/src/SequenceNumberStats.h index 8c16345aaf..f4e85b6fb3 100644 --- a/libraries/networking/src/SequenceNumberStats.h +++ b/libraries/networking/src/SequenceNumberStats.h @@ -48,10 +48,25 @@ public: class SequenceNumberStats { public: + enum ArrivalStatus { + OnTime, + Unreasonable, + Early, + Late, // recovered + Duplicate + }; + + class ArrivalInfo { + public: + ArrivalStatus _status; + int _seqDiffFromExpected; + }; + + SequenceNumberStats(int statsHistoryLength = 0); void reset(); - void sequenceNumberReceived(quint16 incoming, QUuid senderUUID = QUuid(), const bool wantExtraDebugging = false); + ArrivalInfo sequenceNumberReceived(quint16 incoming, QUuid senderUUID = QUuid(), const bool wantExtraDebugging = false); void pruneMissingSet(const bool wantExtraDebugging = false); void pushStatsToHistory() { _statsHistory.insert(_stats); } diff --git a/libraries/shared/src/MovingMinMaxAvg.h b/libraries/shared/src/MovingMinMaxAvg.h index 7c645dbd93..734018b469 100644 --- a/libraries/shared/src/MovingMinMaxAvg.h +++ b/libraries/shared/src/MovingMinMaxAvg.h @@ -126,6 +126,8 @@ public: T getWindowMax() const { return _windowStats._max; } double getWindowAverage() const { return _windowStats._average; } + bool isWindowFilled() const { return _intervalStats.isFilled(); } + private: int _intervalLength; int _windowIntervals; diff --git a/libraries/shared/src/RingBufferHistory.h b/libraries/shared/src/RingBufferHistory.h index e9875ec38a..27a78c0055 100644 --- a/libraries/shared/src/RingBufferHistory.h +++ b/libraries/shared/src/RingBufferHistory.h @@ -73,6 +73,7 @@ public: int getCapacity() const { return _capacity; } int getNumEntries() const { return _numEntries; } + bool isFilled() const { return _numEntries == _capacity; } private: int _size;