From bdbb12f4dcb71f2bbe93b9c5050e176842a6feec Mon Sep 17 00:00:00 2001 From: wangyix Date: Tue, 17 Jun 2014 14:35:40 -0700 Subject: [PATCH 01/14] switched edit nacks to non-verified --- assignment-client/src/octree/OctreeInboundPacketProcessor.cpp | 2 +- libraries/networking/src/PacketHeaders.h | 2 +- libraries/octree/src/OctreeEditPacketSender.cpp | 2 +- 3 files changed, 3 insertions(+), 3 deletions(-) diff --git a/assignment-client/src/octree/OctreeInboundPacketProcessor.cpp b/assignment-client/src/octree/OctreeInboundPacketProcessor.cpp index 44c9576943..78239bdf1f 100644 --- a/assignment-client/src/octree/OctreeInboundPacketProcessor.cpp +++ b/assignment-client/src/octree/OctreeInboundPacketProcessor.cpp @@ -256,7 +256,7 @@ int OctreeInboundPacketProcessor::sendNackPackets() { numSequenceNumbersAvailable -= numSequenceNumbers; // send it - qint64 bytesWritten = NodeList::getInstance()->writeDatagram(packet, dataAt - packet, destinationNode); + qint64 bytesWritten = NodeList::getInstance()->writeUnverifiedDatagram(packet, dataAt - packet, destinationNode); printf("\t\t wrote %lld bytes\n\n", bytesWritten); packetsSent++; diff --git a/libraries/networking/src/PacketHeaders.h b/libraries/networking/src/PacketHeaders.h index 5ed4110627..0f87b0e607 100644 --- a/libraries/networking/src/PacketHeaders.h +++ b/libraries/networking/src/PacketHeaders.h @@ -79,7 +79,7 @@ const QSet NON_VERIFIED_PACKETS = QSet() << PacketTypeDomainList << PacketTypeDomainListRequest << PacketTypeDomainOAuthRequest << PacketTypeCreateAssignment << PacketTypeRequestAssignment << PacketTypeStunResponse << PacketTypeNodeJsonStats << PacketTypeVoxelQuery << PacketTypeParticleQuery << PacketTypeModelQuery - << PacketTypeOctreeDataNack; + << PacketTypeOctreeDataNack << PacketTypeVoxelEditNack << PacketTypeParticleEditNack << PacketTypeModelEditNack; const int NUM_BYTES_MD5_HASH = 16; const int NUM_STATIC_HEADER_BYTES = sizeof(PacketVersion) + NUM_BYTES_RFC4122_UUID; diff --git a/libraries/octree/src/OctreeEditPacketSender.cpp b/libraries/octree/src/OctreeEditPacketSender.cpp index 01d2ecf464..5281617e72 100644 --- a/libraries/octree/src/OctreeEditPacketSender.cpp +++ b/libraries/octree/src/OctreeEditPacketSender.cpp @@ -379,7 +379,7 @@ void OctreeEditPacketSender::processNackPacket(const QByteArray& packet) { // read number of sequence numbers uint16_t numSequenceNumbers = (*(uint16_t*)dataAt); dataAt += sizeof(uint16_t); - + // read sequence numbers and queue packets for resend for (int i = 0; i < numSequenceNumbers; i++) { unsigned short int sequenceNumber = (*(unsigned short int*)dataAt); From aa3602f0fb5fa0def615377e25dd52e2f33e5a3f Mon Sep 17 00:00:00 2001 From: wangyix Date: Thu, 19 Jun 2014 14:34:55 -0700 Subject: [PATCH 02/14] added InterframeTimeGapHistory and frameReceived() calls --- .../src/audio/AvatarAudioRingBuffer.cpp | 2 + libraries/audio/src/AudioRingBuffer.cpp | 57 +++++++++++++++++++ libraries/audio/src/AudioRingBuffer.h | 28 +++++++++ .../audio/src/InjectedAudioRingBuffer.cpp | 2 + 4 files changed, 89 insertions(+) diff --git a/assignment-client/src/audio/AvatarAudioRingBuffer.cpp b/assignment-client/src/audio/AvatarAudioRingBuffer.cpp index 5613a64cc4..47a0ecbea8 100644 --- a/assignment-client/src/audio/AvatarAudioRingBuffer.cpp +++ b/assignment-client/src/audio/AvatarAudioRingBuffer.cpp @@ -19,6 +19,8 @@ AvatarAudioRingBuffer::AvatarAudioRingBuffer(bool isStereo) : } int AvatarAudioRingBuffer::parseData(const QByteArray& packet) { + _timeGapHistory.frameReceived(); + _shouldLoopbackForNode = (packetTypeForPacket(packet) == PacketTypeMicrophoneAudioWithEcho); return PositionalAudioRingBuffer::parseData(packet); } diff --git a/libraries/audio/src/AudioRingBuffer.cpp b/libraries/audio/src/AudioRingBuffer.cpp index 2101fcb9cd..ea15e27ef5 100644 --- a/libraries/audio/src/AudioRingBuffer.cpp +++ b/libraries/audio/src/AudioRingBuffer.cpp @@ -12,6 +12,7 @@ #include #include #include +#include "SharedUtil.h" #include @@ -19,6 +20,62 @@ #include "AudioRingBuffer.h" +InterframeTimeGapHistory::InterframeTimeGapHistory() + : _lastFrameReceivedTime(0), + _numSamplesInCurrentInterval(0), + _currentIntervalMaxGap(0), + _newestIntervalMaxGapAt(0), + _windowMaxGap(0), + _newWindowMaxGapAvailable(false) +{ + memset(_intervalMaxGaps, 0, TIME_GAP_NUM_INTERVALS_IN_WINDOW*sizeof(quint64)); +} + +void InterframeTimeGapHistory::frameReceived() { + quint64 now = usecTimestampNow(); + + // make sure this isn't the first time frameReceived() is called, meaning there's actually a gap to calculate. + if (_lastFrameReceivedTime != 0) { + quint64 gap = now - _lastFrameReceivedTime; + + // update the current interval max + if (gap > _currentIntervalMaxGap) { + _currentIntervalMaxGap = gap; + } + _numSamplesInCurrentInterval++; + + // if the current interval of samples is now full, record it in our interval maxes + if (_numSamplesInCurrentInterval == TIME_GAP_NUM_SAMPLES_IN_INTERVAL) { + + // find location to insert this interval's max (increment index cyclically) + _newestIntervalMaxGapAt = _newestIntervalMaxGapAt == TIME_GAP_NUM_INTERVALS_IN_WINDOW - 1 ? 0 : _newestIntervalMaxGapAt + 1; + + // record the current interval's max gap as the newest + _intervalMaxGaps[_newestIntervalMaxGapAt] = _currentIntervalMaxGap; + + // update the window max gap, which is the max out of all the past intervals' max gaps + _windowMaxGap = 0; + for (int i = 0; i < TIME_GAP_NUM_INTERVALS_IN_WINDOW; i++) { + if (_intervalMaxGaps[i] > _windowMaxGap) { + _windowMaxGap = _intervalMaxGaps[i]; + } + } + _newWindowMaxGapAvailable = true; + + // reset the current interval + _numSamplesInCurrentInterval = 0; + _currentIntervalMaxGap = 0; + } + } + _lastFrameReceivedTime = now; +} + +quint64 InterframeTimeGapHistory::getPastWindowMaxGap() { + _newWindowMaxGapAvailable = false; + return _windowMaxGap; +} + + AudioRingBuffer::AudioRingBuffer(int numFrameSamples, bool randomAccessMode) : NodeData(), _sampleCapacity(numFrameSamples * RING_BUFFER_LENGTH_FRAMES), diff --git a/libraries/audio/src/AudioRingBuffer.h b/libraries/audio/src/AudioRingBuffer.h index 04cc67c8ac..105e3761ae 100644 --- a/libraries/audio/src/AudioRingBuffer.h +++ b/libraries/audio/src/AudioRingBuffer.h @@ -21,6 +21,30 @@ #include "NodeData.h" +// this means that every 500 samples, the max for the past 10*500 samples will be calculated +const int TIME_GAP_NUM_SAMPLES_IN_INTERVAL = 500; +const int TIME_GAP_NUM_INTERVALS_IN_WINDOW = 10; + +class InterframeTimeGapHistory { +public: + InterframeTimeGapHistory(); + + void frameReceived(); + bool isNewWindowMaxGapAvailable() const { return _newWindowMaxGapAvailable; } + quint64 getPastWindowMaxGap(); + +private: + quint64 _lastFrameReceivedTime; + + int _numSamplesInCurrentInterval; + quint64 _currentIntervalMaxGap; + quint64 _intervalMaxGaps[TIME_GAP_NUM_INTERVALS_IN_WINDOW]; + int _newestIntervalMaxGapAt; + quint64 _windowMaxGap; + bool _newWindowMaxGapAvailable; +}; + + const int SAMPLE_RATE = 24000; const int NETWORK_BUFFER_LENGTH_BYTES_STEREO = 1024; @@ -74,6 +98,8 @@ public: bool hasStarted() const { return _hasStarted; } void addSilentFrame(int numSilentSamples); + + InterframeTimeGapHistory& getInterframeTimeGapHistory() { return _timeGapHistory; } protected: // disallow copying of AudioRingBuffer objects AudioRingBuffer(const AudioRingBuffer&); @@ -89,6 +115,8 @@ protected: bool _isStarved; bool _hasStarted; bool _randomAccessMode; /// will this ringbuffer be used for random access? if so, do some special processing + + InterframeTimeGapHistory _timeGapHistory; }; #endif // hifi_AudioRingBuffer_h diff --git a/libraries/audio/src/InjectedAudioRingBuffer.cpp b/libraries/audio/src/InjectedAudioRingBuffer.cpp index 2658b4c336..103d0d8d2e 100644 --- a/libraries/audio/src/InjectedAudioRingBuffer.cpp +++ b/libraries/audio/src/InjectedAudioRingBuffer.cpp @@ -31,6 +31,8 @@ InjectedAudioRingBuffer::InjectedAudioRingBuffer(const QUuid& streamIdentifier) const uchar MAX_INJECTOR_VOLUME = 255; int InjectedAudioRingBuffer::parseData(const QByteArray& packet) { + _timeGapHistory.frameReceived(); + // setup a data stream to read from this packet QDataStream packetStream(packet); packetStream.skipRawData(numBytesForPacketHeader(packet)); From 8176aa129289dc99eeaa85a9727dceac6fc27cbc Mon Sep 17 00:00:00 2001 From: wangyix Date: Thu, 19 Jun 2014 14:54:22 -0700 Subject: [PATCH 03/14] added debug code for testing time gap history --- libraries/audio/src/AudioRingBuffer.cpp | 6 ++++++ libraries/audio/src/AudioRingBuffer.h | 4 ++-- 2 files changed, 8 insertions(+), 2 deletions(-) diff --git a/libraries/audio/src/AudioRingBuffer.cpp b/libraries/audio/src/AudioRingBuffer.cpp index ea15e27ef5..f42ba49380 100644 --- a/libraries/audio/src/AudioRingBuffer.cpp +++ b/libraries/audio/src/AudioRingBuffer.cpp @@ -38,6 +38,8 @@ void InterframeTimeGapHistory::frameReceived() { if (_lastFrameReceivedTime != 0) { quint64 gap = now - _lastFrameReceivedTime; +printf("new gap: %llu\n", gap); + // update the current interval max if (gap > _currentIntervalMaxGap) { _currentIntervalMaxGap = gap; @@ -47,6 +49,8 @@ void InterframeTimeGapHistory::frameReceived() { // if the current interval of samples is now full, record it in our interval maxes if (_numSamplesInCurrentInterval == TIME_GAP_NUM_SAMPLES_IN_INTERVAL) { +printf("\t interval full: max interval gap: %llu\n", _currentIntervalMaxGap); + // find location to insert this interval's max (increment index cyclically) _newestIntervalMaxGapAt = _newestIntervalMaxGapAt == TIME_GAP_NUM_INTERVALS_IN_WINDOW - 1 ? 0 : _newestIntervalMaxGapAt + 1; @@ -62,6 +66,8 @@ void InterframeTimeGapHistory::frameReceived() { } _newWindowMaxGapAvailable = true; +printf("\t\t new window max gap: %llu\n", _windowMaxGap); + // reset the current interval _numSamplesInCurrentInterval = 0; _currentIntervalMaxGap = 0; diff --git a/libraries/audio/src/AudioRingBuffer.h b/libraries/audio/src/AudioRingBuffer.h index 105e3761ae..b702abde7b 100644 --- a/libraries/audio/src/AudioRingBuffer.h +++ b/libraries/audio/src/AudioRingBuffer.h @@ -22,8 +22,8 @@ #include "NodeData.h" // this means that every 500 samples, the max for the past 10*500 samples will be calculated -const int TIME_GAP_NUM_SAMPLES_IN_INTERVAL = 500; -const int TIME_GAP_NUM_INTERVALS_IN_WINDOW = 10; +const int TIME_GAP_NUM_SAMPLES_IN_INTERVAL = 6; +const int TIME_GAP_NUM_INTERVALS_IN_WINDOW = 4; class InterframeTimeGapHistory { public: From 77c377711a28d735405141c305c7d72e7460b620 Mon Sep 17 00:00:00 2001 From: wangyix Date: Thu, 19 Jun 2014 15:39:54 -0700 Subject: [PATCH 04/14] removed debug code --- libraries/audio/src/AudioRingBuffer.cpp | 6 ------ libraries/audio/src/AudioRingBuffer.h | 4 ++-- 2 files changed, 2 insertions(+), 8 deletions(-) diff --git a/libraries/audio/src/AudioRingBuffer.cpp b/libraries/audio/src/AudioRingBuffer.cpp index f42ba49380..ea15e27ef5 100644 --- a/libraries/audio/src/AudioRingBuffer.cpp +++ b/libraries/audio/src/AudioRingBuffer.cpp @@ -38,8 +38,6 @@ void InterframeTimeGapHistory::frameReceived() { if (_lastFrameReceivedTime != 0) { quint64 gap = now - _lastFrameReceivedTime; -printf("new gap: %llu\n", gap); - // update the current interval max if (gap > _currentIntervalMaxGap) { _currentIntervalMaxGap = gap; @@ -49,8 +47,6 @@ printf("new gap: %llu\n", gap); // if the current interval of samples is now full, record it in our interval maxes if (_numSamplesInCurrentInterval == TIME_GAP_NUM_SAMPLES_IN_INTERVAL) { -printf("\t interval full: max interval gap: %llu\n", _currentIntervalMaxGap); - // find location to insert this interval's max (increment index cyclically) _newestIntervalMaxGapAt = _newestIntervalMaxGapAt == TIME_GAP_NUM_INTERVALS_IN_WINDOW - 1 ? 0 : _newestIntervalMaxGapAt + 1; @@ -66,8 +62,6 @@ printf("\t interval full: max interval gap: %llu\n", _currentIntervalMaxGap); } _newWindowMaxGapAvailable = true; -printf("\t\t new window max gap: %llu\n", _windowMaxGap); - // reset the current interval _numSamplesInCurrentInterval = 0; _currentIntervalMaxGap = 0; diff --git a/libraries/audio/src/AudioRingBuffer.h b/libraries/audio/src/AudioRingBuffer.h index b702abde7b..105e3761ae 100644 --- a/libraries/audio/src/AudioRingBuffer.h +++ b/libraries/audio/src/AudioRingBuffer.h @@ -22,8 +22,8 @@ #include "NodeData.h" // this means that every 500 samples, the max for the past 10*500 samples will be calculated -const int TIME_GAP_NUM_SAMPLES_IN_INTERVAL = 6; -const int TIME_GAP_NUM_INTERVALS_IN_WINDOW = 4; +const int TIME_GAP_NUM_SAMPLES_IN_INTERVAL = 500; +const int TIME_GAP_NUM_INTERVALS_IN_WINDOW = 10; class InterframeTimeGapHistory { public: From 100bc022ec55a0851a6d5ffb6bff1d16f06bd03c Mon Sep 17 00:00:00 2001 From: wangyix Date: Thu, 19 Jun 2014 16:42:51 -0700 Subject: [PATCH 05/14] fixed a for loop that erased elements while iterating --- assignment-client/src/audio/AudioMixerClientData.cpp | 9 ++++++--- libraries/audio/src/AudioRingBuffer.h | 1 + 2 files changed, 7 insertions(+), 3 deletions(-) diff --git a/assignment-client/src/audio/AudioMixerClientData.cpp b/assignment-client/src/audio/AudioMixerClientData.cpp index 2f78a4ac78..2329946e49 100644 --- a/assignment-client/src/audio/AudioMixerClientData.cpp +++ b/assignment-client/src/audio/AudioMixerClientData.cpp @@ -120,9 +120,10 @@ void AudioMixerClientData::checkBuffersBeforeFrameSend(int jitterBufferLengthSam } void AudioMixerClientData::pushBuffersAfterFrameSend() { - for (int i = 0; i < _ringBuffers.size(); i++) { + QList::iterator i = _ringBuffers.begin(); + while (i != _ringBuffers.end()) { // this was a used buffer, push the output pointer forwards - PositionalAudioRingBuffer* audioBuffer = _ringBuffers[i]; + PositionalAudioRingBuffer* audioBuffer = *i; if (audioBuffer->willBeAddedToMix()) { audioBuffer->shiftReadPosition(audioBuffer->isStereo() @@ -133,7 +134,9 @@ void AudioMixerClientData::pushBuffersAfterFrameSend() { && audioBuffer->hasStarted() && audioBuffer->isStarved()) { // this is an empty audio buffer that has starved, safe to delete delete audioBuffer; - _ringBuffers.erase(_ringBuffers.begin() + i); + i = _ringBuffers.erase(i); + continue; } + i++; } } diff --git a/libraries/audio/src/AudioRingBuffer.h b/libraries/audio/src/AudioRingBuffer.h index 105e3761ae..46bc333e4b 100644 --- a/libraries/audio/src/AudioRingBuffer.h +++ b/libraries/audio/src/AudioRingBuffer.h @@ -25,6 +25,7 @@ const int TIME_GAP_NUM_SAMPLES_IN_INTERVAL = 500; const int TIME_GAP_NUM_INTERVALS_IN_WINDOW = 10; +// class used to track time between incoming frames for the purpose of varying the jitter buffer length class InterframeTimeGapHistory { public: InterframeTimeGapHistory(); From 435b5094a03437ab9f942a1ab1753961bc63b8ad Mon Sep 17 00:00:00 2001 From: wangyix Date: Thu, 19 Jun 2014 17:58:38 -0700 Subject: [PATCH 06/14] moved time gap history to PositionalAudioRingBuffer, untested removed hard-coded jitter buffer length. --- assignment-client/src/audio/AudioMixer.cpp | 6 +- .../src/audio/AudioMixerClientData.cpp | 5 +- .../src/audio/AudioMixerClientData.h | 3 +- .../src/audio/AvatarAudioRingBuffer.cpp | 3 +- libraries/audio/src/AudioRingBuffer.cpp | 57 ------------- libraries/audio/src/AudioRingBuffer.h | 32 +------- .../audio/src/InjectedAudioRingBuffer.cpp | 3 +- .../audio/src/PositionalAudioRingBuffer.cpp | 79 ++++++++++++++++++- .../audio/src/PositionalAudioRingBuffer.h | 34 +++++++- 9 files changed, 118 insertions(+), 104 deletions(-) diff --git a/assignment-client/src/audio/AudioMixer.cpp b/assignment-client/src/audio/AudioMixer.cpp index 2dc51b44a0..42be1aea5a 100644 --- a/assignment-client/src/audio/AudioMixer.cpp +++ b/assignment-client/src/audio/AudioMixer.cpp @@ -54,9 +54,6 @@ #include "AudioMixer.h" -const short JITTER_BUFFER_MSECS = 12; -const short JITTER_BUFFER_SAMPLES = JITTER_BUFFER_MSECS * (SAMPLE_RATE / 1000.0); - const float LOUDNESS_TO_DISTANCE_RATIO = 0.00001f; const QString AUDIO_MIXER_LOGGING_TARGET_NAME = "audio-mixer"; @@ -487,8 +484,7 @@ void AudioMixer::run() { foreach (const SharedNodePointer& node, nodeList->getNodeHash()) { if (node->getLinkedData()) { - ((AudioMixerClientData*) node->getLinkedData())->checkBuffersBeforeFrameSend(JITTER_BUFFER_SAMPLES, - _sourceUnattenuatedZone, + ((AudioMixerClientData*) node->getLinkedData())->checkBuffersBeforeFrameSend(_sourceUnattenuatedZone, _listenerUnattenuatedZone); } } diff --git a/assignment-client/src/audio/AudioMixerClientData.cpp b/assignment-client/src/audio/AudioMixerClientData.cpp index 2329946e49..3ee571fc46 100644 --- a/assignment-client/src/audio/AudioMixerClientData.cpp +++ b/assignment-client/src/audio/AudioMixerClientData.cpp @@ -98,10 +98,9 @@ int AudioMixerClientData::parseData(const QByteArray& packet) { return 0; } -void AudioMixerClientData::checkBuffersBeforeFrameSend(int jitterBufferLengthSamples, - AABox* checkSourceZone, AABox* listenerZone) { +void AudioMixerClientData::checkBuffersBeforeFrameSend(AABox* checkSourceZone, AABox* listenerZone) { for (int i = 0; i < _ringBuffers.size(); i++) { - if (_ringBuffers[i]->shouldBeAddedToMix(jitterBufferLengthSamples)) { + if (_ringBuffers[i]->shouldBeAddedToMix()) { // this is a ring buffer that is ready to go // set its flag so we know to push its buffer when all is said and done _ringBuffers[i]->setWillBeAddedToMix(true); diff --git a/assignment-client/src/audio/AudioMixerClientData.h b/assignment-client/src/audio/AudioMixerClientData.h index e52b09e134..3c4ddd3459 100644 --- a/assignment-client/src/audio/AudioMixerClientData.h +++ b/assignment-client/src/audio/AudioMixerClientData.h @@ -27,8 +27,7 @@ public: AvatarAudioRingBuffer* getAvatarAudioRingBuffer() const; int parseData(const QByteArray& packet); - void checkBuffersBeforeFrameSend(int jitterBufferLengthSamples, - AABox* checkSourceZone = NULL, AABox* listenerZone = NULL); + void checkBuffersBeforeFrameSend(AABox* checkSourceZone = NULL, AABox* listenerZone = NULL); void pushBuffersAfterFrameSend(); private: QList _ringBuffers; diff --git a/assignment-client/src/audio/AvatarAudioRingBuffer.cpp b/assignment-client/src/audio/AvatarAudioRingBuffer.cpp index 47a0ecbea8..6658e446cf 100644 --- a/assignment-client/src/audio/AvatarAudioRingBuffer.cpp +++ b/assignment-client/src/audio/AvatarAudioRingBuffer.cpp @@ -19,7 +19,8 @@ AvatarAudioRingBuffer::AvatarAudioRingBuffer(bool isStereo) : } int AvatarAudioRingBuffer::parseData(const QByteArray& packet) { - _timeGapHistory.frameReceived(); + _interframeTimeGapHistory.frameReceived(); + updateDesiredJitterBufferNumSamples(); _shouldLoopbackForNode = (packetTypeForPacket(packet) == PacketTypeMicrophoneAudioWithEcho); return PositionalAudioRingBuffer::parseData(packet); diff --git a/libraries/audio/src/AudioRingBuffer.cpp b/libraries/audio/src/AudioRingBuffer.cpp index ea15e27ef5..2101fcb9cd 100644 --- a/libraries/audio/src/AudioRingBuffer.cpp +++ b/libraries/audio/src/AudioRingBuffer.cpp @@ -12,7 +12,6 @@ #include #include #include -#include "SharedUtil.h" #include @@ -20,62 +19,6 @@ #include "AudioRingBuffer.h" -InterframeTimeGapHistory::InterframeTimeGapHistory() - : _lastFrameReceivedTime(0), - _numSamplesInCurrentInterval(0), - _currentIntervalMaxGap(0), - _newestIntervalMaxGapAt(0), - _windowMaxGap(0), - _newWindowMaxGapAvailable(false) -{ - memset(_intervalMaxGaps, 0, TIME_GAP_NUM_INTERVALS_IN_WINDOW*sizeof(quint64)); -} - -void InterframeTimeGapHistory::frameReceived() { - quint64 now = usecTimestampNow(); - - // make sure this isn't the first time frameReceived() is called, meaning there's actually a gap to calculate. - if (_lastFrameReceivedTime != 0) { - quint64 gap = now - _lastFrameReceivedTime; - - // update the current interval max - if (gap > _currentIntervalMaxGap) { - _currentIntervalMaxGap = gap; - } - _numSamplesInCurrentInterval++; - - // if the current interval of samples is now full, record it in our interval maxes - if (_numSamplesInCurrentInterval == TIME_GAP_NUM_SAMPLES_IN_INTERVAL) { - - // find location to insert this interval's max (increment index cyclically) - _newestIntervalMaxGapAt = _newestIntervalMaxGapAt == TIME_GAP_NUM_INTERVALS_IN_WINDOW - 1 ? 0 : _newestIntervalMaxGapAt + 1; - - // record the current interval's max gap as the newest - _intervalMaxGaps[_newestIntervalMaxGapAt] = _currentIntervalMaxGap; - - // update the window max gap, which is the max out of all the past intervals' max gaps - _windowMaxGap = 0; - for (int i = 0; i < TIME_GAP_NUM_INTERVALS_IN_WINDOW; i++) { - if (_intervalMaxGaps[i] > _windowMaxGap) { - _windowMaxGap = _intervalMaxGaps[i]; - } - } - _newWindowMaxGapAvailable = true; - - // reset the current interval - _numSamplesInCurrentInterval = 0; - _currentIntervalMaxGap = 0; - } - } - _lastFrameReceivedTime = now; -} - -quint64 InterframeTimeGapHistory::getPastWindowMaxGap() { - _newWindowMaxGapAvailable = false; - return _windowMaxGap; -} - - AudioRingBuffer::AudioRingBuffer(int numFrameSamples, bool randomAccessMode) : NodeData(), _sampleCapacity(numFrameSamples * RING_BUFFER_LENGTH_FRAMES), diff --git a/libraries/audio/src/AudioRingBuffer.h b/libraries/audio/src/AudioRingBuffer.h index 46bc333e4b..3d2864f373 100644 --- a/libraries/audio/src/AudioRingBuffer.h +++ b/libraries/audio/src/AudioRingBuffer.h @@ -21,31 +21,6 @@ #include "NodeData.h" -// this means that every 500 samples, the max for the past 10*500 samples will be calculated -const int TIME_GAP_NUM_SAMPLES_IN_INTERVAL = 500; -const int TIME_GAP_NUM_INTERVALS_IN_WINDOW = 10; - -// class used to track time between incoming frames for the purpose of varying the jitter buffer length -class InterframeTimeGapHistory { -public: - InterframeTimeGapHistory(); - - void frameReceived(); - bool isNewWindowMaxGapAvailable() const { return _newWindowMaxGapAvailable; } - quint64 getPastWindowMaxGap(); - -private: - quint64 _lastFrameReceivedTime; - - int _numSamplesInCurrentInterval; - quint64 _currentIntervalMaxGap; - quint64 _intervalMaxGaps[TIME_GAP_NUM_INTERVALS_IN_WINDOW]; - int _newestIntervalMaxGapAt; - quint64 _windowMaxGap; - bool _newWindowMaxGapAvailable; -}; - - const int SAMPLE_RATE = 24000; const int NETWORK_BUFFER_LENGTH_BYTES_STEREO = 1024; @@ -99,15 +74,14 @@ public: bool hasStarted() const { return _hasStarted; } void addSilentFrame(int numSilentSamples); - - InterframeTimeGapHistory& getInterframeTimeGapHistory() { return _timeGapHistory; } + protected: // disallow copying of AudioRingBuffer objects AudioRingBuffer(const AudioRingBuffer&); AudioRingBuffer& operator= (const AudioRingBuffer&); int16_t* shiftedPositionAccomodatingWrap(int16_t* position, int numSamplesShift) const; - + int _sampleCapacity; int _numFrameSamples; int16_t* _nextOutput; @@ -116,8 +90,6 @@ protected: bool _isStarved; bool _hasStarted; bool _randomAccessMode; /// will this ringbuffer be used for random access? if so, do some special processing - - InterframeTimeGapHistory _timeGapHistory; }; #endif // hifi_AudioRingBuffer_h diff --git a/libraries/audio/src/InjectedAudioRingBuffer.cpp b/libraries/audio/src/InjectedAudioRingBuffer.cpp index 103d0d8d2e..ffe5876bfd 100644 --- a/libraries/audio/src/InjectedAudioRingBuffer.cpp +++ b/libraries/audio/src/InjectedAudioRingBuffer.cpp @@ -31,7 +31,8 @@ InjectedAudioRingBuffer::InjectedAudioRingBuffer(const QUuid& streamIdentifier) const uchar MAX_INJECTOR_VOLUME = 255; int InjectedAudioRingBuffer::parseData(const QByteArray& packet) { - _timeGapHistory.frameReceived(); + _interframeTimeGapHistory.frameReceived(); + updateDesiredJitterBufferNumSamples(); // setup a data stream to read from this packet QDataStream packetStream(packet); diff --git a/libraries/audio/src/PositionalAudioRingBuffer.cpp b/libraries/audio/src/PositionalAudioRingBuffer.cpp index 1cc4147175..b537e5c6d6 100644 --- a/libraries/audio/src/PositionalAudioRingBuffer.cpp +++ b/libraries/audio/src/PositionalAudioRingBuffer.cpp @@ -19,6 +19,63 @@ #include #include "PositionalAudioRingBuffer.h" +#include "SharedUtil.h" + +InterframeTimeGapHistory::InterframeTimeGapHistory() + : _lastFrameReceivedTime(0), + _numSamplesInCurrentInterval(0), + _currentIntervalMaxGap(0), + _newestIntervalMaxGapAt(0), + _windowMaxGap(0), + _newWindowMaxGapAvailable(false) +{ + memset(_intervalMaxGaps, 0, TIME_GAP_NUM_INTERVALS_IN_WINDOW*sizeof(quint64)); +} + +void InterframeTimeGapHistory::frameReceived() { + quint64 now = usecTimestampNow(); + + // make sure this isn't the first time frameReceived() is called, meaning there's actually a gap to calculate. + if (_lastFrameReceivedTime != 0) { + quint64 gap = now - _lastFrameReceivedTime; + + // update the current interval max + if (gap > _currentIntervalMaxGap) { + _currentIntervalMaxGap = gap; + } + _numSamplesInCurrentInterval++; + + // if the current interval of samples is now full, record it in our interval maxes + if (_numSamplesInCurrentInterval == TIME_GAP_NUM_SAMPLES_IN_INTERVAL) { + + // find location to insert this interval's max (increment index cyclically) + _newestIntervalMaxGapAt = _newestIntervalMaxGapAt == TIME_GAP_NUM_INTERVALS_IN_WINDOW - 1 ? 0 : _newestIntervalMaxGapAt + 1; + + // record the current interval's max gap as the newest + _intervalMaxGaps[_newestIntervalMaxGapAt] = _currentIntervalMaxGap; + + // update the window max gap, which is the max out of all the past intervals' max gaps + _windowMaxGap = 0; + for (int i = 0; i < TIME_GAP_NUM_INTERVALS_IN_WINDOW; i++) { + if (_intervalMaxGaps[i] > _windowMaxGap) { + _windowMaxGap = _intervalMaxGaps[i]; + } + } + _newWindowMaxGapAvailable = true; + + // reset the current interval + _numSamplesInCurrentInterval = 0; + _currentIntervalMaxGap = 0; + } + } + _lastFrameReceivedTime = now; +} + +quint64 InterframeTimeGapHistory::getWindowMaxGap() { + _newWindowMaxGapAvailable = false; + return _windowMaxGap; +} + PositionalAudioRingBuffer::PositionalAudioRingBuffer(PositionalAudioRingBuffer::Type type, bool isStereo) : AudioRingBuffer(isStereo ? NETWORK_BUFFER_LENGTH_SAMPLES_STEREO : NETWORK_BUFFER_LENGTH_SAMPLES_PER_CHANNEL), @@ -29,7 +86,8 @@ PositionalAudioRingBuffer::PositionalAudioRingBuffer(PositionalAudioRingBuffer:: _shouldLoopbackForNode(false), _shouldOutputStarveDebug(true), _isStereo(isStereo), - _listenerUnattenuatedZone(NULL) + _listenerUnattenuatedZone(NULL), + _desiredJitterBufferNumSamples(getNumSamplesPerFrame()) { } @@ -106,14 +164,17 @@ void PositionalAudioRingBuffer::updateNextOutputTrailingLoudness() { } } -bool PositionalAudioRingBuffer::shouldBeAddedToMix(int numJitterBufferSamples) { - if (!isNotStarvedOrHasMinimumSamples(NETWORK_BUFFER_LENGTH_SAMPLES_PER_CHANNEL + numJitterBufferSamples)) { +bool PositionalAudioRingBuffer::shouldBeAddedToMix() { + + int samplesPerFrame = getNumSamplesPerFrame(); + + if (!isNotStarvedOrHasMinimumSamples(samplesPerFrame + _desiredJitterBufferNumSamples)) { if (_shouldOutputStarveDebug) { _shouldOutputStarveDebug = false; } return false; - } else if (samplesAvailable() < NETWORK_BUFFER_LENGTH_SAMPLES_PER_CHANNEL) { + } else if (samplesAvailable() < samplesPerFrame) { _isStarved = true; // reset our _shouldOutputStarveDebug to true so the next is printed @@ -132,3 +193,13 @@ bool PositionalAudioRingBuffer::shouldBeAddedToMix(int numJitterBufferSamples) { return false; } + +void PositionalAudioRingBuffer::updateDesiredJitterBufferNumSamples() { + + const float USECS_PER_FRAME = NETWORK_BUFFER_LENGTH_SAMPLES_PER_CHANNEL * USECS_PER_SECOND / (float)SAMPLE_RATE; + + if (_interframeTimeGapHistory.hasNewWindowMaxGapAvailable()) { + int desiredJitterBufferNumFrames = (int)((float)_interframeTimeGapHistory.getWindowMaxGap() / USECS_PER_FRAME + 0.5f); + _desiredJitterBufferNumSamples = desiredJitterBufferNumFrames * getNumSamplesPerFrame(); + } +} \ No newline at end of file diff --git a/libraries/audio/src/PositionalAudioRingBuffer.h b/libraries/audio/src/PositionalAudioRingBuffer.h index 00362c245a..de731c6136 100644 --- a/libraries/audio/src/PositionalAudioRingBuffer.h +++ b/libraries/audio/src/PositionalAudioRingBuffer.h @@ -18,6 +18,31 @@ #include "AudioRingBuffer.h" +// this means that every 500 samples, the max for the past 10*500 samples will be calculated +const int TIME_GAP_NUM_SAMPLES_IN_INTERVAL = 500; +const int TIME_GAP_NUM_INTERVALS_IN_WINDOW = 10; + +// class used to track time between incoming frames for the purpose of varying the jitter buffer length +class InterframeTimeGapHistory { +public: + InterframeTimeGapHistory(); + + void frameReceived(); + bool hasNewWindowMaxGapAvailable() const { return _newWindowMaxGapAvailable; } + quint64 getWindowMaxGap(); + +private: + quint64 _lastFrameReceivedTime; + + int _numSamplesInCurrentInterval; + quint64 _currentIntervalMaxGap; + quint64 _intervalMaxGaps[TIME_GAP_NUM_INTERVALS_IN_WINDOW]; + int _newestIntervalMaxGapAt; + quint64 _windowMaxGap; + bool _newWindowMaxGapAvailable; +}; + + class PositionalAudioRingBuffer : public AudioRingBuffer { public: enum Type { @@ -34,7 +59,7 @@ public: void updateNextOutputTrailingLoudness(); float getNextOutputTrailingLoudness() const { return _nextOutputTrailingLoudness; } - bool shouldBeAddedToMix(int numJitterBufferSamples); + bool shouldBeAddedToMix(); bool willBeAddedToMix() const { return _willBeAddedToMix; } void setWillBeAddedToMix(bool willBeAddedToMix) { _willBeAddedToMix = willBeAddedToMix; } @@ -50,10 +75,14 @@ public: AABox* getListenerUnattenuatedZone() const { return _listenerUnattenuatedZone; } void setListenerUnattenuatedZone(AABox* listenerUnattenuatedZone) { _listenerUnattenuatedZone = listenerUnattenuatedZone; } + int getNumSamplesPerFrame() const { return _isStereo ? NETWORK_BUFFER_LENGTH_SAMPLES_STEREO : NETWORK_BUFFER_LENGTH_BYTES_PER_CHANNEL; } + protected: // disallow copying of PositionalAudioRingBuffer objects PositionalAudioRingBuffer(const PositionalAudioRingBuffer&); PositionalAudioRingBuffer& operator= (const PositionalAudioRingBuffer&); + + void updateDesiredJitterBufferNumSamples(); PositionalAudioRingBuffer::Type _type; glm::vec3 _position; @@ -65,6 +94,9 @@ protected: float _nextOutputTrailingLoudness; AABox* _listenerUnattenuatedZone; + + InterframeTimeGapHistory _interframeTimeGapHistory; + int _desiredJitterBufferNumSamples; }; #endif // hifi_PositionalAudioRingBuffer_h From a5457eb86b7e6d766b46d4106dfa098ba3a49a4d Mon Sep 17 00:00:00 2001 From: wangyix Date: Fri, 20 Jun 2014 14:43:18 -0700 Subject: [PATCH 07/14] fixed bug in Audio.cpp where numSilentSamples was written to wrong place in packet added more debug stuff, other minor changes and fixes --- .../src/audio/AudioMixerClientData.cpp | 5 +- .../src/audio/AvatarAudioRingBuffer.cpp | 2 +- interface/src/Audio.cpp | 12 ++-- libraries/audio/src/AudioRingBuffer.cpp | 1 + .../audio/src/InjectedAudioRingBuffer.cpp | 2 +- .../audio/src/PositionalAudioRingBuffer.cpp | 63 +++++++++++++------ .../audio/src/PositionalAudioRingBuffer.h | 10 ++- 7 files changed, 63 insertions(+), 32 deletions(-) diff --git a/assignment-client/src/audio/AudioMixerClientData.cpp b/assignment-client/src/audio/AudioMixerClientData.cpp index 3ee571fc46..0c41cc70f9 100644 --- a/assignment-client/src/audio/AudioMixerClientData.cpp +++ b/assignment-client/src/audio/AudioMixerClientData.cpp @@ -119,15 +119,14 @@ void AudioMixerClientData::checkBuffersBeforeFrameSend(AABox* checkSourceZone, A } void AudioMixerClientData::pushBuffersAfterFrameSend() { + QList::iterator i = _ringBuffers.begin(); while (i != _ringBuffers.end()) { // this was a used buffer, push the output pointer forwards PositionalAudioRingBuffer* audioBuffer = *i; if (audioBuffer->willBeAddedToMix()) { - audioBuffer->shiftReadPosition(audioBuffer->isStereo() - ? NETWORK_BUFFER_LENGTH_SAMPLES_STEREO : NETWORK_BUFFER_LENGTH_SAMPLES_PER_CHANNEL); - + audioBuffer->shiftReadPosition(audioBuffer->getSamplesPerFrame()); audioBuffer->setWillBeAddedToMix(false); } else if (audioBuffer->getType() == PositionalAudioRingBuffer::Injector && audioBuffer->hasStarted() && audioBuffer->isStarved()) { diff --git a/assignment-client/src/audio/AvatarAudioRingBuffer.cpp b/assignment-client/src/audio/AvatarAudioRingBuffer.cpp index 6658e446cf..0df50f99db 100644 --- a/assignment-client/src/audio/AvatarAudioRingBuffer.cpp +++ b/assignment-client/src/audio/AvatarAudioRingBuffer.cpp @@ -20,7 +20,7 @@ AvatarAudioRingBuffer::AvatarAudioRingBuffer(bool isStereo) : int AvatarAudioRingBuffer::parseData(const QByteArray& packet) { _interframeTimeGapHistory.frameReceived(); - updateDesiredJitterBufferNumSamples(); + updateDesiredJitterBufferFrames(); _shouldLoopbackForNode = (packetTypeForPacket(packet) == PacketTypeMicrophoneAudioWithEcho); return PositionalAudioRingBuffer::parseData(packet); diff --git a/interface/src/Audio.cpp b/interface/src/Audio.cpp index 271bcd5279..47e55ddc90 100644 --- a/interface/src/Audio.cpp +++ b/interface/src/Audio.cpp @@ -461,8 +461,8 @@ void Audio::handleAudioInput() { int16_t* inputAudioSamples = new int16_t[inputSamplesRequired]; _inputRingBuffer.readSamples(inputAudioSamples, inputSamplesRequired); - int numNetworkBytes = _isStereoInput ? NETWORK_BUFFER_LENGTH_BYTES_STEREO : NETWORK_BUFFER_LENGTH_BYTES_PER_CHANNEL; - int numNetworkSamples = _isStereoInput ? NETWORK_BUFFER_LENGTH_SAMPLES_STEREO : NETWORK_BUFFER_LENGTH_SAMPLES_PER_CHANNEL; + const int numNetworkBytes = _isStereoInput ? NETWORK_BUFFER_LENGTH_BYTES_STEREO : NETWORK_BUFFER_LENGTH_BYTES_PER_CHANNEL; + const int numNetworkSamples = _isStereoInput ? NETWORK_BUFFER_LENGTH_SAMPLES_STEREO : NETWORK_BUFFER_LENGTH_SAMPLES_PER_CHANNEL; // zero out the monoAudioSamples array and the locally injected audio memset(networkAudioSamples, 0, numNetworkBytes); @@ -622,6 +622,7 @@ void Audio::handleAudioInput() { SharedNodePointer audioMixer = nodeList->soloNodeOfType(NodeType::AudioMixer); if (audioMixer && audioMixer->getActiveSocket()) { + MyAvatar* interfaceAvatar = Application::getInstance()->getAvatar(); glm::vec3 headPosition = interfaceAvatar->getHead()->getPosition(); glm::quat headOrientation = interfaceAvatar->getHead()->getFinalOrientationInWorldFrame(); @@ -634,12 +635,11 @@ void Audio::handleAudioInput() { packetType = PacketTypeSilentAudioFrame; // we need to indicate how many silent samples this is to the audio mixer - audioDataPacket[0] = _isStereoInput - ? NETWORK_BUFFER_LENGTH_SAMPLES_STEREO - : NETWORK_BUFFER_LENGTH_SAMPLES_PER_CHANNEL; + networkAudioSamples[0] = numNetworkSamples; numAudioBytes = sizeof(int16_t); } else { - numAudioBytes = _isStereoInput ? NETWORK_BUFFER_LENGTH_BYTES_STEREO : NETWORK_BUFFER_LENGTH_BYTES_PER_CHANNEL; + numAudioBytes = numNetworkBytes; + //_isStereoInput ? NETWORK_BUFFER_LENGTH_BYTES_STEREO : NETWORK_BUFFER_LENGTH_BYTES_PER_CHANNEL; if (Menu::getInstance()->isOptionChecked(MenuOption::EchoServerAudio)) { packetType = PacketTypeMicrophoneAudioWithEcho; diff --git a/libraries/audio/src/AudioRingBuffer.cpp b/libraries/audio/src/AudioRingBuffer.cpp index 2101fcb9cd..7b1e68ffbd 100644 --- a/libraries/audio/src/AudioRingBuffer.cpp +++ b/libraries/audio/src/AudioRingBuffer.cpp @@ -158,6 +158,7 @@ const int16_t& AudioRingBuffer::operator[] (const int index) const { void AudioRingBuffer::shiftReadPosition(unsigned int numSamples) { _nextOutput = shiftedPositionAccomodatingWrap(_nextOutput, numSamples); + printf("\n mixed. %d samples remaining\n", samplesAvailable()); } unsigned int AudioRingBuffer::samplesAvailable() const { diff --git a/libraries/audio/src/InjectedAudioRingBuffer.cpp b/libraries/audio/src/InjectedAudioRingBuffer.cpp index ffe5876bfd..80bcda5acb 100644 --- a/libraries/audio/src/InjectedAudioRingBuffer.cpp +++ b/libraries/audio/src/InjectedAudioRingBuffer.cpp @@ -32,7 +32,7 @@ const uchar MAX_INJECTOR_VOLUME = 255; int InjectedAudioRingBuffer::parseData(const QByteArray& packet) { _interframeTimeGapHistory.frameReceived(); - updateDesiredJitterBufferNumSamples(); + updateDesiredJitterBufferFrames(); // setup a data stream to read from this packet QDataStream packetStream(packet); diff --git a/libraries/audio/src/PositionalAudioRingBuffer.cpp b/libraries/audio/src/PositionalAudioRingBuffer.cpp index b537e5c6d6..813323ecb1 100644 --- a/libraries/audio/src/PositionalAudioRingBuffer.cpp +++ b/libraries/audio/src/PositionalAudioRingBuffer.cpp @@ -33,6 +33,10 @@ InterframeTimeGapHistory::InterframeTimeGapHistory() } void InterframeTimeGapHistory::frameReceived() { + + static QQueue gaps; + static quint64 gapsSum = 0; + quint64 now = usecTimestampNow(); // make sure this isn't the first time frameReceived() is called, meaning there's actually a gap to calculate. @@ -87,9 +91,9 @@ PositionalAudioRingBuffer::PositionalAudioRingBuffer(PositionalAudioRingBuffer:: _shouldOutputStarveDebug(true), _isStereo(isStereo), _listenerUnattenuatedZone(NULL), - _desiredJitterBufferNumSamples(getNumSamplesPerFrame()) + _desiredJitterBufferFrames(1), + _currentJitterBufferFrames(0) { - } int PositionalAudioRingBuffer::parseData(const QByteArray& packet) { @@ -114,10 +118,15 @@ int PositionalAudioRingBuffer::parseData(const QByteArray& packet) { if (numSilentSamples > 0) { addSilentFrame(numSilentSamples); } + printf("\nparsed silent packet of %d samples\n", numSilentSamples); } else { // there is audio data to read - readBytes += writeData(packet.data() + readBytes, packet.size() - readBytes); + int dataBytes = writeData(packet.data() + readBytes, packet.size() - readBytes); + readBytes += dataBytes; + + printf("\nparsed packet of %d data bytes\n", dataBytes); } + printf("%d samples available\n", samplesAvailable()); return readBytes; } @@ -166,40 +175,58 @@ void PositionalAudioRingBuffer::updateNextOutputTrailingLoudness() { bool PositionalAudioRingBuffer::shouldBeAddedToMix() { - int samplesPerFrame = getNumSamplesPerFrame(); + int samplesPerFrame = getSamplesPerFrame(); + int currentJitterBufferSamples = 3 * samplesPerFrame; //_currentJitterBufferFrames * samplesPerFrame; + +//printf("\nsamples available: %d frames available: %d\n", samplesAvailable(), samplesAvailable() / samplesPerFrame); + if (!isNotStarvedOrHasMinimumSamples(samplesPerFrame + currentJitterBufferSamples)) { - if (!isNotStarvedOrHasMinimumSamples(samplesPerFrame + _desiredJitterBufferNumSamples)) { +//printf("\nMIXING DELAYED! waiting for jitter buffer to fill after being starved\n"); +//printf("samples available: %d frames available: %d\n", samplesAvailable(), samplesAvailable() / samplesPerFrame); + // if the buffer was starved and hasn't filled back up all the way, don't mix yet if (_shouldOutputStarveDebug) { _shouldOutputStarveDebug = false; } - - return false; + + return false; + } else if (samplesAvailable() < samplesPerFrame) { + +//printf("\nMIXING DELAYED! jitter buffer is starved!!!\n"); +//printf("samples available: %d frames available: %d\n", samplesAvailable(), samplesAvailable() / samplesPerFrame); + // if the buffer doesn't have a full frame of samples for mixing, it is starved _isStarved = true; // reset our _shouldOutputStarveDebug to true so the next is printed _shouldOutputStarveDebug = true; + + // if buffer was starved, we've effectively increased the jitter buffer by one frame + // by "holding back" this ring buffer's contents until the next client frame is prepared. + _currentJitterBufferFrames++; +//printf("jbuffer size increased: new size: %d\n", _currentJitterBufferFrames); return false; - } else { - // good buffer, add this to the mix - _isStarved = false; - // since we've read data from ring buffer at least once - we've started - _hasStarted = true; - - return true; } +//printf("WILL MIX\n"); - return false; + // good buffer, add this to the mix + _isStarved = false; + + // since we've read data from ring buffer at least once - we've started + _hasStarted = true; + + return true; } -void PositionalAudioRingBuffer::updateDesiredJitterBufferNumSamples() { +void PositionalAudioRingBuffer::updateDesiredJitterBufferFrames() { const float USECS_PER_FRAME = NETWORK_BUFFER_LENGTH_SAMPLES_PER_CHANNEL * USECS_PER_SECOND / (float)SAMPLE_RATE; if (_interframeTimeGapHistory.hasNewWindowMaxGapAvailable()) { - int desiredJitterBufferNumFrames = (int)((float)_interframeTimeGapHistory.getWindowMaxGap() / USECS_PER_FRAME + 0.5f); - _desiredJitterBufferNumSamples = desiredJitterBufferNumFrames * getNumSamplesPerFrame(); + _desiredJitterBufferFrames = ceilf((float)_interframeTimeGapHistory.getWindowMaxGap() / USECS_PER_FRAME); + if (_desiredJitterBufferFrames < 1) { + _desiredJitterBufferFrames = 1; + } } } \ No newline at end of file diff --git a/libraries/audio/src/PositionalAudioRingBuffer.h b/libraries/audio/src/PositionalAudioRingBuffer.h index de731c6136..87a53038a3 100644 --- a/libraries/audio/src/PositionalAudioRingBuffer.h +++ b/libraries/audio/src/PositionalAudioRingBuffer.h @@ -29,6 +29,7 @@ public: void frameReceived(); bool hasNewWindowMaxGapAvailable() const { return _newWindowMaxGapAvailable; } + quint64 peekWindowMaxGap() const { return _windowMaxGap; } quint64 getWindowMaxGap(); private: @@ -75,14 +76,14 @@ public: AABox* getListenerUnattenuatedZone() const { return _listenerUnattenuatedZone; } void setListenerUnattenuatedZone(AABox* listenerUnattenuatedZone) { _listenerUnattenuatedZone = listenerUnattenuatedZone; } - int getNumSamplesPerFrame() const { return _isStereo ? NETWORK_BUFFER_LENGTH_SAMPLES_STEREO : NETWORK_BUFFER_LENGTH_BYTES_PER_CHANNEL; } + int getSamplesPerFrame() const { return _isStereo ? NETWORK_BUFFER_LENGTH_SAMPLES_STEREO : NETWORK_BUFFER_LENGTH_SAMPLES_PER_CHANNEL; } protected: // disallow copying of PositionalAudioRingBuffer objects PositionalAudioRingBuffer(const PositionalAudioRingBuffer&); PositionalAudioRingBuffer& operator= (const PositionalAudioRingBuffer&); - void updateDesiredJitterBufferNumSamples(); + void updateDesiredJitterBufferFrames(); PositionalAudioRingBuffer::Type _type; glm::vec3 _position; @@ -96,7 +97,10 @@ protected: AABox* _listenerUnattenuatedZone; InterframeTimeGapHistory _interframeTimeGapHistory; - int _desiredJitterBufferNumSamples; + int _desiredJitterBufferFrames; + int _currentJitterBufferFrames; + +quint64 _lastMixTime; }; #endif // hifi_PositionalAudioRingBuffer_h From cb6681f6f88e21196e97d3f6c5dd65a046cdd71d Mon Sep 17 00:00:00 2001 From: wangyix Date: Fri, 20 Jun 2014 17:13:42 -0700 Subject: [PATCH 08/14] completed code for dynamicly resizing jitter buffer --- libraries/audio/src/AudioRingBuffer.cpp | 2 +- .../audio/src/PositionalAudioRingBuffer.cpp | 81 ++++++++++++++----- 2 files changed, 60 insertions(+), 23 deletions(-) diff --git a/libraries/audio/src/AudioRingBuffer.cpp b/libraries/audio/src/AudioRingBuffer.cpp index 7b1e68ffbd..ae100dcdf1 100644 --- a/libraries/audio/src/AudioRingBuffer.cpp +++ b/libraries/audio/src/AudioRingBuffer.cpp @@ -158,7 +158,7 @@ const int16_t& AudioRingBuffer::operator[] (const int index) const { void AudioRingBuffer::shiftReadPosition(unsigned int numSamples) { _nextOutput = shiftedPositionAccomodatingWrap(_nextOutput, numSamples); - printf("\n mixed. %d samples remaining\n", samplesAvailable()); +//printf("\nmixed. %d samples remaining\n", samplesAvailable()); } unsigned int AudioRingBuffer::samplesAvailable() const { diff --git a/libraries/audio/src/PositionalAudioRingBuffer.cpp b/libraries/audio/src/PositionalAudioRingBuffer.cpp index 813323ecb1..04559fc910 100644 --- a/libraries/audio/src/PositionalAudioRingBuffer.cpp +++ b/libraries/audio/src/PositionalAudioRingBuffer.cpp @@ -33,8 +33,7 @@ InterframeTimeGapHistory::InterframeTimeGapHistory() } void InterframeTimeGapHistory::frameReceived() { - - static QQueue gaps; + static quint64 gapsSum = 0; quint64 now = usecTimestampNow(); @@ -46,6 +45,14 @@ void InterframeTimeGapHistory::frameReceived() { // update the current interval max if (gap > _currentIntervalMaxGap) { _currentIntervalMaxGap = gap; + + // keep the window max gap at least as large as the current interval max + // this allows the window max gap to respond immediately to a sudden spike in gap times + // also, this prevents the window max gap from staying at 0 until the first interval of samples filled up + if (_currentIntervalMaxGap > _windowMaxGap) { + _windowMaxGap = _currentIntervalMaxGap; + _newWindowMaxGapAvailable = true; + } } _numSamplesInCurrentInterval++; @@ -116,17 +123,35 @@ int PositionalAudioRingBuffer::parseData(const QByteArray& packet) { readBytes += sizeof(int16_t); if (numSilentSamples > 0) { - addSilentFrame(numSilentSamples); + + if (_currentJitterBufferFrames > _desiredJitterBufferFrames) { + // our current jitter buffer size exceeds its desired value, so ignore some silent + // frames to get that size as close to desired as possible + int samplesPerFrame = getSamplesPerFrame(); + int numSilentFrames = numSilentSamples / samplesPerFrame; + int numFramesToDropDesired = _currentJitterBufferFrames - _desiredJitterBufferFrames; + + if (numSilentFrames > numFramesToDropDesired) { + // we have more than enough frames to drop to get the jitter buffer to its desired length + int numSilentFramesToAdd = numSilentFrames - numFramesToDropDesired; + addSilentFrame(numSilentFramesToAdd * samplesPerFrame); + _currentJitterBufferFrames = _desiredJitterBufferFrames; + + } else { + // we need to drop all frames to get the jitter buffer close as possible to its desired length + _currentJitterBufferFrames -= numSilentFrames; + printf("dropped silent packet! jbuffer size is now %d ###############", _currentJitterBufferFrames); + } + } else { + addSilentFrame(numSilentSamples); + } } - printf("\nparsed silent packet of %d samples\n", numSilentSamples); } else { // there is audio data to read int dataBytes = writeData(packet.data() + readBytes, packet.size() - readBytes); readBytes += dataBytes; - - printf("\nparsed packet of %d data bytes\n", dataBytes); } - printf("%d samples available\n", samplesAvailable()); +//printf("parsed packet. %d samples available\n", samplesAvailable()); return readBytes; } @@ -174,16 +199,18 @@ void PositionalAudioRingBuffer::updateNextOutputTrailingLoudness() { } bool PositionalAudioRingBuffer::shouldBeAddedToMix() { - int samplesPerFrame = getSamplesPerFrame(); - int currentJitterBufferSamples = 3 * samplesPerFrame; //_currentJitterBufferFrames * samplesPerFrame; + int desiredJitterBufferSamples = _desiredJitterBufferFrames * samplesPerFrame; //printf("\nsamples available: %d frames available: %d\n", samplesAvailable(), samplesAvailable() / samplesPerFrame); - if (!isNotStarvedOrHasMinimumSamples(samplesPerFrame + currentJitterBufferSamples)) { + if (!isNotStarvedOrHasMinimumSamples(samplesPerFrame + _desiredJitterBufferFrames)) { + +printf("\nMIXING DELAYED! waiting for jitter buffer to fill after being starved\n"); +printf("samples available: %d frames available: %d\n", samplesAvailable(), samplesAvailable() / samplesPerFrame); + + // if the buffer was starved, allow it to accrue at least the desired number of + // jitter buffer frames before we start taking frames from it for mixing -//printf("\nMIXING DELAYED! waiting for jitter buffer to fill after being starved\n"); -//printf("samples available: %d frames available: %d\n", samplesAvailable(), samplesAvailable() / samplesPerFrame); - // if the buffer was starved and hasn't filled back up all the way, don't mix yet if (_shouldOutputStarveDebug) { _shouldOutputStarveDebug = false; } @@ -192,26 +219,34 @@ bool PositionalAudioRingBuffer::shouldBeAddedToMix() { } else if (samplesAvailable() < samplesPerFrame) { -//printf("\nMIXING DELAYED! jitter buffer is starved!!!\n"); -//printf("samples available: %d frames available: %d\n", samplesAvailable(), samplesAvailable() / samplesPerFrame); - // if the buffer doesn't have a full frame of samples for mixing, it is starved +printf("\nMIXING DELAYED! jitter buffer is starved!!!\n"); +printf("samples available: %d frames available: %d\n", samplesAvailable(), samplesAvailable() / samplesPerFrame); + + // if the buffer doesn't have a full frame of samples to take for mixing, it is starved _isStarved = true; + + // set to 0 to indicate the jitter buffer is starved + _currentJitterBufferFrames = 0; // reset our _shouldOutputStarveDebug to true so the next is printed _shouldOutputStarveDebug = true; - // if buffer was starved, we've effectively increased the jitter buffer by one frame - // by "holding back" this ring buffer's contents until the next client frame is prepared. - _currentJitterBufferFrames++; -//printf("jbuffer size increased: new size: %d\n", _currentJitterBufferFrames); - return false; } //printf("WILL MIX\n"); // good buffer, add this to the mix - _isStarved = false; + + if (_isStarved) { + + // if this buffer was just finished replenishing after being starved, the number of frames in it now + // is the length of the jitter buffer + _currentJitterBufferFrames = samplesAvailable() / samplesPerFrame; + printf("jitter buffer length is now %d; desired: %d --------\n", _currentJitterBufferFrames, _desiredJitterBufferFrames); + + _isStarved = false; + } // since we've read data from ring buffer at least once - we've started _hasStarted = true; @@ -228,5 +263,7 @@ void PositionalAudioRingBuffer::updateDesiredJitterBufferFrames() { if (_desiredJitterBufferFrames < 1) { _desiredJitterBufferFrames = 1; } + + printf(" desired jbuffer size is now %d --------------------------------\n", _desiredJitterBufferFrames); } } \ No newline at end of file From 124fba35db2e4d2330234d35e6f144c6469cfb37 Mon Sep 17 00:00:00 2001 From: wangyix Date: Fri, 20 Jun 2014 17:39:05 -0700 Subject: [PATCH 09/14] fixed minor comment typo --- libraries/audio/src/PositionalAudioRingBuffer.cpp | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/libraries/audio/src/PositionalAudioRingBuffer.cpp b/libraries/audio/src/PositionalAudioRingBuffer.cpp index 04559fc910..adadf5af91 100644 --- a/libraries/audio/src/PositionalAudioRingBuffer.cpp +++ b/libraries/audio/src/PositionalAudioRingBuffer.cpp @@ -240,7 +240,7 @@ printf("samples available: %d frames available: %d\n", samplesAvailable(), samp if (_isStarved) { - // if this buffer was just finished replenishing after being starved, the number of frames in it now + // if this buffer has just finished replenishing after being starved, the number of frames in it now // is the length of the jitter buffer _currentJitterBufferFrames = samplesAvailable() / samplesPerFrame; printf("jitter buffer length is now %d; desired: %d --------\n", _currentJitterBufferFrames, _desiredJitterBufferFrames); From 6b72b161d3f7ac33838369f51e3a9756bffe860b Mon Sep 17 00:00:00 2001 From: wangyix Date: Fri, 20 Jun 2014 17:52:46 -0700 Subject: [PATCH 10/14] fixed typo with frames/samples confusion. --- libraries/audio/src/PositionalAudioRingBuffer.cpp | 8 ++++---- 1 file changed, 4 insertions(+), 4 deletions(-) diff --git a/libraries/audio/src/PositionalAudioRingBuffer.cpp b/libraries/audio/src/PositionalAudioRingBuffer.cpp index adadf5af91..93f68b4dff 100644 --- a/libraries/audio/src/PositionalAudioRingBuffer.cpp +++ b/libraries/audio/src/PositionalAudioRingBuffer.cpp @@ -140,7 +140,7 @@ int PositionalAudioRingBuffer::parseData(const QByteArray& packet) { } else { // we need to drop all frames to get the jitter buffer close as possible to its desired length _currentJitterBufferFrames -= numSilentFrames; - printf("dropped silent packet! jbuffer size is now %d ###############", _currentJitterBufferFrames); + printf("dropped silent packet! jbuffer size is now %d ###############\n", _currentJitterBufferFrames); } } else { addSilentFrame(numSilentSamples); @@ -203,7 +203,7 @@ bool PositionalAudioRingBuffer::shouldBeAddedToMix() { int desiredJitterBufferSamples = _desiredJitterBufferFrames * samplesPerFrame; //printf("\nsamples available: %d frames available: %d\n", samplesAvailable(), samplesAvailable() / samplesPerFrame); - if (!isNotStarvedOrHasMinimumSamples(samplesPerFrame + _desiredJitterBufferFrames)) { + if (!isNotStarvedOrHasMinimumSamples(samplesPerFrame + desiredJitterBufferSamples)) { printf("\nMIXING DELAYED! waiting for jitter buffer to fill after being starved\n"); printf("samples available: %d frames available: %d\n", samplesAvailable(), samplesAvailable() / samplesPerFrame); @@ -241,8 +241,8 @@ printf("samples available: %d frames available: %d\n", samplesAvailable(), samp if (_isStarved) { // if this buffer has just finished replenishing after being starved, the number of frames in it now - // is the length of the jitter buffer - _currentJitterBufferFrames = samplesAvailable() / samplesPerFrame; + // minus one (since a frame will be read immediately after this) is the length of the jitter buffer + _currentJitterBufferFrames = samplesAvailable() / samplesPerFrame - 1; printf("jitter buffer length is now %d; desired: %d --------\n", _currentJitterBufferFrames, _desiredJitterBufferFrames); _isStarved = false; From 7196814ec3b2a087f966816becd7803475f5c810 Mon Sep 17 00:00:00 2001 From: wangyix Date: Mon, 23 Jun 2014 10:22:16 -0700 Subject: [PATCH 11/14] added debug prints to look at injected ring buffers --- assignment-client/src/audio/AudioMixerClientData.cpp | 2 ++ libraries/audio/src/AudioRingBuffer.cpp | 10 +++++++--- libraries/audio/src/PositionalAudioRingBuffer.cpp | 2 -- 3 files changed, 9 insertions(+), 5 deletions(-) diff --git a/assignment-client/src/audio/AudioMixerClientData.cpp b/assignment-client/src/audio/AudioMixerClientData.cpp index 0c41cc70f9..0080af4716 100644 --- a/assignment-client/src/audio/AudioMixerClientData.cpp +++ b/assignment-client/src/audio/AudioMixerClientData.cpp @@ -90,6 +90,7 @@ int AudioMixerClientData::parseData(const QByteArray& packet) { // we don't have a matching injected audio ring buffer, so add it matchingInjectedRingBuffer = new InjectedAudioRingBuffer(streamIdentifier); _ringBuffers.push_back(matchingInjectedRingBuffer); +printf("created InjectedRingBuffer %p\n", (void*)matchingInjectedRingBuffer); } matchingInjectedRingBuffer->parseData(packet); @@ -132,6 +133,7 @@ void AudioMixerClientData::pushBuffersAfterFrameSend() { && audioBuffer->hasStarted() && audioBuffer->isStarved()) { // this is an empty audio buffer that has starved, safe to delete delete audioBuffer; +printf("deleted injector %p\n", (void*)audioBuffer); i = _ringBuffers.erase(i); continue; } diff --git a/libraries/audio/src/AudioRingBuffer.cpp b/libraries/audio/src/AudioRingBuffer.cpp index ae100dcdf1..8ce48a39e3 100644 --- a/libraries/audio/src/AudioRingBuffer.cpp +++ b/libraries/audio/src/AudioRingBuffer.cpp @@ -125,16 +125,19 @@ qint64 AudioRingBuffer::writeData(const char* data, qint64 maxSize) { std::less less; std::less_equal lessEqual; + // TODO: why is less(_endOfLastWrite, _nextOutput) a condition here? if (_hasStarted + /* && (less(_endOfLastWrite, _nextOutput) - && lessEqual(_nextOutput, shiftedPositionAccomodatingWrap(_endOfLastWrite, samplesToCopy)))) { + && lessEqual(_nextOutput, shiftedPositionAccomodatingWrap(_endOfLastWrite, samplesToCopy)))*/ + && samplesToCopy > _sampleCapacity - samplesAvailable()) { // this read will cross the next output, so call us starved and reset the buffer qDebug() << "Filled the ring buffer. Resetting."; _endOfLastWrite = _buffer; _nextOutput = _buffer; _isStarved = true; } - + if (_endOfLastWrite + samplesToCopy <= _buffer + _sampleCapacity) { memcpy(_endOfLastWrite, data, samplesToCopy * sizeof(int16_t)); } else { @@ -145,6 +148,7 @@ qint64 AudioRingBuffer::writeData(const char* data, qint64 maxSize) { _endOfLastWrite = shiftedPositionAccomodatingWrap(_endOfLastWrite, samplesToCopy); +printf("%p: writeData. %d samples available\n", (void*)this, samplesAvailable()); return samplesToCopy * sizeof(int16_t); } @@ -158,7 +162,7 @@ const int16_t& AudioRingBuffer::operator[] (const int index) const { void AudioRingBuffer::shiftReadPosition(unsigned int numSamples) { _nextOutput = shiftedPositionAccomodatingWrap(_nextOutput, numSamples); -//printf("\nmixed. %d samples remaining\n", samplesAvailable()); +printf("%p: mixed. %d samples remaining\n", (void*)this, samplesAvailable()); } unsigned int AudioRingBuffer::samplesAvailable() const { diff --git a/libraries/audio/src/PositionalAudioRingBuffer.cpp b/libraries/audio/src/PositionalAudioRingBuffer.cpp index 93f68b4dff..03ac36d09e 100644 --- a/libraries/audio/src/PositionalAudioRingBuffer.cpp +++ b/libraries/audio/src/PositionalAudioRingBuffer.cpp @@ -151,8 +151,6 @@ int PositionalAudioRingBuffer::parseData(const QByteArray& packet) { int dataBytes = writeData(packet.data() + readBytes, packet.size() - readBytes); readBytes += dataBytes; } -//printf("parsed packet. %d samples available\n", samplesAvailable()); - return readBytes; } From e5d7d471e34b7037b9726c381f00d36247d880ff Mon Sep 17 00:00:00 2001 From: wangyix Date: Mon, 23 Jun 2014 10:55:30 -0700 Subject: [PATCH 12/14] removed debug code --- .../src/audio/AudioMixerClientData.cpp | 2 -- interface/src/Audio.cpp | 2 -- libraries/audio/src/AudioRingBuffer.cpp | 13 ++----- libraries/audio/src/AudioRingBuffer.h | 3 +- .../audio/src/PositionalAudioRingBuffer.cpp | 36 ++++--------------- .../audio/src/PositionalAudioRingBuffer.h | 3 -- 6 files changed, 10 insertions(+), 49 deletions(-) diff --git a/assignment-client/src/audio/AudioMixerClientData.cpp b/assignment-client/src/audio/AudioMixerClientData.cpp index 0080af4716..0c41cc70f9 100644 --- a/assignment-client/src/audio/AudioMixerClientData.cpp +++ b/assignment-client/src/audio/AudioMixerClientData.cpp @@ -90,7 +90,6 @@ int AudioMixerClientData::parseData(const QByteArray& packet) { // we don't have a matching injected audio ring buffer, so add it matchingInjectedRingBuffer = new InjectedAudioRingBuffer(streamIdentifier); _ringBuffers.push_back(matchingInjectedRingBuffer); -printf("created InjectedRingBuffer %p\n", (void*)matchingInjectedRingBuffer); } matchingInjectedRingBuffer->parseData(packet); @@ -133,7 +132,6 @@ void AudioMixerClientData::pushBuffersAfterFrameSend() { && audioBuffer->hasStarted() && audioBuffer->isStarved()) { // this is an empty audio buffer that has starved, safe to delete delete audioBuffer; -printf("deleted injector %p\n", (void*)audioBuffer); i = _ringBuffers.erase(i); continue; } diff --git a/interface/src/Audio.cpp b/interface/src/Audio.cpp index 47e55ddc90..a0a85f8888 100644 --- a/interface/src/Audio.cpp +++ b/interface/src/Audio.cpp @@ -622,7 +622,6 @@ void Audio::handleAudioInput() { SharedNodePointer audioMixer = nodeList->soloNodeOfType(NodeType::AudioMixer); if (audioMixer && audioMixer->getActiveSocket()) { - MyAvatar* interfaceAvatar = Application::getInstance()->getAvatar(); glm::vec3 headPosition = interfaceAvatar->getHead()->getPosition(); glm::quat headOrientation = interfaceAvatar->getHead()->getFinalOrientationInWorldFrame(); @@ -639,7 +638,6 @@ void Audio::handleAudioInput() { numAudioBytes = sizeof(int16_t); } else { numAudioBytes = numNetworkBytes; - //_isStereoInput ? NETWORK_BUFFER_LENGTH_BYTES_STEREO : NETWORK_BUFFER_LENGTH_BYTES_PER_CHANNEL; if (Menu::getInstance()->isOptionChecked(MenuOption::EchoServerAudio)) { packetType = PacketTypeMicrophoneAudioWithEcho; diff --git a/libraries/audio/src/AudioRingBuffer.cpp b/libraries/audio/src/AudioRingBuffer.cpp index 8ce48a39e3..1b6bdaa5d8 100644 --- a/libraries/audio/src/AudioRingBuffer.cpp +++ b/libraries/audio/src/AudioRingBuffer.cpp @@ -124,13 +124,8 @@ qint64 AudioRingBuffer::writeData(const char* data, qint64 maxSize) { std::less less; std::less_equal lessEqual; - - // TODO: why is less(_endOfLastWrite, _nextOutput) a condition here? - if (_hasStarted - /* - && (less(_endOfLastWrite, _nextOutput) - && lessEqual(_nextOutput, shiftedPositionAccomodatingWrap(_endOfLastWrite, samplesToCopy)))*/ - && samplesToCopy > _sampleCapacity - samplesAvailable()) { + + if (_hasStarted && samplesToCopy > _sampleCapacity - samplesAvailable()) { // this read will cross the next output, so call us starved and reset the buffer qDebug() << "Filled the ring buffer. Resetting."; _endOfLastWrite = _buffer; @@ -147,8 +142,7 @@ qint64 AudioRingBuffer::writeData(const char* data, qint64 maxSize) { } _endOfLastWrite = shiftedPositionAccomodatingWrap(_endOfLastWrite, samplesToCopy); - -printf("%p: writeData. %d samples available\n", (void*)this, samplesAvailable()); + return samplesToCopy * sizeof(int16_t); } @@ -162,7 +156,6 @@ const int16_t& AudioRingBuffer::operator[] (const int index) const { void AudioRingBuffer::shiftReadPosition(unsigned int numSamples) { _nextOutput = shiftedPositionAccomodatingWrap(_nextOutput, numSamples); -printf("%p: mixed. %d samples remaining\n", (void*)this, samplesAvailable()); } unsigned int AudioRingBuffer::samplesAvailable() const { diff --git a/libraries/audio/src/AudioRingBuffer.h b/libraries/audio/src/AudioRingBuffer.h index 3d2864f373..04cc67c8ac 100644 --- a/libraries/audio/src/AudioRingBuffer.h +++ b/libraries/audio/src/AudioRingBuffer.h @@ -74,14 +74,13 @@ public: bool hasStarted() const { return _hasStarted; } void addSilentFrame(int numSilentSamples); - protected: // disallow copying of AudioRingBuffer objects AudioRingBuffer(const AudioRingBuffer&); AudioRingBuffer& operator= (const AudioRingBuffer&); int16_t* shiftedPositionAccomodatingWrap(int16_t* position, int numSamplesShift) const; - + int _sampleCapacity; int _numFrameSamples; int16_t* _nextOutput; diff --git a/libraries/audio/src/PositionalAudioRingBuffer.cpp b/libraries/audio/src/PositionalAudioRingBuffer.cpp index 03ac36d09e..7c2369d7e5 100644 --- a/libraries/audio/src/PositionalAudioRingBuffer.cpp +++ b/libraries/audio/src/PositionalAudioRingBuffer.cpp @@ -33,12 +33,9 @@ InterframeTimeGapHistory::InterframeTimeGapHistory() } void InterframeTimeGapHistory::frameReceived() { - - static quint64 gapsSum = 0; - quint64 now = usecTimestampNow(); - // make sure this isn't the first time frameReceived() is called, meaning there's actually a gap to calculate. + // make sure this isn't the first time frameReceived() is called so can actually calculate a gap. if (_lastFrameReceivedTime != 0) { quint64 gap = now - _lastFrameReceivedTime; @@ -123,7 +120,6 @@ int PositionalAudioRingBuffer::parseData(const QByteArray& packet) { readBytes += sizeof(int16_t); if (numSilentSamples > 0) { - if (_currentJitterBufferFrames > _desiredJitterBufferFrames) { // our current jitter buffer size exceeds its desired value, so ignore some silent // frames to get that size as close to desired as possible @@ -140,7 +136,6 @@ int PositionalAudioRingBuffer::parseData(const QByteArray& packet) { } else { // we need to drop all frames to get the jitter buffer close as possible to its desired length _currentJitterBufferFrames -= numSilentFrames; - printf("dropped silent packet! jbuffer size is now %d ###############\n", _currentJitterBufferFrames); } } else { addSilentFrame(numSilentSamples); @@ -148,8 +143,7 @@ int PositionalAudioRingBuffer::parseData(const QByteArray& packet) { } } else { // there is audio data to read - int dataBytes = writeData(packet.data() + readBytes, packet.size() - readBytes); - readBytes += dataBytes; + readBytes += writeData(packet.data() + readBytes, packet.size() - readBytes); } return readBytes; } @@ -200,12 +194,7 @@ bool PositionalAudioRingBuffer::shouldBeAddedToMix() { int samplesPerFrame = getSamplesPerFrame(); int desiredJitterBufferSamples = _desiredJitterBufferFrames * samplesPerFrame; -//printf("\nsamples available: %d frames available: %d\n", samplesAvailable(), samplesAvailable() / samplesPerFrame); if (!isNotStarvedOrHasMinimumSamples(samplesPerFrame + desiredJitterBufferSamples)) { - -printf("\nMIXING DELAYED! waiting for jitter buffer to fill after being starved\n"); -printf("samples available: %d frames available: %d\n", samplesAvailable(), samplesAvailable() / samplesPerFrame); - // if the buffer was starved, allow it to accrue at least the desired number of // jitter buffer frames before we start taking frames from it for mixing @@ -214,15 +203,10 @@ printf("samples available: %d frames available: %d\n", samplesAvailable(), samp } return false; - - } else if (samplesAvailable() < samplesPerFrame) { - -printf("\nMIXING DELAYED! jitter buffer is starved!!!\n"); -printf("samples available: %d frames available: %d\n", samplesAvailable(), samplesAvailable() / samplesPerFrame); - + } else if (samplesAvailable() < samplesPerFrame) { // if the buffer doesn't have a full frame of samples to take for mixing, it is starved _isStarved = true; - + // set to 0 to indicate the jitter buffer is starved _currentJitterBufferFrames = 0; @@ -230,19 +214,13 @@ printf("samples available: %d frames available: %d\n", samplesAvailable(), samp _shouldOutputStarveDebug = true; return false; - } -//printf("WILL MIX\n"); - + // good buffer, add this to the mix - if (_isStarved) { - // if this buffer has just finished replenishing after being starved, the number of frames in it now // minus one (since a frame will be read immediately after this) is the length of the jitter buffer _currentJitterBufferFrames = samplesAvailable() / samplesPerFrame - 1; - printf("jitter buffer length is now %d; desired: %d --------\n", _currentJitterBufferFrames, _desiredJitterBufferFrames); - _isStarved = false; } @@ -261,7 +239,5 @@ void PositionalAudioRingBuffer::updateDesiredJitterBufferFrames() { if (_desiredJitterBufferFrames < 1) { _desiredJitterBufferFrames = 1; } - - printf(" desired jbuffer size is now %d --------------------------------\n", _desiredJitterBufferFrames); } -} \ No newline at end of file +} diff --git a/libraries/audio/src/PositionalAudioRingBuffer.h b/libraries/audio/src/PositionalAudioRingBuffer.h index 87a53038a3..5fa24ff782 100644 --- a/libraries/audio/src/PositionalAudioRingBuffer.h +++ b/libraries/audio/src/PositionalAudioRingBuffer.h @@ -43,7 +43,6 @@ private: bool _newWindowMaxGapAvailable; }; - class PositionalAudioRingBuffer : public AudioRingBuffer { public: enum Type { @@ -99,8 +98,6 @@ protected: InterframeTimeGapHistory _interframeTimeGapHistory; int _desiredJitterBufferFrames; int _currentJitterBufferFrames; - -quint64 _lastMixTime; }; #endif // hifi_PositionalAudioRingBuffer_h From ffc20b8876e86fea8364a7535870ad9e8b4f8c6d Mon Sep 17 00:00:00 2001 From: wangyix Date: Mon, 23 Jun 2014 11:22:04 -0700 Subject: [PATCH 13/14] changed InterframeTimeGapHistory to InterframeTimeGapStats --- assignment-client/src/audio/AvatarAudioRingBuffer.cpp | 2 +- libraries/audio/src/InjectedAudioRingBuffer.cpp | 2 +- libraries/audio/src/PositionalAudioRingBuffer.cpp | 10 +++++----- libraries/audio/src/PositionalAudioRingBuffer.h | 6 +++--- 4 files changed, 10 insertions(+), 10 deletions(-) diff --git a/assignment-client/src/audio/AvatarAudioRingBuffer.cpp b/assignment-client/src/audio/AvatarAudioRingBuffer.cpp index 0df50f99db..79ad7ae411 100644 --- a/assignment-client/src/audio/AvatarAudioRingBuffer.cpp +++ b/assignment-client/src/audio/AvatarAudioRingBuffer.cpp @@ -19,7 +19,7 @@ AvatarAudioRingBuffer::AvatarAudioRingBuffer(bool isStereo) : } int AvatarAudioRingBuffer::parseData(const QByteArray& packet) { - _interframeTimeGapHistory.frameReceived(); + _interframeTimeGapStats.frameReceived(); updateDesiredJitterBufferFrames(); _shouldLoopbackForNode = (packetTypeForPacket(packet) == PacketTypeMicrophoneAudioWithEcho); diff --git a/libraries/audio/src/InjectedAudioRingBuffer.cpp b/libraries/audio/src/InjectedAudioRingBuffer.cpp index 80bcda5acb..ed0476f6bd 100644 --- a/libraries/audio/src/InjectedAudioRingBuffer.cpp +++ b/libraries/audio/src/InjectedAudioRingBuffer.cpp @@ -31,7 +31,7 @@ InjectedAudioRingBuffer::InjectedAudioRingBuffer(const QUuid& streamIdentifier) const uchar MAX_INJECTOR_VOLUME = 255; int InjectedAudioRingBuffer::parseData(const QByteArray& packet) { - _interframeTimeGapHistory.frameReceived(); + _interframeTimeGapStats.frameReceived(); updateDesiredJitterBufferFrames(); // setup a data stream to read from this packet diff --git a/libraries/audio/src/PositionalAudioRingBuffer.cpp b/libraries/audio/src/PositionalAudioRingBuffer.cpp index 7c2369d7e5..a22f57f3d7 100644 --- a/libraries/audio/src/PositionalAudioRingBuffer.cpp +++ b/libraries/audio/src/PositionalAudioRingBuffer.cpp @@ -21,7 +21,7 @@ #include "PositionalAudioRingBuffer.h" #include "SharedUtil.h" -InterframeTimeGapHistory::InterframeTimeGapHistory() +InterframeTimeGapStats::InterframeTimeGapStats() : _lastFrameReceivedTime(0), _numSamplesInCurrentInterval(0), _currentIntervalMaxGap(0), @@ -32,7 +32,7 @@ InterframeTimeGapHistory::InterframeTimeGapHistory() memset(_intervalMaxGaps, 0, TIME_GAP_NUM_INTERVALS_IN_WINDOW*sizeof(quint64)); } -void InterframeTimeGapHistory::frameReceived() { +void InterframeTimeGapStats::frameReceived() { quint64 now = usecTimestampNow(); // make sure this isn't the first time frameReceived() is called so can actually calculate a gap. @@ -79,7 +79,7 @@ void InterframeTimeGapHistory::frameReceived() { _lastFrameReceivedTime = now; } -quint64 InterframeTimeGapHistory::getWindowMaxGap() { +quint64 InterframeTimeGapStats::getWindowMaxGap() { _newWindowMaxGapAvailable = false; return _windowMaxGap; } @@ -234,8 +234,8 @@ void PositionalAudioRingBuffer::updateDesiredJitterBufferFrames() { const float USECS_PER_FRAME = NETWORK_BUFFER_LENGTH_SAMPLES_PER_CHANNEL * USECS_PER_SECOND / (float)SAMPLE_RATE; - if (_interframeTimeGapHistory.hasNewWindowMaxGapAvailable()) { - _desiredJitterBufferFrames = ceilf((float)_interframeTimeGapHistory.getWindowMaxGap() / USECS_PER_FRAME); + if (_interframeTimeGapStats.hasNewWindowMaxGapAvailable()) { + _desiredJitterBufferFrames = ceilf((float)_interframeTimeGapStats.getWindowMaxGap() / USECS_PER_FRAME); if (_desiredJitterBufferFrames < 1) { _desiredJitterBufferFrames = 1; } diff --git a/libraries/audio/src/PositionalAudioRingBuffer.h b/libraries/audio/src/PositionalAudioRingBuffer.h index 5fa24ff782..8c2122f29e 100644 --- a/libraries/audio/src/PositionalAudioRingBuffer.h +++ b/libraries/audio/src/PositionalAudioRingBuffer.h @@ -23,9 +23,9 @@ const int TIME_GAP_NUM_SAMPLES_IN_INTERVAL = 500; const int TIME_GAP_NUM_INTERVALS_IN_WINDOW = 10; // class used to track time between incoming frames for the purpose of varying the jitter buffer length -class InterframeTimeGapHistory { +class InterframeTimeGapStats { public: - InterframeTimeGapHistory(); + InterframeTimeGapStats(); void frameReceived(); bool hasNewWindowMaxGapAvailable() const { return _newWindowMaxGapAvailable; } @@ -95,7 +95,7 @@ protected: float _nextOutputTrailingLoudness; AABox* _listenerUnattenuatedZone; - InterframeTimeGapHistory _interframeTimeGapHistory; + InterframeTimeGapStats _interframeTimeGapStats; int _desiredJitterBufferFrames; int _currentJitterBufferFrames; }; From fbdca59d3777a9b9561cd212e8644f14fd5bce43 Mon Sep 17 00:00:00 2001 From: wangyix Date: Mon, 23 Jun 2014 11:56:14 -0700 Subject: [PATCH 14/14] fixed spacing issue --- libraries/audio/src/PositionalAudioRingBuffer.cpp | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/libraries/audio/src/PositionalAudioRingBuffer.cpp b/libraries/audio/src/PositionalAudioRingBuffer.cpp index a22f57f3d7..0e2e785973 100644 --- a/libraries/audio/src/PositionalAudioRingBuffer.cpp +++ b/libraries/audio/src/PositionalAudioRingBuffer.cpp @@ -29,7 +29,7 @@ InterframeTimeGapStats::InterframeTimeGapStats() _windowMaxGap(0), _newWindowMaxGapAvailable(false) { - memset(_intervalMaxGaps, 0, TIME_GAP_NUM_INTERVALS_IN_WINDOW*sizeof(quint64)); + memset(_intervalMaxGaps, 0, TIME_GAP_NUM_INTERVALS_IN_WINDOW * sizeof(quint64)); } void InterframeTimeGapStats::frameReceived() {