From fb73b6e1cea0d5ed064df6af695c27eb061f417e Mon Sep 17 00:00:00 2001 From: Stephen Birarda Date: Wed, 19 Mar 2014 17:57:01 -0700 Subject: [PATCH 1/3] collect min and max loudness for each frame --- assignment-client/src/audio/AudioMixer.cpp | 11 +++++++++-- assignment-client/src/audio/AudioMixer.h | 3 +++ .../src/audio/AudioMixerClientData.cpp | 14 +++++++++++++- assignment-client/src/audio/AudioMixerClientData.h | 2 +- 4 files changed, 26 insertions(+), 4 deletions(-) diff --git a/assignment-client/src/audio/AudioMixer.cpp b/assignment-client/src/audio/AudioMixer.cpp index 988bcc1da7..2bece55ac4 100644 --- a/assignment-client/src/audio/AudioMixer.cpp +++ b/assignment-client/src/audio/AudioMixer.cpp @@ -62,7 +62,9 @@ void attachNewBufferToNode(Node *newNode) { } AudioMixer::AudioMixer(const QByteArray& packet) : - ThreadedAssignment(packet) + ThreadedAssignment(packet), + _minSourceLoudnessInFrame(1.0f), + _maxSourceLoudnessInFrame(0.0f) { } @@ -353,9 +355,14 @@ void AudioMixer::run() { while (!_isFinished) { + _minSourceLoudnessInFrame = 1.0f; + _maxSourceLoudnessInFrame = 0.0f; + foreach (const SharedNodePointer& node, nodeList->getNodeHash()) { if (node->getLinkedData()) { - ((AudioMixerClientData*) node->getLinkedData())->checkBuffersBeforeFrameSend(JITTER_BUFFER_SAMPLES); + ((AudioMixerClientData*) node->getLinkedData())->checkBuffersBeforeFrameSend(JITTER_BUFFER_SAMPLES, + _minSourceLoudnessInFrame, + _maxSourceLoudnessInFrame); } } diff --git a/assignment-client/src/audio/AudioMixer.h b/assignment-client/src/audio/AudioMixer.h index 5a68b0023f..6056dbd318 100644 --- a/assignment-client/src/audio/AudioMixer.h +++ b/assignment-client/src/audio/AudioMixer.h @@ -38,6 +38,9 @@ private: // client samples capacity is larger than what will be sent to optimize mixing int16_t _clientSamples[NETWORK_BUFFER_LENGTH_SAMPLES_STEREO + SAMPLE_PHASE_DELAY_AT_90]; + + float _minSourceLoudnessInFrame; + float _maxSourceLoudnessInFrame; }; #endif /* defined(__hifi__AudioMixer__) */ diff --git a/assignment-client/src/audio/AudioMixerClientData.cpp b/assignment-client/src/audio/AudioMixerClientData.cpp index b2da0a0aaa..9b8cdb8709 100644 --- a/assignment-client/src/audio/AudioMixerClientData.cpp +++ b/assignment-client/src/audio/AudioMixerClientData.cpp @@ -6,6 +6,8 @@ // Copyright (c) 2013 HighFidelity, Inc. All rights reserved. // +#include + #include #include @@ -82,7 +84,9 @@ int AudioMixerClientData::parseData(const QByteArray& packet) { return 0; } -void AudioMixerClientData::checkBuffersBeforeFrameSend(int jitterBufferLengthSamples) { +void AudioMixerClientData::checkBuffersBeforeFrameSend(int jitterBufferLengthSamples, + float& currentMinLoudness, + float& currentMaxLoudness) { for (unsigned int i = 0; i < _ringBuffers.size(); i++) { if (_ringBuffers[i]->shouldBeAddedToMix(jitterBufferLengthSamples)) { // this is a ring buffer that is ready to go @@ -92,6 +96,14 @@ void AudioMixerClientData::checkBuffersBeforeFrameSend(int jitterBufferLengthSam // calculate the average loudness for the next NETWORK_BUFFER_LENGTH_SAMPLES_PER_CHANNEL // that would be mixed in _nextOutputLoudness = _ringBuffers[i]->averageLoudnessForBoundarySamples(NETWORK_BUFFER_LENGTH_SAMPLES_PER_CHANNEL); + + if (_nextOutputLoudness < currentMinLoudness) { + currentMinLoudness = _nextOutputLoudness; + } + + if (_nextOutputLoudness > currentMaxLoudness) { + currentMaxLoudness = _nextOutputLoudness; + } } } } diff --git a/assignment-client/src/audio/AudioMixerClientData.h b/assignment-client/src/audio/AudioMixerClientData.h index bb10098e23..701b2befba 100644 --- a/assignment-client/src/audio/AudioMixerClientData.h +++ b/assignment-client/src/audio/AudioMixerClientData.h @@ -27,7 +27,7 @@ public: float getNextOutputLoudness() const { return _nextOutputLoudness; } int parseData(const QByteArray& packet); - void checkBuffersBeforeFrameSend(int jitterBufferLengthSamples); + void checkBuffersBeforeFrameSend(int jitterBufferLengthSamples, float& currentMinLoudness, float& currentMaxLoudness); void pushBuffersAfterFrameSend(); private: std::vector _ringBuffers; From a8ef64e0cecd266ee46245002485d367c35dff0d Mon Sep 17 00:00:00 2001 From: Stephen Birarda Date: Thu, 20 Mar 2014 12:52:17 -0700 Subject: [PATCH 2/3] calculate a cutoff loudness for mixer recovery --- assignment-client/src/audio/AudioMixer.cpp | 42 ++++++++++++++++++- assignment-client/src/audio/AudioMixer.h | 5 ++- .../src/audio/AudioMixerClientData.cpp | 2 +- .../audio/src/PositionalAudioRingBuffer.cpp | 1 + 4 files changed, 46 insertions(+), 4 deletions(-) diff --git a/assignment-client/src/audio/AudioMixer.cpp b/assignment-client/src/audio/AudioMixer.cpp index 2bece55ac4..1f403e0f3d 100644 --- a/assignment-client/src/audio/AudioMixer.cpp +++ b/assignment-client/src/audio/AudioMixer.cpp @@ -64,7 +64,9 @@ void attachNewBufferToNode(Node *newNode) { AudioMixer::AudioMixer(const QByteArray& packet) : ThreadedAssignment(packet), _minSourceLoudnessInFrame(1.0f), - _maxSourceLoudnessInFrame(0.0f) + _maxSourceLoudnessInFrame(0.0f), + _loudnessCutoffRatio(0.0f), + _minRequiredLoudness(0.0f) { } @@ -352,6 +354,9 @@ void AudioMixer::run() { char* clientMixBuffer = new char[NETWORK_BUFFER_LENGTH_BYTES_STEREO + numBytesForPacketHeaderGivenPacketType(PacketTypeMixedAudio)]; + + int usecToSleep = 0; + bool isFirstRun = true; while (!_isFinished) { @@ -365,6 +370,39 @@ void AudioMixer::run() { _maxSourceLoudnessInFrame); } } + + if (!isFirstRun) { + const float STRUGGLE_TRIGGER_SLEEP_PERCENTAGE_THRESHOLD = 0.10; + const float BACK_OFF_TRIGGER_SLEEP_PERCENTAGE_THRESHOLD = 0.30; + const float CUTOFF_EPSILON = 0.0001; + + float percentageSleep = (usecToSleep / (float) BUFFER_SEND_INTERVAL_USECS); + + float lastCutoffRatio = _loudnessCutoffRatio; + + if (percentageSleep <= STRUGGLE_TRIGGER_SLEEP_PERCENTAGE_THRESHOLD || usecToSleep < 0) { + // we're struggling - change our min required loudness to reduce some load + _loudnessCutoffRatio += (1 - _loudnessCutoffRatio) / 2; + + qDebug() << "Mixer is struggling, sleeping" << percentageSleep * 100 << "% of frame time. Old cutoff was" + << lastCutoffRatio << "and is now" << _loudnessCutoffRatio; + } else if (percentageSleep >= BACK_OFF_TRIGGER_SLEEP_PERCENTAGE_THRESHOLD && _loudnessCutoffRatio != 0) { + // we've recovered and can back off the required loudness + _loudnessCutoffRatio -= _loudnessCutoffRatio / 2; + + if (_loudnessCutoffRatio < CUTOFF_EPSILON) { + _loudnessCutoffRatio = 0; + } + + qDebug() << "Mixer is recovering, sleeping" << percentageSleep * 100 << "% of frame time. Old cutoff was" + << lastCutoffRatio << "and is now" << _loudnessCutoffRatio; + } + + // set out min required loudness from the new ratio + _minRequiredLoudness = _loudnessCutoffRatio * (_maxSourceLoudnessInFrame - _minSourceLoudnessInFrame); + } else { + isFirstRun = false; + } foreach (const SharedNodePointer& node, nodeList->getNodeHash()) { if (node->getType() == NodeType::Agent && node->getActiveSocket() && node->getLinkedData() @@ -391,7 +429,7 @@ void AudioMixer::run() { break; } - int usecToSleep = usecTimestamp(&startTime) + (++nextFrame * BUFFER_SEND_INTERVAL_USECS) - usecTimestampNow(); + usecToSleep = usecTimestamp(&startTime) + (++nextFrame * BUFFER_SEND_INTERVAL_USECS) - usecTimestampNow(); if (usecToSleep > 0) { usleep(usecToSleep); diff --git a/assignment-client/src/audio/AudioMixer.h b/assignment-client/src/audio/AudioMixer.h index 6056dbd318..3827b2917a 100644 --- a/assignment-client/src/audio/AudioMixer.h +++ b/assignment-client/src/audio/AudioMixer.h @@ -37,10 +37,13 @@ private: void prepareMixForListeningNode(Node* node); // client samples capacity is larger than what will be sent to optimize mixing - int16_t _clientSamples[NETWORK_BUFFER_LENGTH_SAMPLES_STEREO + SAMPLE_PHASE_DELAY_AT_90]; + // we are MMX adding 4 samples at a time so we need client samples to have an extra 4 + int16_t _clientSamples[NETWORK_BUFFER_LENGTH_SAMPLES_STEREO + (SAMPLE_PHASE_DELAY_AT_90 * 2)]; float _minSourceLoudnessInFrame; float _maxSourceLoudnessInFrame; + float _loudnessCutoffRatio; + float _minRequiredLoudness; }; #endif /* defined(__hifi__AudioMixer__) */ diff --git a/assignment-client/src/audio/AudioMixerClientData.cpp b/assignment-client/src/audio/AudioMixerClientData.cpp index 9b8cdb8709..6830e67aa3 100644 --- a/assignment-client/src/audio/AudioMixerClientData.cpp +++ b/assignment-client/src/audio/AudioMixerClientData.cpp @@ -97,7 +97,7 @@ void AudioMixerClientData::checkBuffersBeforeFrameSend(int jitterBufferLengthSam // that would be mixed in _nextOutputLoudness = _ringBuffers[i]->averageLoudnessForBoundarySamples(NETWORK_BUFFER_LENGTH_SAMPLES_PER_CHANNEL); - if (_nextOutputLoudness < currentMinLoudness) { + if (_nextOutputLoudness != 0 && _nextOutputLoudness < currentMinLoudness) { currentMinLoudness = _nextOutputLoudness; } diff --git a/libraries/audio/src/PositionalAudioRingBuffer.cpp b/libraries/audio/src/PositionalAudioRingBuffer.cpp index 66a27647d6..ccbe5d3f23 100644 --- a/libraries/audio/src/PositionalAudioRingBuffer.cpp +++ b/libraries/audio/src/PositionalAudioRingBuffer.cpp @@ -50,6 +50,7 @@ int PositionalAudioRingBuffer::parseData(const QByteArray& packet) { int16_t numSilentSamples; memcpy(&numSilentSamples, packet.data() + readBytes, sizeof(int16_t)); + readBytes += sizeof(int16_t); addSilentFrame(numSilentSamples); From 64f946b6405ce158ed75b3c61e762ec10b1c6bdd Mon Sep 17 00:00:00 2001 From: Stephen Birarda Date: Thu, 20 Mar 2014 13:01:52 -0700 Subject: [PATCH 3/3] require that buffers be above min loudness to be mixed in --- assignment-client/src/audio/AudioMixer.cpp | 15 ++++++++++++--- 1 file changed, 12 insertions(+), 3 deletions(-) diff --git a/assignment-client/src/audio/AudioMixer.cpp b/assignment-client/src/audio/AudioMixer.cpp index 1f403e0f3d..d8017c0a02 100644 --- a/assignment-client/src/audio/AudioMixer.cpp +++ b/assignment-client/src/audio/AudioMixer.cpp @@ -379,13 +379,15 @@ void AudioMixer::run() { float percentageSleep = (usecToSleep / (float) BUFFER_SEND_INTERVAL_USECS); float lastCutoffRatio = _loudnessCutoffRatio; + bool hasRatioChanged = false; if (percentageSleep <= STRUGGLE_TRIGGER_SLEEP_PERCENTAGE_THRESHOLD || usecToSleep < 0) { // we're struggling - change our min required loudness to reduce some load _loudnessCutoffRatio += (1 - _loudnessCutoffRatio) / 2; qDebug() << "Mixer is struggling, sleeping" << percentageSleep * 100 << "% of frame time. Old cutoff was" - << lastCutoffRatio << "and is now" << _loudnessCutoffRatio; + << lastCutoffRatio << "and is now" << _loudnessCutoffRatio; + hasRatioChanged = true; } else if (percentageSleep >= BACK_OFF_TRIGGER_SLEEP_PERCENTAGE_THRESHOLD && _loudnessCutoffRatio != 0) { // we've recovered and can back off the required loudness _loudnessCutoffRatio -= _loudnessCutoffRatio / 2; @@ -396,10 +398,17 @@ void AudioMixer::run() { qDebug() << "Mixer is recovering, sleeping" << percentageSleep * 100 << "% of frame time. Old cutoff was" << lastCutoffRatio << "and is now" << _loudnessCutoffRatio; + hasRatioChanged = true; } - // set out min required loudness from the new ratio - _minRequiredLoudness = _loudnessCutoffRatio * (_maxSourceLoudnessInFrame - _minSourceLoudnessInFrame); + if (hasRatioChanged) { + // set out min required loudness from the new ratio + _minRequiredLoudness = _loudnessCutoffRatio * (_maxSourceLoudnessInFrame - _minSourceLoudnessInFrame); + qDebug() << "Minimum loudness required to be mixed is now" << _minRequiredLoudness; + } + + + } else { isFirstRun = false; }