Merge pull request #2409 from birarda/audio-scaling

add N loudest throttling for audio-scaling

assignment-client/src/audio/AudioMixer.cpp

@@ -62,7 +62,11 @@ void attachNewBufferToNode(Node *newNode) {
 }
 
 AudioMixer::AudioMixer(const QByteArray& packet) :
-    ThreadedAssignment(packet)
+    ThreadedAssignment(packet),
+    _minSourceLoudnessInFrame(1.0f),
+    _maxSourceLoudnessInFrame(0.0f),
+    _loudnessCutoffRatio(0.0f),
+    _minRequiredLoudness(0.0f)
 {
     
 }
@@ -350,14 +354,64 @@ void AudioMixer::run() {
     
     char* clientMixBuffer = new char[NETWORK_BUFFER_LENGTH_BYTES_STEREO
                                      + numBytesForPacketHeaderGivenPacketType(PacketTypeMixedAudio)];
+    
+    int usecToSleep = 0;
+    bool isFirstRun = true;
 
     while (!_isFinished) {
 
+        _minSourceLoudnessInFrame = 1.0f;
+        _maxSourceLoudnessInFrame = 0.0f;
+        
         foreach (const SharedNodePointer& node, nodeList->getNodeHash()) {
             if (node->getLinkedData()) {
-                ((AudioMixerClientData*) node->getLinkedData())->checkBuffersBeforeFrameSend(JITTER_BUFFER_SAMPLES);
+                ((AudioMixerClientData*) node->getLinkedData())->checkBuffersBeforeFrameSend(JITTER_BUFFER_SAMPLES,
+                                                                                             _minSourceLoudnessInFrame,
+                                                                                             _maxSourceLoudnessInFrame);
             }
         }
+        
+        if (!isFirstRun) {
+            const float STRUGGLE_TRIGGER_SLEEP_PERCENTAGE_THRESHOLD = 0.10;
+            const float BACK_OFF_TRIGGER_SLEEP_PERCENTAGE_THRESHOLD = 0.30;
+            const float CUTOFF_EPSILON = 0.0001;
+            
+            float percentageSleep = (usecToSleep / (float) BUFFER_SEND_INTERVAL_USECS);
+            
+            float lastCutoffRatio = _loudnessCutoffRatio;
+            bool hasRatioChanged = false;
+            
+            if (percentageSleep <= STRUGGLE_TRIGGER_SLEEP_PERCENTAGE_THRESHOLD || usecToSleep < 0) {
+                // we're struggling - change our min required loudness to reduce some load
+                _loudnessCutoffRatio += (1 - _loudnessCutoffRatio) / 2;
+                
+                qDebug() << "Mixer is struggling, sleeping" << percentageSleep * 100 << "% of frame time. Old cutoff was"
+                    << lastCutoffRatio << "and is now" << _loudnessCutoffRatio;
+                hasRatioChanged = true;
+            } else if (percentageSleep >= BACK_OFF_TRIGGER_SLEEP_PERCENTAGE_THRESHOLD && _loudnessCutoffRatio != 0) {
+                // we've recovered and can back off the required loudness
+                _loudnessCutoffRatio -= _loudnessCutoffRatio / 2;
+                
+                if (_loudnessCutoffRatio < CUTOFF_EPSILON) {
+                    _loudnessCutoffRatio = 0;
+                }
+                
+                qDebug() << "Mixer is recovering, sleeping" << percentageSleep * 100 << "% of frame time. Old cutoff was"
+                    << lastCutoffRatio << "and is now" << _loudnessCutoffRatio;
+                hasRatioChanged = true;
+            }
+            
+            if (hasRatioChanged) {
+                // set out min required loudness from the new ratio
+                _minRequiredLoudness = _loudnessCutoffRatio * (_maxSourceLoudnessInFrame - _minSourceLoudnessInFrame);
+                qDebug() << "Minimum loudness required to be mixed is now" << _minRequiredLoudness;
+            }
+            
+            
+        
+        } else {
+            isFirstRun = false;
+        }
 
         foreach (const SharedNodePointer& node, nodeList->getNodeHash()) {
             if (node->getType() == NodeType::Agent && node->getActiveSocket() && node->getLinkedData()
@@ -384,7 +438,7 @@ void AudioMixer::run() {
             break;
         }
 
-        int usecToSleep = usecTimestamp(&startTime) + (++nextFrame * BUFFER_SEND_INTERVAL_USECS) - usecTimestampNow();
+        usecToSleep = usecTimestamp(&startTime) + (++nextFrame * BUFFER_SEND_INTERVAL_USECS) - usecTimestampNow();
 
         if (usecToSleep > 0) {
             usleep(usecToSleep);

assignment-client/src/audio/AudioMixer.h

@@ -37,7 +37,13 @@ private:
     void prepareMixForListeningNode(Node* node);
     
     // client samples capacity is larger than what will be sent to optimize mixing
-    int16_t _clientSamples[NETWORK_BUFFER_LENGTH_SAMPLES_STEREO + SAMPLE_PHASE_DELAY_AT_90];
+    // we are MMX adding 4 samples at a time so we need client samples to have an extra 4
+    int16_t _clientSamples[NETWORK_BUFFER_LENGTH_SAMPLES_STEREO + (SAMPLE_PHASE_DELAY_AT_90 * 2)];
+    
+    float _minSourceLoudnessInFrame;
+    float _maxSourceLoudnessInFrame;
+    float _loudnessCutoffRatio;
+    float _minRequiredLoudness;
 };
 
 #endif /* defined(__hifi__AudioMixer__) */

assignment-client/src/audio/AudioMixerClientData.cpp

@@ -6,6 +6,8 @@
 //  Copyright (c) 2013 HighFidelity, Inc. All rights reserved.
 //
 
+#include <QDebug>
+
 #include <PacketHeaders.h>
 #include <UUID.h>
 
@@ -82,7 +84,9 @@ int AudioMixerClientData::parseData(const QByteArray& packet) {
     return 0;
 }
 
-void AudioMixerClientData::checkBuffersBeforeFrameSend(int jitterBufferLengthSamples) {
+void AudioMixerClientData::checkBuffersBeforeFrameSend(int jitterBufferLengthSamples,
+                                                       float& currentMinLoudness,
+                                                       float& currentMaxLoudness) {
     for (unsigned int i = 0; i < _ringBuffers.size(); i++) {
         if (_ringBuffers[i]->shouldBeAddedToMix(jitterBufferLengthSamples)) {
             // this is a ring buffer that is ready to go
@@ -92,6 +96,14 @@ void AudioMixerClientData::checkBuffersBeforeFrameSend(int jitterBufferLengthSam
             // calculate the average loudness for the next NETWORK_BUFFER_LENGTH_SAMPLES_PER_CHANNEL
             // that would be mixed in
             _nextOutputLoudness = _ringBuffers[i]->averageLoudnessForBoundarySamples(NETWORK_BUFFER_LENGTH_SAMPLES_PER_CHANNEL);
+            
+            if (_nextOutputLoudness != 0 && _nextOutputLoudness < currentMinLoudness) {
+                currentMinLoudness = _nextOutputLoudness;
+            }
+            
+            if (_nextOutputLoudness > currentMaxLoudness) {
+                currentMaxLoudness = _nextOutputLoudness;
+            }
         }
     }
 }

assignment-client/src/audio/AudioMixerClientData.h

@@ -27,7 +27,7 @@ public:
     float getNextOutputLoudness() const { return _nextOutputLoudness; }
     
     int parseData(const QByteArray& packet);
-    void checkBuffersBeforeFrameSend(int jitterBufferLengthSamples);
+    void checkBuffersBeforeFrameSend(int jitterBufferLengthSamples, float& currentMinLoudness, float& currentMaxLoudness);
     void pushBuffersAfterFrameSend();
 private:
     std::vector<PositionalAudioRingBuffer*> _ringBuffers;

libraries/audio/src/PositionalAudioRingBuffer.cpp

@@ -50,6 +50,7 @@ int PositionalAudioRingBuffer::parseData(const QByteArray& packet) {
         int16_t numSilentSamples;
         
         memcpy(&numSilentSamples, packet.data() + readBytes, sizeof(int16_t));
+        
         readBytes += sizeof(int16_t);
         
         addSilentFrame(numSilentSamples);