Merge pull request #2510 from birarda/agent-loudness

add a method to Agent for last read frame loudness

assignment-client/src/Agent.cpp

@@ -26,7 +26,8 @@
 Agent::Agent(const QByteArray& packet) :
     ThreadedAssignment(packet),
     _voxelEditSender(),
-    _particleEditSender()
+    _particleEditSender(),
+    _receivedAudioBuffer(NETWORK_BUFFER_LENGTH_SAMPLES_STEREO)
 {
     // be the parent of the script engine so it gets moved when we do
     _scriptEngine.setParent(this);
@@ -113,6 +114,16 @@ void Agent::readPendingDatagrams() {
                     _voxelViewer.processDatagram(mutablePacket, sourceNode);
                 }
 
+            } else if (datagramPacketType == PacketTypeMixedAudio) {
+                // parse the data and grab the average loudness
+                _receivedAudioBuffer.parseData(receivedPacket);
+                
+                // pretend like we have read the samples from this buffer so it does not fill
+                static int16_t garbageAudioBuffer[NETWORK_BUFFER_LENGTH_SAMPLES_STEREO];
+                _receivedAudioBuffer.readSamples(garbageAudioBuffer, NETWORK_BUFFER_LENGTH_SAMPLES_STEREO);
+                
+                // let this continue through to the NodeList so it updates last heard timestamp
+                // for the sending audio mixer
             } else {
                 NodeList::getInstance()->processNodeData(senderSockAddr, receivedPacket);
             }

assignment-client/src/Agent.h

@@ -15,6 +15,7 @@
 #include <QtCore/QObject>
 #include <QtCore/QUrl>
 
+#include <MixedAudioRingBuffer.h>
 #include <ParticleEditPacketSender.h>
 #include <ParticleTree.h>
 #include <ParticleTreeHeadlessViewer.h>
@@ -30,6 +31,7 @@ class Agent : public ThreadedAssignment {
     Q_PROPERTY(bool isAvatar READ isAvatar WRITE setIsAvatar)
     Q_PROPERTY(bool isPlayingAvatarSound READ isPlayingAvatarSound)
     Q_PROPERTY(bool isListeningToAudioStream READ isListeningToAudioStream WRITE setIsListeningToAudioStream)
+    Q_PROPERTY(float lastReceivedAudioLoudness READ getLastReceivedAudioLoudness)
 public:
     Agent(const QByteArray& packet);
     
@@ -41,6 +43,8 @@ public:
     bool isListeningToAudioStream() const { return _scriptEngine.isListeningToAudioStream(); }
     void setIsListeningToAudioStream(bool isListeningToAudioStream)
         { _scriptEngine.setIsListeningToAudioStream(isListeningToAudioStream); }
+    
+    float getLastReceivedAudioLoudness() const { return _receivedAudioBuffer.getLastReadFrameAverageLoudness(); }
 
     virtual void aboutToFinish();
     
@@ -56,6 +60,8 @@ private:
 
     ParticleTreeHeadlessViewer _particleViewer;
     VoxelTreeHeadlessViewer _voxelViewer;
+    
+    MixedAudioRingBuffer _receivedAudioBuffer;
 };
 
 #endif /* defined(__hifi__Agent__) */

assignment-client/src/audio/AudioMixer.cpp

@@ -93,7 +93,7 @@ void AudioMixer::addBufferToMixForListeningNodeWithBuffer(PositionalAudioRingBuf
             distanceBetween = EPSILON;
         }
         
-        if (bufferToAdd->getAverageLoudness() / distanceBetween <= _minAudibilityThreshold) {
+        if (bufferToAdd->getNextOutputTrailingLoudness() / distanceBetween <= _minAudibilityThreshold) {
             // according to mixer performance we have decided this does not get to be mixed in
             // bail out
             return;
@@ -324,7 +324,7 @@ void AudioMixer::prepareMixForListeningNode(Node* node) {
                 if ((*otherNode != *node
                      || otherNodeBuffer->shouldLoopbackForNode())
                     && otherNodeBuffer->willBeAddedToMix()
-                    && otherNodeBuffer->getAverageLoudness() > 0) {
+                    && otherNodeBuffer->getNextOutputTrailingLoudness() > 0) {
                     addBufferToMixForListeningNodeWithBuffer(otherNodeBuffer, nodeRingBuffer);
                 }
             }

assignment-client/src/audio/AudioMixerClientData.cpp

@@ -92,7 +92,7 @@ void AudioMixerClientData::checkBuffersBeforeFrameSend(int jitterBufferLengthSam
             
             // calculate the average loudness for the next NETWORK_BUFFER_LENGTH_SAMPLES_PER_CHANNEL
             // that would be mixed in
-            _ringBuffers[i]->updateAverageLoudnessForBoundarySamples(NETWORK_BUFFER_LENGTH_SAMPLES_PER_CHANNEL);
+            _ringBuffers[i]->updateNextOutputTrailingLoudness();
         }
     }
 }

libraries/audio/src/AudioRingBuffer.cpp

@@ -18,9 +18,9 @@
 AudioRingBuffer::AudioRingBuffer(int numFrameSamples) :
     NodeData(),
     _sampleCapacity(numFrameSamples * RING_BUFFER_LENGTH_FRAMES),
+    _numFrameSamples(numFrameSamples),
     _isStarved(true),
-    _hasStarted(false),
-    _averageLoudness(0)
+    _hasStarted(false)
 {
     if (numFrameSamples) {
         _buffer = new int16_t[_sampleCapacity];
@@ -56,33 +56,6 @@ int AudioRingBuffer::parseData(const QByteArray& packet) {
     return writeData(packet.data() + numBytesPacketHeader, packet.size() - numBytesPacketHeader);
 }
 
-void AudioRingBuffer::updateAverageLoudnessForBoundarySamples(int numSamples) {
-    // ForBoundarySamples means that we expect the number of samples not to roll of the end of the ring buffer
-    float nextLoudness = 0;
-    
-    for (int i = 0; i < numSamples; ++i) {
-        nextLoudness += fabsf(_nextOutput[i]);
-    }
-    
-    nextLoudness /= numSamples;
-    nextLoudness /= MAX_SAMPLE_VALUE;
-    
-    const int TRAILING_AVERAGE_FRAMES = 100;
-    const float CURRENT_FRAME_RATIO = 1.0f / TRAILING_AVERAGE_FRAMES;
-    const float PREVIOUS_FRAMES_RATIO = 1.0f - CURRENT_FRAME_RATIO;
-    const float LOUDNESS_EPSILON = 0.01f;
-    
-    if (nextLoudness >= _averageLoudness) {
-        _averageLoudness = nextLoudness;
-    } else {
-        _averageLoudness = (_averageLoudness * PREVIOUS_FRAMES_RATIO) + (CURRENT_FRAME_RATIO * nextLoudness);
-        
-        if (_averageLoudness < LOUDNESS_EPSILON) {
-            _averageLoudness = 0;
-        }
-    }
-}
-
 qint64 AudioRingBuffer::readSamples(int16_t* destination, qint64 maxSamples) {
     return readData((char*) destination, maxSamples * sizeof(int16_t));
 }
@@ -112,7 +85,7 @@ qint64 AudioRingBuffer::readData(char *data, qint64 maxSize) {
     return numReadSamples * sizeof(int16_t);
 }
 
-qint64 AudioRingBuffer::writeSamples(const int16_t* source, qint64 maxSamples) {
+qint64 AudioRingBuffer::writeSamples(const int16_t* source, qint64 maxSamples) {    
     return writeData((const char*) source, maxSamples * sizeof(int16_t));
 }
 

libraries/audio/src/AudioRingBuffer.h

@@ -49,9 +49,6 @@ public:
     // assume callers using this will never wrap around the end
     const int16_t* getNextOutput() { return _nextOutput; }
     const int16_t* getBuffer() { return _buffer; }
-    
-    void updateAverageLoudnessForBoundarySamples(int numSamples);
-    float getAverageLoudness() const { return _averageLoudness; }
 
     qint64 readSamples(int16_t* destination, qint64 maxSamples);
     qint64 writeSamples(const int16_t* source, qint64 maxSamples);
@@ -81,13 +78,12 @@ protected:
     int16_t* shiftedPositionAccomodatingWrap(int16_t* position, int numSamplesShift) const;
     
     int _sampleCapacity;
+    int _numFrameSamples;
     int16_t* _nextOutput;
     int16_t* _endOfLastWrite;
     int16_t* _buffer;
     bool _isStarved;
     bool _hasStarted;
-    
-    float _averageLoudness;
 };
 
 #endif /* defined(__interface__AudioRingBuffer__) */

libraries/audio/src/MixedAudioRingBuffer.cpp

@@ -0,0 +1,49 @@
+//
+//  MixedAudioRingBuffer.cpp
+//  hifi
+//
+//  Created by Stephen Birarda on 2014-03-26.
+//  Copyright (c) 2014 HighFidelity, Inc. All rights reserved.
+//
+
+#include "MixedAudioRingBuffer.h"
+
+MixedAudioRingBuffer::MixedAudioRingBuffer(int numFrameSamples) :
+    AudioRingBuffer(numFrameSamples),
+    _lastReadFrameAverageLoudness(0.0f)
+{
+    
+}
+
+qint64 MixedAudioRingBuffer::readSamples(int16_t* destination, qint64 maxSamples) {
+    // calculate the average loudness for the frame about to go out
+    
+    // read from _nextOutput either _numFrameSamples or to the end of the buffer
+    int samplesFromNextOutput = _buffer + _sampleCapacity - _nextOutput;
+    if (samplesFromNextOutput > _numFrameSamples) {
+        samplesFromNextOutput = _numFrameSamples;
+    }
+    
+    float averageLoudness = 0.0f;
+    
+    for (int s = 0; s < samplesFromNextOutput; s++) {
+        averageLoudness += fabsf(_nextOutput[s]);
+    }
+    
+    // read samples from the beginning of the buffer, if any
+    int samplesFromBeginning = _numFrameSamples - samplesFromNextOutput;
+    
+    if (samplesFromBeginning > 0) {
+        for (int b = 0; b < samplesFromBeginning; b++) {
+            averageLoudness += fabsf(_buffer[b]);
+        }
+    }
+    
+    // divide by the number of samples and the MAX_SAMPLE_VALUE to get a float from 0 - 1
+    averageLoudness /= (float) _numFrameSamples;
+    averageLoudness /= (float) MAX_SAMPLE_VALUE;
+    
+    _lastReadFrameAverageLoudness = averageLoudness;
+    
+    return AudioRingBuffer::readSamples(destination, maxSamples);
+}

libraries/audio/src/MixedAudioRingBuffer.h

@@ -0,0 +1,26 @@
+//
+//  MixedAudioRingBuffer.h
+//  hifi
+//
+//  Created by Stephen Birarda on 2014-03-26.
+//  Copyright (c) 2014 HighFidelity, Inc. All rights reserved.
+//
+
+#ifndef __hifi__MixedAudioRingBuffer__
+#define __hifi__MixedAudioRingBuffer__
+
+#include "AudioRingBuffer.h"
+
+class MixedAudioRingBuffer : public AudioRingBuffer {
+    Q_OBJECT
+public:
+    MixedAudioRingBuffer(int numFrameSamples);
+    
+    float getLastReadFrameAverageLoudness() const { return _lastReadFrameAverageLoudness; }
+    
+    qint64 readSamples(int16_t* destination, qint64 maxSamples);    
+private:
+     float _lastReadFrameAverageLoudness;
+};
+
+#endif /* defined(__hifi__MixedAudioRingBuffer__) */

libraries/audio/src/PositionalAudioRingBuffer.cpp

@@ -72,6 +72,33 @@ int PositionalAudioRingBuffer::parsePositionalData(const QByteArray& positionalB
     return packetStream.device()->pos();
 }
 
+void PositionalAudioRingBuffer::updateNextOutputTrailingLoudness() {
+    // ForBoundarySamples means that we expect the number of samples not to roll of the end of the ring buffer
+    float nextLoudness = 0;
+    
+    for (int i = 0; i < _numFrameSamples; ++i) {
+        nextLoudness += fabsf(_nextOutput[i]);
+    }
+    
+    nextLoudness /= _numFrameSamples;
+    nextLoudness /= MAX_SAMPLE_VALUE;
+    
+    const int TRAILING_AVERAGE_FRAMES = 100;
+    const float CURRENT_FRAME_RATIO = 1.0f / TRAILING_AVERAGE_FRAMES;
+    const float PREVIOUS_FRAMES_RATIO = 1.0f - CURRENT_FRAME_RATIO;
+    const float LOUDNESS_EPSILON = 0.01f;
+    
+    if (nextLoudness >= _nextOutputTrailingLoudness) {
+        _nextOutputTrailingLoudness = nextLoudness;
+    } else {
+        _nextOutputTrailingLoudness = (_nextOutputTrailingLoudness * PREVIOUS_FRAMES_RATIO) + (CURRENT_FRAME_RATIO * nextLoudness);
+        
+        if (_nextOutputTrailingLoudness < LOUDNESS_EPSILON) {
+            _nextOutputTrailingLoudness = 0;
+        }
+    }
+}
+
 bool PositionalAudioRingBuffer::shouldBeAddedToMix(int numJitterBufferSamples) {
     if (!isNotStarvedOrHasMinimumSamples(NETWORK_BUFFER_LENGTH_SAMPLES_PER_CHANNEL + numJitterBufferSamples)) {
         if (_shouldOutputStarveDebug) {

libraries/audio/src/PositionalAudioRingBuffer.h

@@ -28,6 +28,9 @@ public:
     int parsePositionalData(const QByteArray& positionalByteArray);
     int parseListenModeData(const QByteArray& listenModeByteArray);
     
+    void updateNextOutputTrailingLoudness();
+    float getNextOutputTrailingLoudness() const { return _nextOutputTrailingLoudness; }
+    
     bool shouldBeAddedToMix(int numJitterBufferSamples);
     
     bool willBeAddedToMix() const { return _willBeAddedToMix; }
@@ -50,6 +53,8 @@ protected:
     bool _willBeAddedToMix;
     bool _shouldLoopbackForNode;
     bool _shouldOutputStarveDebug;
+    
+    float _nextOutputTrailingLoudness;
 };
 
 #endif /* defined(__hifi__PositionalAudioRingBuffer__) */