From e2b230445fc0663c6c4b2d285c9d6eaaab351fa8 Mon Sep 17 00:00:00 2001 From: Stephen Birarda Date: Wed, 26 Mar 2014 10:28:59 -0700 Subject: [PATCH] add a method to Agent for last read frame loudness --- assignment-client/src/Agent.cpp | 13 ++++- assignment-client/src/Agent.h | 6 +++ assignment-client/src/audio/AudioMixer.cpp | 4 +- .../src/audio/AudioMixerClientData.cpp | 2 +- libraries/audio/src/AudioRingBuffer.cpp | 33 ++----------- libraries/audio/src/AudioRingBuffer.h | 6 +-- libraries/audio/src/MixedAudioRingBuffer.cpp | 49 +++++++++++++++++++ libraries/audio/src/MixedAudioRingBuffer.h | 26 ++++++++++ .../audio/src/PositionalAudioRingBuffer.cpp | 27 ++++++++++ .../audio/src/PositionalAudioRingBuffer.h | 5 ++ 10 files changed, 132 insertions(+), 39 deletions(-) create mode 100644 libraries/audio/src/MixedAudioRingBuffer.cpp create mode 100644 libraries/audio/src/MixedAudioRingBuffer.h diff --git a/assignment-client/src/Agent.cpp b/assignment-client/src/Agent.cpp index b23f9d210a..18051d816c 100644 --- a/assignment-client/src/Agent.cpp +++ b/assignment-client/src/Agent.cpp @@ -26,7 +26,8 @@ Agent::Agent(const QByteArray& packet) : ThreadedAssignment(packet), _voxelEditSender(), - _particleEditSender() + _particleEditSender(), + _receivedAudioBuffer(NETWORK_BUFFER_LENGTH_SAMPLES_STEREO) { // be the parent of the script engine so it gets moved when we do _scriptEngine.setParent(this); @@ -113,6 +114,16 @@ void Agent::readPendingDatagrams() { _voxelViewer.processDatagram(mutablePacket, sourceNode); } + } else if (datagramPacketType == PacketTypeMixedAudio) { + // parse the data and grab the average loudness + _receivedAudioBuffer.parseData(receivedPacket); + + // pretend like we have read the samples from this buffer so it does not fill + static int16_t garbageAudioBuffer[NETWORK_BUFFER_LENGTH_SAMPLES_STEREO]; + _receivedAudioBuffer.readSamples(garbageAudioBuffer, NETWORK_BUFFER_LENGTH_SAMPLES_STEREO); + + // let this continue through to the NodeList so it updates last heard timestamp + // for the sending audio mixer } else { NodeList::getInstance()->processNodeData(senderSockAddr, receivedPacket); } diff --git a/assignment-client/src/Agent.h b/assignment-client/src/Agent.h index 0a61bd73f7..9e5f31213c 100644 --- a/assignment-client/src/Agent.h +++ b/assignment-client/src/Agent.h @@ -15,6 +15,7 @@ #include #include +#include #include #include #include @@ -30,6 +31,7 @@ class Agent : public ThreadedAssignment { Q_PROPERTY(bool isAvatar READ isAvatar WRITE setIsAvatar) Q_PROPERTY(bool isPlayingAvatarSound READ isPlayingAvatarSound) Q_PROPERTY(bool isListeningToAudioStream READ isListeningToAudioStream WRITE setIsListeningToAudioStream) + Q_PROPERTY(float lastReceivedAudioLoudness READ getLastReceivedAudioLoudness) public: Agent(const QByteArray& packet); @@ -41,6 +43,8 @@ public: bool isListeningToAudioStream() const { return _scriptEngine.isListeningToAudioStream(); } void setIsListeningToAudioStream(bool isListeningToAudioStream) { _scriptEngine.setIsListeningToAudioStream(isListeningToAudioStream); } + + float getLastReceivedAudioLoudness() const { return _receivedAudioBuffer.getLastReadFrameAverageLoudness(); } virtual void aboutToFinish(); @@ -56,6 +60,8 @@ private: ParticleTreeHeadlessViewer _particleViewer; VoxelTreeHeadlessViewer _voxelViewer; + + MixedAudioRingBuffer _receivedAudioBuffer; }; #endif /* defined(__hifi__Agent__) */ diff --git a/assignment-client/src/audio/AudioMixer.cpp b/assignment-client/src/audio/AudioMixer.cpp index 82674e5141..f183abade9 100644 --- a/assignment-client/src/audio/AudioMixer.cpp +++ b/assignment-client/src/audio/AudioMixer.cpp @@ -93,7 +93,7 @@ void AudioMixer::addBufferToMixForListeningNodeWithBuffer(PositionalAudioRingBuf distanceBetween = EPSILON; } - if (bufferToAdd->getAverageLoudness() / distanceBetween <= _minAudibilityThreshold) { + if (bufferToAdd->getNextOutputTrailingLoudness() / distanceBetween <= _minAudibilityThreshold) { // according to mixer performance we have decided this does not get to be mixed in // bail out return; @@ -324,7 +324,7 @@ void AudioMixer::prepareMixForListeningNode(Node* node) { if ((*otherNode != *node || otherNodeBuffer->shouldLoopbackForNode()) && otherNodeBuffer->willBeAddedToMix() - && otherNodeBuffer->getAverageLoudness() > 0) { + && otherNodeBuffer->getNextOutputTrailingLoudness() > 0) { addBufferToMixForListeningNodeWithBuffer(otherNodeBuffer, nodeRingBuffer); } } diff --git a/assignment-client/src/audio/AudioMixerClientData.cpp b/assignment-client/src/audio/AudioMixerClientData.cpp index 381c80cb09..4a9f1f84de 100644 --- a/assignment-client/src/audio/AudioMixerClientData.cpp +++ b/assignment-client/src/audio/AudioMixerClientData.cpp @@ -92,7 +92,7 @@ void AudioMixerClientData::checkBuffersBeforeFrameSend(int jitterBufferLengthSam // calculate the average loudness for the next NETWORK_BUFFER_LENGTH_SAMPLES_PER_CHANNEL // that would be mixed in - _ringBuffers[i]->updateAverageLoudnessForBoundarySamples(NETWORK_BUFFER_LENGTH_SAMPLES_PER_CHANNEL); + _ringBuffers[i]->updateNextOutputTrailingLoudness(); } } } diff --git a/libraries/audio/src/AudioRingBuffer.cpp b/libraries/audio/src/AudioRingBuffer.cpp index 4732cc77d1..376b60ffa1 100644 --- a/libraries/audio/src/AudioRingBuffer.cpp +++ b/libraries/audio/src/AudioRingBuffer.cpp @@ -18,9 +18,9 @@ AudioRingBuffer::AudioRingBuffer(int numFrameSamples) : NodeData(), _sampleCapacity(numFrameSamples * RING_BUFFER_LENGTH_FRAMES), + _numFrameSamples(numFrameSamples), _isStarved(true), - _hasStarted(false), - _averageLoudness(0) + _hasStarted(false) { if (numFrameSamples) { _buffer = new int16_t[_sampleCapacity]; @@ -56,33 +56,6 @@ int AudioRingBuffer::parseData(const QByteArray& packet) { return writeData(packet.data() + numBytesPacketHeader, packet.size() - numBytesPacketHeader); } -void AudioRingBuffer::updateAverageLoudnessForBoundarySamples(int numSamples) { - // ForBoundarySamples means that we expect the number of samples not to roll of the end of the ring buffer - float nextLoudness = 0; - - for (int i = 0; i < numSamples; ++i) { - nextLoudness += fabsf(_nextOutput[i]); - } - - nextLoudness /= numSamples; - nextLoudness /= MAX_SAMPLE_VALUE; - - const int TRAILING_AVERAGE_FRAMES = 100; - const float CURRENT_FRAME_RATIO = 1.0f / TRAILING_AVERAGE_FRAMES; - const float PREVIOUS_FRAMES_RATIO = 1.0f - CURRENT_FRAME_RATIO; - const float LOUDNESS_EPSILON = 0.01f; - - if (nextLoudness >= _averageLoudness) { - _averageLoudness = nextLoudness; - } else { - _averageLoudness = (_averageLoudness * PREVIOUS_FRAMES_RATIO) + (CURRENT_FRAME_RATIO * nextLoudness); - - if (_averageLoudness < LOUDNESS_EPSILON) { - _averageLoudness = 0; - } - } -} - qint64 AudioRingBuffer::readSamples(int16_t* destination, qint64 maxSamples) { return readData((char*) destination, maxSamples * sizeof(int16_t)); } @@ -112,7 +85,7 @@ qint64 AudioRingBuffer::readData(char *data, qint64 maxSize) { return numReadSamples * sizeof(int16_t); } -qint64 AudioRingBuffer::writeSamples(const int16_t* source, qint64 maxSamples) { +qint64 AudioRingBuffer::writeSamples(const int16_t* source, qint64 maxSamples) { return writeData((const char*) source, maxSamples * sizeof(int16_t)); } diff --git a/libraries/audio/src/AudioRingBuffer.h b/libraries/audio/src/AudioRingBuffer.h index efc85ac94b..e55eeda40e 100644 --- a/libraries/audio/src/AudioRingBuffer.h +++ b/libraries/audio/src/AudioRingBuffer.h @@ -49,9 +49,6 @@ public: // assume callers using this will never wrap around the end const int16_t* getNextOutput() { return _nextOutput; } const int16_t* getBuffer() { return _buffer; } - - void updateAverageLoudnessForBoundarySamples(int numSamples); - float getAverageLoudness() const { return _averageLoudness; } qint64 readSamples(int16_t* destination, qint64 maxSamples); qint64 writeSamples(const int16_t* source, qint64 maxSamples); @@ -81,13 +78,12 @@ protected: int16_t* shiftedPositionAccomodatingWrap(int16_t* position, int numSamplesShift) const; int _sampleCapacity; + int _numFrameSamples; int16_t* _nextOutput; int16_t* _endOfLastWrite; int16_t* _buffer; bool _isStarved; bool _hasStarted; - - float _averageLoudness; }; #endif /* defined(__interface__AudioRingBuffer__) */ diff --git a/libraries/audio/src/MixedAudioRingBuffer.cpp b/libraries/audio/src/MixedAudioRingBuffer.cpp new file mode 100644 index 0000000000..0ffab23519 --- /dev/null +++ b/libraries/audio/src/MixedAudioRingBuffer.cpp @@ -0,0 +1,49 @@ +// +// MixedAudioRingBuffer.cpp +// hifi +// +// Created by Stephen Birarda on 2014-03-26. +// Copyright (c) 2014 HighFidelity, Inc. All rights reserved. +// + +#include "MixedAudioRingBuffer.h" + +MixedAudioRingBuffer::MixedAudioRingBuffer(int numFrameSamples) : + AudioRingBuffer(numFrameSamples), + _lastReadFrameAverageLoudness(0.0f) +{ + +} + +qint64 MixedAudioRingBuffer::readSamples(int16_t* destination, qint64 maxSamples) { + // calculate the average loudness for the frame about to go out + + // read from _nextOutput either _numFrameSamples or to the end of the buffer + int samplesFromNextOutput = _buffer + _sampleCapacity - _nextOutput; + if (samplesFromNextOutput > _numFrameSamples) { + samplesFromNextOutput = _numFrameSamples; + } + + float averageLoudness = 0.0f; + + for (int s = 0; s < samplesFromNextOutput; s++) { + averageLoudness += fabsf(_nextOutput[s]); + } + + // read samples from the beginning of the buffer, if any + int samplesFromBeginning = _numFrameSamples - samplesFromNextOutput; + + if (samplesFromBeginning > 0) { + for (int b = 0; b < samplesFromBeginning; b++) { + averageLoudness += fabsf(_buffer[b]); + } + } + + // divide by the number of samples and the MAX_SAMPLE_VALUE to get a float from 0 - 1 + averageLoudness /= (float) _numFrameSamples; + averageLoudness /= (float) MAX_SAMPLE_VALUE; + + _lastReadFrameAverageLoudness = averageLoudness; + + return AudioRingBuffer::readSamples(destination, maxSamples); +} \ No newline at end of file diff --git a/libraries/audio/src/MixedAudioRingBuffer.h b/libraries/audio/src/MixedAudioRingBuffer.h new file mode 100644 index 0000000000..c116361689 --- /dev/null +++ b/libraries/audio/src/MixedAudioRingBuffer.h @@ -0,0 +1,26 @@ +// +// MixedAudioRingBuffer.h +// hifi +// +// Created by Stephen Birarda on 2014-03-26. +// Copyright (c) 2014 HighFidelity, Inc. All rights reserved. +// + +#ifndef __hifi__MixedAudioRingBuffer__ +#define __hifi__MixedAudioRingBuffer__ + +#include "AudioRingBuffer.h" + +class MixedAudioRingBuffer : public AudioRingBuffer { + Q_OBJECT +public: + MixedAudioRingBuffer(int numFrameSamples); + + float getLastReadFrameAverageLoudness() const { return _lastReadFrameAverageLoudness; } + + qint64 readSamples(int16_t* destination, qint64 maxSamples); +private: + float _lastReadFrameAverageLoudness; +}; + +#endif /* defined(__hifi__MixedAudioRingBuffer__) */ diff --git a/libraries/audio/src/PositionalAudioRingBuffer.cpp b/libraries/audio/src/PositionalAudioRingBuffer.cpp index d1729ddfef..7bdedfc793 100644 --- a/libraries/audio/src/PositionalAudioRingBuffer.cpp +++ b/libraries/audio/src/PositionalAudioRingBuffer.cpp @@ -72,6 +72,33 @@ int PositionalAudioRingBuffer::parsePositionalData(const QByteArray& positionalB return packetStream.device()->pos(); } +void PositionalAudioRingBuffer::updateNextOutputTrailingLoudness() { + // ForBoundarySamples means that we expect the number of samples not to roll of the end of the ring buffer + float nextLoudness = 0; + + for (int i = 0; i < _numFrameSamples; ++i) { + nextLoudness += fabsf(_nextOutput[i]); + } + + nextLoudness /= _numFrameSamples; + nextLoudness /= MAX_SAMPLE_VALUE; + + const int TRAILING_AVERAGE_FRAMES = 100; + const float CURRENT_FRAME_RATIO = 1.0f / TRAILING_AVERAGE_FRAMES; + const float PREVIOUS_FRAMES_RATIO = 1.0f - CURRENT_FRAME_RATIO; + const float LOUDNESS_EPSILON = 0.01f; + + if (nextLoudness >= _nextOutputTrailingLoudness) { + _nextOutputTrailingLoudness = nextLoudness; + } else { + _nextOutputTrailingLoudness = (_nextOutputTrailingLoudness * PREVIOUS_FRAMES_RATIO) + (CURRENT_FRAME_RATIO * nextLoudness); + + if (_nextOutputTrailingLoudness < LOUDNESS_EPSILON) { + _nextOutputTrailingLoudness = 0; + } + } +} + bool PositionalAudioRingBuffer::shouldBeAddedToMix(int numJitterBufferSamples) { if (!isNotStarvedOrHasMinimumSamples(NETWORK_BUFFER_LENGTH_SAMPLES_PER_CHANNEL + numJitterBufferSamples)) { if (_shouldOutputStarveDebug) { diff --git a/libraries/audio/src/PositionalAudioRingBuffer.h b/libraries/audio/src/PositionalAudioRingBuffer.h index e3fffa1689..a82df0b857 100644 --- a/libraries/audio/src/PositionalAudioRingBuffer.h +++ b/libraries/audio/src/PositionalAudioRingBuffer.h @@ -28,6 +28,9 @@ public: int parsePositionalData(const QByteArray& positionalByteArray); int parseListenModeData(const QByteArray& listenModeByteArray); + void updateNextOutputTrailingLoudness(); + float getNextOutputTrailingLoudness() const { return _nextOutputTrailingLoudness; } + bool shouldBeAddedToMix(int numJitterBufferSamples); bool willBeAddedToMix() const { return _willBeAddedToMix; } @@ -50,6 +53,8 @@ protected: bool _willBeAddedToMix; bool _shouldLoopbackForNode; bool _shouldOutputStarveDebug; + + float _nextOutputTrailingLoudness; }; #endif /* defined(__hifi__PositionalAudioRingBuffer__) */