From bc9deb5db7075e069426f22ef3caeb14cbd6cab7 Mon Sep 17 00:00:00 2001 From: Stephen Birarda Date: Mon, 17 Mar 2014 14:29:53 -0700 Subject: [PATCH 1/8] handle trivial case of not mixing silent audio streams --- assignment-client/src/audio/AudioMixer.cpp | 15 ++++++++------- .../src/audio/AudioMixerClientData.cpp | 11 +++++++++++ .../src/audio/AudioMixerClientData.h | 4 ++++ libraries/audio/src/AudioRingBuffer.cpp | 14 ++++++++++++++ libraries/audio/src/AudioRingBuffer.h | 2 ++ 5 files changed, 39 insertions(+), 7 deletions(-) diff --git a/assignment-client/src/audio/AudioMixer.cpp b/assignment-client/src/audio/AudioMixer.cpp index 1637f88859..be4da8870d 100644 --- a/assignment-client/src/audio/AudioMixer.cpp +++ b/assignment-client/src/audio/AudioMixer.cpp @@ -301,7 +301,8 @@ void AudioMixer::prepareMixForListeningNode(Node* node) { if ((*otherNode != *node || otherNodeBuffer->shouldLoopbackForNode()) - && otherNodeBuffer->willBeAddedToMix()) { + && otherNodeBuffer->willBeAddedToMix() + && otherNodeClientData->getNextOutputLoudness() != 0) { addBufferToMixForListeningNodeWithBuffer(otherNodeBuffer, nodeRingBuffer); } } @@ -355,12 +356,6 @@ void AudioMixer::run() { while (!_isFinished) { - QCoreApplication::processEvents(); - - if (_isFinished) { - break; - } - foreach (const SharedNodePointer& node, nodeList->getNodeHash()) { if (node->getLinkedData()) { ((AudioMixerClientData*) node->getLinkedData())->checkBuffersBeforeFrameSend(JITTER_BUFFER_SAMPLES); @@ -383,6 +378,12 @@ void AudioMixer::run() { ((AudioMixerClientData*) node->getLinkedData())->pushBuffersAfterFrameSend(); } } + + QCoreApplication::processEvents(); + + if (_isFinished) { + break; + } int usecToSleep = usecTimestamp(&startTime) + (++nextFrame * BUFFER_SEND_INTERVAL_USECS) - usecTimestampNow(); diff --git a/assignment-client/src/audio/AudioMixerClientData.cpp b/assignment-client/src/audio/AudioMixerClientData.cpp index a41889e77c..8907796094 100644 --- a/assignment-client/src/audio/AudioMixerClientData.cpp +++ b/assignment-client/src/audio/AudioMixerClientData.cpp @@ -13,6 +13,13 @@ #include "AudioMixerClientData.h" +AudioMixerClientData::AudioMixerClientData() : + _ringBuffers(), + _nextOutputLoudness(0) +{ + +} + AudioMixerClientData::~AudioMixerClientData() { for (unsigned int i = 0; i < _ringBuffers.size(); i++) { // delete this attached PositionalAudioRingBuffer @@ -80,6 +87,10 @@ void AudioMixerClientData::checkBuffersBeforeFrameSend(int jitterBufferLengthSam // this is a ring buffer that is ready to go // set its flag so we know to push its buffer when all is said and done _ringBuffers[i]->setWillBeAddedToMix(true); + + // calculate the average loudness for the next NETWORK_BUFFER_LENGTH_SAMPLES_PER_CHANNEL + // that would be mixed in + _nextOutputLoudness = _ringBuffers[i]->averageLoudnessForBoundarySamples(NETWORK_BUFFER_LENGTH_SAMPLES_PER_CHANNEL); } } } diff --git a/assignment-client/src/audio/AudioMixerClientData.h b/assignment-client/src/audio/AudioMixerClientData.h index 8031dfec3e..bb10098e23 100644 --- a/assignment-client/src/audio/AudioMixerClientData.h +++ b/assignment-client/src/audio/AudioMixerClientData.h @@ -18,16 +18,20 @@ class AudioMixerClientData : public NodeData { public: + AudioMixerClientData(); ~AudioMixerClientData(); const std::vector getRingBuffers() const { return _ringBuffers; } AvatarAudioRingBuffer* getAvatarAudioRingBuffer() const; + float getNextOutputLoudness() const { return _nextOutputLoudness; } + int parseData(const QByteArray& packet); void checkBuffersBeforeFrameSend(int jitterBufferLengthSamples); void pushBuffersAfterFrameSend(); private: std::vector _ringBuffers; + float _nextOutputLoudness; }; #endif /* defined(__hifi__AudioMixerClientData__) */ diff --git a/libraries/audio/src/AudioRingBuffer.cpp b/libraries/audio/src/AudioRingBuffer.cpp index 56f6d038c2..aed601ec39 100644 --- a/libraries/audio/src/AudioRingBuffer.cpp +++ b/libraries/audio/src/AudioRingBuffer.cpp @@ -55,6 +55,20 @@ int AudioRingBuffer::parseData(const QByteArray& packet) { return writeData(packet.data() + numBytesPacketHeader, packet.size() - numBytesPacketHeader); } +float AudioRingBuffer::averageLoudnessForBoundarySamples(int numSamples) { + // ForBoundarySamples means that we expect the number of samples not to roll of the end of the ring buffer + float averageLoudness = 0; + + for (int i = 0; i < numSamples; ++i) { + averageLoudness += _nextOutput[i]; + } + + averageLoudness /= numSamples; + averageLoudness /= MAX_SAMPLE_VALUE; + + return averageLoudness; +} + qint64 AudioRingBuffer::readSamples(int16_t* destination, qint64 maxSamples) { return readData((char*) destination, maxSamples * sizeof(int16_t)); } diff --git a/libraries/audio/src/AudioRingBuffer.h b/libraries/audio/src/AudioRingBuffer.h index 577c01a63c..c4d3f87814 100644 --- a/libraries/audio/src/AudioRingBuffer.h +++ b/libraries/audio/src/AudioRingBuffer.h @@ -49,6 +49,8 @@ public: // assume callers using this will never wrap around the end const int16_t* getNextOutput() { return _nextOutput; } const int16_t* getBuffer() { return _buffer; } + + float averageLoudnessForBoundarySamples(int numSamples); qint64 readSamples(int16_t* destination, qint64 maxSamples); qint64 writeSamples(const int16_t* source, qint64 maxSamples); From 1428d2d1de772a611baa7d182f48be5a0f014fcc Mon Sep 17 00:00:00 2001 From: Stephen Birarda Date: Mon, 17 Mar 2014 14:35:04 -0700 Subject: [PATCH 2/8] take an absolute value for correct loudness --- libraries/audio/src/AudioRingBuffer.cpp | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/libraries/audio/src/AudioRingBuffer.cpp b/libraries/audio/src/AudioRingBuffer.cpp index aed601ec39..d07a334d81 100644 --- a/libraries/audio/src/AudioRingBuffer.cpp +++ b/libraries/audio/src/AudioRingBuffer.cpp @@ -60,7 +60,7 @@ float AudioRingBuffer::averageLoudnessForBoundarySamples(int numSamples) { float averageLoudness = 0; for (int i = 0; i < numSamples; ++i) { - averageLoudness += _nextOutput[i]; + averageLoudness += fabsf(_nextOutput[i]); } averageLoudness /= numSamples; From c7e12824a8e03c6c06ee0fe41a18a6d0c7ebb36a Mon Sep 17 00:00:00 2001 From: Stephen Birarda Date: Mon, 17 Mar 2014 14:35:26 -0700 Subject: [PATCH 3/8] clarify check for audio loudness in AudioMixer --- assignment-client/src/audio/AudioMixer.cpp | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/assignment-client/src/audio/AudioMixer.cpp b/assignment-client/src/audio/AudioMixer.cpp index be4da8870d..aeabd1548b 100644 --- a/assignment-client/src/audio/AudioMixer.cpp +++ b/assignment-client/src/audio/AudioMixer.cpp @@ -302,7 +302,7 @@ void AudioMixer::prepareMixForListeningNode(Node* node) { if ((*otherNode != *node || otherNodeBuffer->shouldLoopbackForNode()) && otherNodeBuffer->willBeAddedToMix() - && otherNodeClientData->getNextOutputLoudness() != 0) { + && otherNodeClientData->getNextOutputLoudness() > 0) { addBufferToMixForListeningNodeWithBuffer(otherNodeBuffer, nodeRingBuffer); } } From 07a71d87963953d8a758ac239082c2662e37c14a Mon Sep 17 00:00:00 2001 From: Stephen Birarda Date: Mon, 17 Mar 2014 17:22:16 -0700 Subject: [PATCH 4/8] use char instead of QByteArray for mix samples in AudioMixer --- assignment-client/src/audio/AudioMixer.cpp | 7 +++---- assignment-client/src/audio/AudioMixer.h | 3 +-- 2 files changed, 4 insertions(+), 6 deletions(-) diff --git a/assignment-client/src/audio/AudioMixer.cpp b/assignment-client/src/audio/AudioMixer.cpp index 1637f88859..34f53a65b4 100644 --- a/assignment-client/src/audio/AudioMixer.cpp +++ b/assignment-client/src/audio/AudioMixer.cpp @@ -62,8 +62,7 @@ void attachNewBufferToNode(Node *newNode) { } AudioMixer::AudioMixer(const QByteArray& packet) : - ThreadedAssignment(packet), - _clientMixBuffer(NETWORK_BUFFER_LENGTH_BYTES_STEREO + numBytesForPacketHeaderGivenPacketType(PacketTypeMixedAudio), 0) + ThreadedAssignment(packet) { connect(NodeList::getInstance(), &NodeList::uuidChanged, this, &AudioMixer::receivedSessionUUID); } @@ -372,8 +371,8 @@ void AudioMixer::run() { && ((AudioMixerClientData*) node->getLinkedData())->getAvatarAudioRingBuffer()) { prepareMixForListeningNode(node.data()); - memcpy(_clientMixBuffer.data() + numBytesPacketHeader, _clientSamples, NETWORK_BUFFER_LENGTH_BYTES_STEREO); - nodeList->writeDatagram(_clientMixBuffer, node); + memcpy(_clientMixBuffer + numBytesPacketHeader, _clientSamples, NETWORK_BUFFER_LENGTH_BYTES_STEREO); + nodeList->writeDatagram(_clientMixBuffer, NETWORK_BUFFER_LENGTH_BYTES_STEREO + numBytesPacketHeader, node); } } diff --git a/assignment-client/src/audio/AudioMixer.h b/assignment-client/src/audio/AudioMixer.h index 8032649c17..0db205d845 100644 --- a/assignment-client/src/audio/AudioMixer.h +++ b/assignment-client/src/audio/AudioMixer.h @@ -38,10 +38,9 @@ private: /// prepares and sends a mix to one Node void prepareMixForListeningNode(Node* node); - QByteArray _clientMixBuffer; - // client samples capacity is larger than what will be sent to optimize mixing int16_t _clientSamples[NETWORK_BUFFER_LENGTH_SAMPLES_STEREO + SAMPLE_PHASE_DELAY_AT_90]; + char _clientMixBuffer[NETWORK_BUFFER_LENGTH_BYTES_STEREO + MAX_PACKET_HEADER_BYTES]; }; #endif /* defined(__hifi__AudioMixer__) */ From 7ba595338ea9460d41c11e56f91fd2e34971cfab Mon Sep 17 00:00:00 2001 From: Stephen Birarda Date: Mon, 17 Mar 2014 17:39:08 -0700 Subject: [PATCH 5/8] guard around update to last heard for a NULL matching node --- libraries/shared/src/NodeList.cpp | 4 +++- 1 file changed, 3 insertions(+), 1 deletion(-) diff --git a/libraries/shared/src/NodeList.cpp b/libraries/shared/src/NodeList.cpp index 0cca9731b0..0354bd8bf9 100644 --- a/libraries/shared/src/NodeList.cpp +++ b/libraries/shared/src/NodeList.cpp @@ -267,7 +267,9 @@ void NodeList::processNodeData(const HifiSockAddr& senderSockAddr, const QByteAr // the node decided not to do anything with this packet // if it comes from a known source we should keep that node alive SharedNodePointer matchingNode = sendingNodeForPacket(packet); - matchingNode->setLastHeardMicrostamp(usecTimestampNow()); + if (matchingNode) { + matchingNode->setLastHeardMicrostamp(usecTimestampNow()); + } break; } From f3910f8c1af675d4b7fc6dc6efdb381b87cb3b9a Mon Sep 17 00:00:00 2001 From: Stephen Birarda Date: Mon, 17 Mar 2014 17:54:27 -0700 Subject: [PATCH 6/8] scope clientMixBuffer only to AudioMixer run --- assignment-client/src/audio/AudioMixer.cpp | 16 +++++++--------- assignment-client/src/audio/AudioMixer.h | 3 --- 2 files changed, 7 insertions(+), 12 deletions(-) diff --git a/assignment-client/src/audio/AudioMixer.cpp b/assignment-client/src/audio/AudioMixer.cpp index 90a901f5c9..c5c112d0f0 100644 --- a/assignment-client/src/audio/AudioMixer.cpp +++ b/assignment-client/src/audio/AudioMixer.cpp @@ -64,7 +64,7 @@ void attachNewBufferToNode(Node *newNode) { AudioMixer::AudioMixer(const QByteArray& packet) : ThreadedAssignment(packet) { - connect(NodeList::getInstance(), &NodeList::uuidChanged, this, &AudioMixer::receivedSessionUUID); + } void AudioMixer::addBufferToMixForListeningNodeWithBuffer(PositionalAudioRingBuffer* bufferToAdd, @@ -332,10 +332,6 @@ void AudioMixer::readPendingDatagrams() { } } -void AudioMixer::receivedSessionUUID(const QUuid& sessionUUID) { - populatePacketHeader(_clientMixBuffer, PacketTypeMixedAudio); -} - void AudioMixer::run() { commonInit(AUDIO_MIXER_LOGGING_TARGET_NAME, NodeType::AudioMixer); @@ -350,8 +346,8 @@ void AudioMixer::run() { timeval startTime; gettimeofday(&startTime, NULL); - - int numBytesPacketHeader = numBytesForPacketHeaderGivenPacketType(PacketTypeMixedAudio); + + char clientMixBuffer[NETWORK_BUFFER_LENGTH_BYTES_STEREO + MAX_PACKET_HEADER_BYTES]; while (!_isFinished) { @@ -365,9 +361,11 @@ void AudioMixer::run() { if (node->getType() == NodeType::Agent && node->getActiveSocket() && node->getLinkedData() && ((AudioMixerClientData*) node->getLinkedData())->getAvatarAudioRingBuffer()) { prepareMixForListeningNode(node.data()); + + int numBytesPacketHeader = populatePacketHeader(clientMixBuffer, PacketTypeMixedAudio); - memcpy(_clientMixBuffer + numBytesPacketHeader, _clientSamples, NETWORK_BUFFER_LENGTH_BYTES_STEREO); - nodeList->writeDatagram(_clientMixBuffer, NETWORK_BUFFER_LENGTH_BYTES_STEREO + numBytesPacketHeader, node); + memcpy(clientMixBuffer + numBytesPacketHeader, _clientSamples, NETWORK_BUFFER_LENGTH_BYTES_STEREO); + nodeList->writeDatagram(clientMixBuffer, NETWORK_BUFFER_LENGTH_BYTES_STEREO + numBytesPacketHeader, node); } } diff --git a/assignment-client/src/audio/AudioMixer.h b/assignment-client/src/audio/AudioMixer.h index 0db205d845..5a68b0023f 100644 --- a/assignment-client/src/audio/AudioMixer.h +++ b/assignment-client/src/audio/AudioMixer.h @@ -28,8 +28,6 @@ public slots: void run(); void readPendingDatagrams(); -private slots: - void receivedSessionUUID(const QUuid& sessionUUID); private: /// adds one buffer to the mix for a listening node void addBufferToMixForListeningNodeWithBuffer(PositionalAudioRingBuffer* bufferToAdd, @@ -40,7 +38,6 @@ private: // client samples capacity is larger than what will be sent to optimize mixing int16_t _clientSamples[NETWORK_BUFFER_LENGTH_SAMPLES_STEREO + SAMPLE_PHASE_DELAY_AT_90]; - char _clientMixBuffer[NETWORK_BUFFER_LENGTH_BYTES_STEREO + MAX_PACKET_HEADER_BYTES]; }; #endif /* defined(__hifi__AudioMixer__) */ From 1fc9ac18151876edd3bc38a1a058e9453caf8ad3 Mon Sep 17 00:00:00 2001 From: Stephen Birarda Date: Mon, 17 Mar 2014 17:55:47 -0700 Subject: [PATCH 7/8] make the client mix buffer only as large as it needs to be --- assignment-client/src/audio/AudioMixer.cpp | 5 ++++- 1 file changed, 4 insertions(+), 1 deletion(-) diff --git a/assignment-client/src/audio/AudioMixer.cpp b/assignment-client/src/audio/AudioMixer.cpp index c5c112d0f0..a2c56405ec 100644 --- a/assignment-client/src/audio/AudioMixer.cpp +++ b/assignment-client/src/audio/AudioMixer.cpp @@ -347,7 +347,8 @@ void AudioMixer::run() { gettimeofday(&startTime, NULL); - char clientMixBuffer[NETWORK_BUFFER_LENGTH_BYTES_STEREO + MAX_PACKET_HEADER_BYTES]; + char* clientMixBuffer = new char[NETWORK_BUFFER_LENGTH_BYTES_STEREO + + numBytesForPacketHeaderGivenPacketType(PacketTypeMixedAudio)]; while (!_isFinished) { @@ -391,4 +392,6 @@ void AudioMixer::run() { } } + + delete[] clientMixBuffer; } From d80f110961277728a6022c5c569b23963d61b0c8 Mon Sep 17 00:00:00 2001 From: Andrzej Kapolka Date: Tue, 18 Mar 2014 09:12:23 -0700 Subject: [PATCH 8/8] Prevent audio average from becoming less than zero (thus causing sqrt to return nan, thus making our fake blendshape coefficient nan, thus making the jaw disappear). Closes #2334. --- interface/src/avatar/Head.cpp | 3 +-- 1 file changed, 1 insertion(+), 2 deletions(-) diff --git a/interface/src/avatar/Head.cpp b/interface/src/avatar/Head.cpp index a9b85ffce2..44001a2015 100644 --- a/interface/src/avatar/Head.cpp +++ b/interface/src/avatar/Head.cpp @@ -90,8 +90,7 @@ void Head::simulate(float deltaTime, bool isMine, bool billboard) { _saccade += (_saccadeTarget - _saccade) * 0.50f; const float AUDIO_AVERAGING_SECS = 0.05f; - _averageLoudness = (1.f - deltaTime / AUDIO_AVERAGING_SECS) * _averageLoudness + - (deltaTime / AUDIO_AVERAGING_SECS) * _audioLoudness; + _averageLoudness = glm::mix(_averageLoudness, _audioLoudness, glm::min(deltaTime / AUDIO_AVERAGING_SECS, 1.0f)); // Detect transition from talking to not; force blink after that and a delay bool forceBlink = false;