From 9f24bd9c4718b359b173c396622da0aab78376d7 Mon Sep 17 00:00:00 2001 From: Stephen Birarda Date: Mon, 17 Mar 2014 15:12:02 -0700 Subject: [PATCH 1/6] have the AudioMixer handle silent audio, send silence from Interface when gate closed --- interface/src/Audio.cpp | 48 +++++++++++++++---- libraries/audio/src/AudioRingBuffer.cpp | 13 +++++ libraries/audio/src/AudioRingBuffer.h | 2 + .../audio/src/PositionalAudioRingBuffer.cpp | 15 +++++- libraries/shared/src/PacketHeaders.h | 2 +- 5 files changed, 67 insertions(+), 13 deletions(-) diff --git a/interface/src/Audio.cpp b/interface/src/Audio.cpp index cf51a479ab..dcb955597e 100644 --- a/interface/src/Audio.cpp +++ b/interface/src/Audio.cpp @@ -436,9 +436,8 @@ void Audio::handleAudioInput() { } } if (!_noiseGateOpen) { - for (int i = 0; i < NETWORK_BUFFER_LENGTH_SAMPLES_PER_CHANNEL; i++) { - monoAudioSamples[i] = 0; - } + memset(monoAudioSamples, 0, NETWORK_BUFFER_LENGTH_SAMPLES_PER_CHANNEL); + _lastInputLoudness = 0; } } @@ -482,9 +481,26 @@ void Audio::handleAudioInput() { // we need the amount of bytes in the buffer + 1 for type // + 12 for 3 floats for position + float for bearing + 1 attenuation byte - - PacketType packetType = Menu::getInstance()->isOptionChecked(MenuOption::EchoServerAudio) - ? PacketTypeMicrophoneAudioWithEcho : PacketTypeMicrophoneAudioNoEcho; + + int numAudioBytes = 0; + + PacketType packetType; + if (_lastInputLoudness == 0) { + packetType = PacketTypeSilentAudioListener; + + // we need to indicate how many silent samples this is to the audio mixer + monoAudioSamples[0] = NETWORK_BUFFER_LENGTH_SAMPLES_PER_CHANNEL; + numAudioBytes = sizeof(int16_t); + + } else { + numAudioBytes = NETWORK_BUFFER_LENGTH_BYTES_PER_CHANNEL; + + if (Menu::getInstance()->isOptionChecked(MenuOption::EchoServerAudio)) { + packetType = PacketTypeMicrophoneAudioWithEcho; + } else { + packetType = PacketTypeMicrophoneAudioNoEcho; + } + } char* currentPacketPtr = monoAudioDataPacket + populatePacketHeader(monoAudioDataPacket, packetType); @@ -495,10 +511,8 @@ void Audio::handleAudioInput() { // memcpy our orientation memcpy(currentPacketPtr, &headOrientation, sizeof(headOrientation)); currentPacketPtr += sizeof(headOrientation); - - nodeList->writeDatagram(monoAudioDataPacket, - NETWORK_BUFFER_LENGTH_BYTES_PER_CHANNEL + leadingBytes, - audioMixer); + + nodeList->writeDatagram(monoAudioDataPacket, numAudioBytes + leadingBytes, audioMixer); Application::getInstance()->getBandwidthMeter()->outputStream(BandwidthMeter::AUDIO) .updateValue(NETWORK_BUFFER_LENGTH_BYTES_PER_CHANNEL + leadingBytes); @@ -638,7 +652,14 @@ void Audio::addProceduralSounds(int16_t* monoInput, int numSamples) { int16_t collisionSample = (int16_t) sample; + _lastInputLoudness = 0; + monoInput[i] = glm::clamp(monoInput[i] + collisionSample, MIN_SAMPLE_VALUE, MAX_SAMPLE_VALUE); + + _lastInputLoudness += fabsf(monoInput[i]); + _lastInputLoudness /= numSamples; + _lastInputLoudness /= MAX_SAMPLE_VALUE; + _localProceduralSamples[i] = glm::clamp(_localProceduralSamples[i] + collisionSample, MIN_SAMPLE_VALUE, MAX_SAMPLE_VALUE); @@ -662,7 +683,14 @@ void Audio::addProceduralSounds(int16_t* monoInput, int numSamples) { int16_t collisionSample = (int16_t) sample; + _lastInputLoudness = 0; + monoInput[i] = glm::clamp(monoInput[i] + collisionSample, MIN_SAMPLE_VALUE, MAX_SAMPLE_VALUE); + + _lastInputLoudness += fabsf(monoInput[i]); + _lastInputLoudness /= numSamples; + _lastInputLoudness /= MAX_SAMPLE_VALUE; + _localProceduralSamples[i] = glm::clamp(_localProceduralSamples[i] + collisionSample, MIN_SAMPLE_VALUE, MAX_SAMPLE_VALUE); diff --git a/libraries/audio/src/AudioRingBuffer.cpp b/libraries/audio/src/AudioRingBuffer.cpp index d07a334d81..ba21c6c9b9 100644 --- a/libraries/audio/src/AudioRingBuffer.cpp +++ b/libraries/audio/src/AudioRingBuffer.cpp @@ -156,6 +156,19 @@ unsigned int AudioRingBuffer::samplesAvailable() const { } } +void AudioRingBuffer::addSilentFrame(int numSilentSamples) { + // setup the silent frame + int numSilentBytes = numSilentSamples * sizeof(int16_t); + char* silentFrame = new char[numSilentBytes]; + memset(silentFrame, 0, numSilentBytes); + + // write it + writeData(silentFrame, numSilentBytes); + + // delete the temporary silent frame + delete[] silentFrame; +} + bool AudioRingBuffer::isNotStarvedOrHasMinimumSamples(unsigned int numRequiredSamples) const { if (!_isStarved) { return true; diff --git a/libraries/audio/src/AudioRingBuffer.h b/libraries/audio/src/AudioRingBuffer.h index c4d3f87814..0beb45ca18 100644 --- a/libraries/audio/src/AudioRingBuffer.h +++ b/libraries/audio/src/AudioRingBuffer.h @@ -70,6 +70,8 @@ public: void setIsStarved(bool isStarved) { _isStarved = isStarved; } bool hasStarted() const { return _hasStarted; } + + void addSilentFrame(int numSilentSamples); protected: // disallow copying of AudioRingBuffer objects AudioRingBuffer(const AudioRingBuffer&); diff --git a/libraries/audio/src/PositionalAudioRingBuffer.cpp b/libraries/audio/src/PositionalAudioRingBuffer.cpp index 27b62a8a56..0ac53231a1 100644 --- a/libraries/audio/src/PositionalAudioRingBuffer.cpp +++ b/libraries/audio/src/PositionalAudioRingBuffer.cpp @@ -44,8 +44,19 @@ int PositionalAudioRingBuffer::parseData(const QByteArray& packet) { packetStream.skipRawData(numBytesForPacketHeader(packet)); packetStream.skipRawData(parsePositionalData(packet.mid(packetStream.device()->pos()))); - packetStream.skipRawData(writeData(packet.data() + packetStream.device()->pos(), - packet.size() - packetStream.device()->pos())); + + if (packetTypeForPacket(packet) == PacketTypeSilentAudioListener) { + // this source had no audio to send us, but this counts as a packet + // write silence equivalent to the number of silent samples they just sent us + int16_t numSilentSamples; + packetStream >> numSilentSamples; + addSilentFrame(numSilentSamples); + } else { + // there is audio data to read + packetStream.skipRawData(writeData(packet.data() + packetStream.device()->pos(), + packet.size() - packetStream.device()->pos())); + } + return packetStream.device()->pos(); } diff --git a/libraries/shared/src/PacketHeaders.h b/libraries/shared/src/PacketHeaders.h index 0122e36a61..6f87ac8e3f 100644 --- a/libraries/shared/src/PacketHeaders.h +++ b/libraries/shared/src/PacketHeaders.h @@ -32,7 +32,7 @@ enum PacketType { PacketTypeMicrophoneAudioNoEcho, PacketTypeMicrophoneAudioWithEcho, PacketTypeBulkAvatarData, - PacketTypeTransmitterData, // usable + PacketTypeSilentAudioListener, PacketTypeEnvironmentData, PacketTypeDomainListRequest, PacketTypeRequestAssignment, From 61594b9a1ebb9e723b3203c5c6344412cc6ac78e Mon Sep 17 00:00:00 2001 From: Stephen Birarda Date: Mon, 17 Mar 2014 16:27:49 -0700 Subject: [PATCH 2/6] fixes to silent audio frame sending --- assignment-client/src/audio/AudioMixer.cpp | 3 ++- assignment-client/src/audio/AudioMixerClientData.cpp | 3 ++- interface/src/Audio.cpp | 8 ++++---- libraries/audio/src/PositionalAudioRingBuffer.cpp | 5 +++-- libraries/shared/src/PacketHeaders.h | 2 +- 5 files changed, 12 insertions(+), 9 deletions(-) diff --git a/assignment-client/src/audio/AudioMixer.cpp b/assignment-client/src/audio/AudioMixer.cpp index aeabd1548b..54f9d962b0 100644 --- a/assignment-client/src/audio/AudioMixer.cpp +++ b/assignment-client/src/audio/AudioMixer.cpp @@ -322,7 +322,8 @@ void AudioMixer::readPendingDatagrams() { PacketType mixerPacketType = packetTypeForPacket(receivedPacket); if (mixerPacketType == PacketTypeMicrophoneAudioNoEcho || mixerPacketType == PacketTypeMicrophoneAudioWithEcho - || mixerPacketType == PacketTypeInjectAudio) { + || mixerPacketType == PacketTypeInjectAudio + || mixerPacketType == PacketTypeSilentAudioFrame) { nodeList->findNodeAndUpdateWithDataFromPacket(receivedPacket); } else { diff --git a/assignment-client/src/audio/AudioMixerClientData.cpp b/assignment-client/src/audio/AudioMixerClientData.cpp index 8907796094..b2da0a0aaa 100644 --- a/assignment-client/src/audio/AudioMixerClientData.cpp +++ b/assignment-client/src/audio/AudioMixerClientData.cpp @@ -41,7 +41,8 @@ AvatarAudioRingBuffer* AudioMixerClientData::getAvatarAudioRingBuffer() const { int AudioMixerClientData::parseData(const QByteArray& packet) { PacketType packetType = packetTypeForPacket(packet); if (packetType == PacketTypeMicrophoneAudioWithEcho - || packetType == PacketTypeMicrophoneAudioNoEcho) { + || packetType == PacketTypeMicrophoneAudioNoEcho + || packetType == PacketTypeSilentAudioFrame) { // grab the AvatarAudioRingBuffer from the vector (or create it if it doesn't exist) AvatarAudioRingBuffer* avatarRingBuffer = getAvatarAudioRingBuffer(); diff --git a/interface/src/Audio.cpp b/interface/src/Audio.cpp index dcb955597e..b684cec46e 100644 --- a/interface/src/Audio.cpp +++ b/interface/src/Audio.cpp @@ -436,7 +436,7 @@ void Audio::handleAudioInput() { } } if (!_noiseGateOpen) { - memset(monoAudioSamples, 0, NETWORK_BUFFER_LENGTH_SAMPLES_PER_CHANNEL); + memset(monoAudioSamples, 0, NETWORK_BUFFER_LENGTH_BYTES_PER_CHANNEL); _lastInputLoudness = 0; } } @@ -450,7 +450,7 @@ void Audio::handleAudioInput() { // our input loudness is 0, since we're muted _lastInputLoudness = 0; } - + // add procedural effects to the appropriate input samples addProceduralSounds(monoAudioSamples, NETWORK_BUFFER_LENGTH_SAMPLES_PER_CHANNEL); @@ -486,7 +486,7 @@ void Audio::handleAudioInput() { PacketType packetType; if (_lastInputLoudness == 0) { - packetType = PacketTypeSilentAudioListener; + packetType = PacketTypeSilentAudioFrame; // we need to indicate how many silent samples this is to the audio mixer monoAudioSamples[0] = NETWORK_BUFFER_LENGTH_SAMPLES_PER_CHANNEL; @@ -515,7 +515,7 @@ void Audio::handleAudioInput() { nodeList->writeDatagram(monoAudioDataPacket, numAudioBytes + leadingBytes, audioMixer); Application::getInstance()->getBandwidthMeter()->outputStream(BandwidthMeter::AUDIO) - .updateValue(NETWORK_BUFFER_LENGTH_BYTES_PER_CHANNEL + leadingBytes); + .updateValue(numAudioBytes + leadingBytes); } delete[] inputAudioSamples; } diff --git a/libraries/audio/src/PositionalAudioRingBuffer.cpp b/libraries/audio/src/PositionalAudioRingBuffer.cpp index 0ac53231a1..11e1a7c2ae 100644 --- a/libraries/audio/src/PositionalAudioRingBuffer.cpp +++ b/libraries/audio/src/PositionalAudioRingBuffer.cpp @@ -45,11 +45,12 @@ int PositionalAudioRingBuffer::parseData(const QByteArray& packet) { packetStream.skipRawData(parsePositionalData(packet.mid(packetStream.device()->pos()))); - if (packetTypeForPacket(packet) == PacketTypeSilentAudioListener) { + if (packetTypeForPacket(packet) == PacketTypeSilentAudioFrame) { // this source had no audio to send us, but this counts as a packet // write silence equivalent to the number of silent samples they just sent us int16_t numSilentSamples; - packetStream >> numSilentSamples; + packetStream.readRawData(reinterpret_cast(&numSilentSamples), sizeof(int16_t)); + addSilentFrame(numSilentSamples); } else { // there is audio data to read diff --git a/libraries/shared/src/PacketHeaders.h b/libraries/shared/src/PacketHeaders.h index 6f87ac8e3f..c6ce6bdd6b 100644 --- a/libraries/shared/src/PacketHeaders.h +++ b/libraries/shared/src/PacketHeaders.h @@ -32,7 +32,7 @@ enum PacketType { PacketTypeMicrophoneAudioNoEcho, PacketTypeMicrophoneAudioWithEcho, PacketTypeBulkAvatarData, - PacketTypeSilentAudioListener, + PacketTypeSilentAudioFrame, PacketTypeEnvironmentData, PacketTypeDomainListRequest, PacketTypeRequestAssignment, From d652cec7667466886961c9961c8ee16bfbce5a7e Mon Sep 17 00:00:00 2001 From: Stephen Birarda Date: Mon, 17 Mar 2014 16:30:58 -0700 Subject: [PATCH 3/6] some potential fixes to QByteArray crash --- assignment-client/src/audio/AudioMixer.cpp | 7 ++++--- 1 file changed, 4 insertions(+), 3 deletions(-) diff --git a/assignment-client/src/audio/AudioMixer.cpp b/assignment-client/src/audio/AudioMixer.cpp index 54f9d962b0..d4b5c91488 100644 --- a/assignment-client/src/audio/AudioMixer.cpp +++ b/assignment-client/src/audio/AudioMixer.cpp @@ -287,7 +287,7 @@ void AudioMixer::prepareMixForListeningNode(Node* node) { AvatarAudioRingBuffer* nodeRingBuffer = ((AudioMixerClientData*) node->getLinkedData())->getAvatarAudioRingBuffer(); // zero out the client mix for this node - memset(_clientSamples, 0, sizeof(_clientSamples)); + memset(_clientSamples, 0, NETWORK_BUFFER_LENGTH_BYTES_STEREO); // loop through all other nodes that have sufficient audio to mix foreach (const SharedNodePointer& otherNode, NodeList::getInstance()->getNodeHash()) { @@ -367,8 +367,9 @@ void AudioMixer::run() { if (node->getType() == NodeType::Agent && node->getActiveSocket() && node->getLinkedData() && ((AudioMixerClientData*) node->getLinkedData())->getAvatarAudioRingBuffer()) { prepareMixForListeningNode(node.data()); - - memcpy(_clientMixBuffer.data() + numBytesPacketHeader, _clientSamples, NETWORK_BUFFER_LENGTH_BYTES_STEREO); + + _clientMixBuffer.replace(numBytesPacketHeader, NETWORK_BUFFER_LENGTH_BYTES_STEREO, + reinterpret_cast(_clientSamples)); nodeList->writeDatagram(_clientMixBuffer, node); } } From 7e0da71527602d216c676dc72bb3b4ddf1bdc75e Mon Sep 17 00:00:00 2001 From: Stephen Birarda Date: Mon, 17 Mar 2014 16:38:39 -0700 Subject: [PATCH 4/6] have agent optionally send silent stream flag --- libraries/script-engine/src/ScriptEngine.cpp | 27 +++++++++++++++++--- 1 file changed, 24 insertions(+), 3 deletions(-) diff --git a/libraries/script-engine/src/ScriptEngine.cpp b/libraries/script-engine/src/ScriptEngine.cpp index 9a672317ae..cc61633e60 100644 --- a/libraries/script-engine/src/ScriptEngine.cpp +++ b/libraries/script-engine/src/ScriptEngine.cpp @@ -267,15 +267,36 @@ void ScriptEngine::run() { if (_avatarAudioBuffer && _numAvatarAudioBufferSamples > 0) { // if have an avatar audio stream then send it out to our audio-mixer - QByteArray audioPacket = byteArrayWithPopulatedHeader(PacketTypeMicrophoneAudioNoEcho); + + bool silentFrame = true; + + // check if the all of the _numAvatarAudioBufferSamples to be sent are silence + for (int i = 0; i < _numAvatarAudioBufferSamples; ++i) { + if (_avatarAudioBuffer[i] != 0) { + silentFrame = false; + break; + } + } + + QByteArray audioPacket = byteArrayWithPopulatedHeader(silentFrame + ? PacketTypeSilentAudioFrame + : PacketTypeMicrophoneAudioNoEcho); QDataStream packetStream(&audioPacket, QIODevice::Append); // use the orientation and position of this avatar for the source of this audio packetStream.writeRawData(reinterpret_cast(&_avatarData->getPosition()), sizeof(glm::vec3)); glm::quat headOrientation = _avatarData->getHeadOrientation(); packetStream.writeRawData(reinterpret_cast(&headOrientation), sizeof(glm::quat)); - packetStream.writeRawData(reinterpret_cast(_avatarAudioBuffer), - _numAvatarAudioBufferSamples * sizeof(int16_t)); + + if (silentFrame) { + // write the number of silent samples so the audio-mixer can uphold timing + int16_t numSilentSamples = _numAvatarAudioBufferSamples; + packetStream.writeRawData(reinterpret_cast(&numSilentSamples), sizeof(int16_t)); + } else { + // write the raw audio data + packetStream.writeRawData(reinterpret_cast(_avatarAudioBuffer), + _numAvatarAudioBufferSamples * sizeof(int16_t)); + } nodeList->broadcastToNodes(audioPacket, NodeSet() << NodeType::AudioMixer); } From 447f23a1c947ab41558f3892e0e1a1c8546db327 Mon Sep 17 00:00:00 2001 From: Stephen Birarda Date: Mon, 17 Mar 2014 16:56:29 -0700 Subject: [PATCH 5/6] repair _clientSamples copying into _clientMix --- assignment-client/src/audio/AudioMixer.cpp | 3 +-- 1 file changed, 1 insertion(+), 2 deletions(-) diff --git a/assignment-client/src/audio/AudioMixer.cpp b/assignment-client/src/audio/AudioMixer.cpp index d4b5c91488..e097fd22a1 100644 --- a/assignment-client/src/audio/AudioMixer.cpp +++ b/assignment-client/src/audio/AudioMixer.cpp @@ -368,8 +368,7 @@ void AudioMixer::run() { && ((AudioMixerClientData*) node->getLinkedData())->getAvatarAudioRingBuffer()) { prepareMixForListeningNode(node.data()); - _clientMixBuffer.replace(numBytesPacketHeader, NETWORK_BUFFER_LENGTH_BYTES_STEREO, - reinterpret_cast(_clientSamples)); + memcpy(_clientMixBuffer.data() + numBytesPacketHeader, _clientSamples, NETWORK_BUFFER_LENGTH_BYTES_STEREO); nodeList->writeDatagram(_clientMixBuffer, node); } } From 7ed982a07333cdfcb3721a9845e789c227b4d80b Mon Sep 17 00:00:00 2001 From: Stephen Birarda Date: Tue, 18 Mar 2014 11:12:36 -0700 Subject: [PATCH 6/6] cleanup writing of silent frame to ring buffer --- libraries/audio/src/AudioRingBuffer.cpp | 22 ++++++++++++---------- 1 file changed, 12 insertions(+), 10 deletions(-) diff --git a/libraries/audio/src/AudioRingBuffer.cpp b/libraries/audio/src/AudioRingBuffer.cpp index ba21c6c9b9..9cef64e216 100644 --- a/libraries/audio/src/AudioRingBuffer.cpp +++ b/libraries/audio/src/AudioRingBuffer.cpp @@ -157,16 +157,18 @@ unsigned int AudioRingBuffer::samplesAvailable() const { } void AudioRingBuffer::addSilentFrame(int numSilentSamples) { - // setup the silent frame - int numSilentBytes = numSilentSamples * sizeof(int16_t); - char* silentFrame = new char[numSilentBytes]; - memset(silentFrame, 0, numSilentBytes); - - // write it - writeData(silentFrame, numSilentBytes); - - // delete the temporary silent frame - delete[] silentFrame; + // memset zeroes into the buffer, accomodate a wrap around the end + // push the _endOfLastWrite to the correct spot + if (_endOfLastWrite + numSilentSamples <= _buffer + _sampleCapacity) { + memset(_endOfLastWrite, 0, numSilentSamples * sizeof(int16_t)); + _endOfLastWrite += numSilentSamples; + } else { + int numSamplesToEnd = (_buffer + _sampleCapacity) - _endOfLastWrite; + memset(_endOfLastWrite, 0, numSilentSamples * sizeof(int16_t)); + memset(_buffer, 0, (numSilentSamples - numSamplesToEnd) * sizeof(int16_t)); + + _endOfLastWrite = _buffer + (numSilentSamples - numSamplesToEnd); + } } bool AudioRingBuffer::isNotStarvedOrHasMinimumSamples(unsigned int numRequiredSamples) const {