diff --git a/assignment-client/src/audio/AudioMixer.cpp b/assignment-client/src/audio/AudioMixer.cpp index a2c56405ec..988bcc1da7 100644 --- a/assignment-client/src/audio/AudioMixer.cpp +++ b/assignment-client/src/audio/AudioMixer.cpp @@ -286,7 +286,7 @@ void AudioMixer::prepareMixForListeningNode(Node* node) { AvatarAudioRingBuffer* nodeRingBuffer = ((AudioMixerClientData*) node->getLinkedData())->getAvatarAudioRingBuffer(); // zero out the client mix for this node - memset(_clientSamples, 0, sizeof(_clientSamples)); + memset(_clientSamples, 0, NETWORK_BUFFER_LENGTH_BYTES_STEREO); // loop through all other nodes that have sufficient audio to mix foreach (const SharedNodePointer& otherNode, NodeList::getInstance()->getNodeHash()) { @@ -321,7 +321,8 @@ void AudioMixer::readPendingDatagrams() { PacketType mixerPacketType = packetTypeForPacket(receivedPacket); if (mixerPacketType == PacketTypeMicrophoneAudioNoEcho || mixerPacketType == PacketTypeMicrophoneAudioWithEcho - || mixerPacketType == PacketTypeInjectAudio) { + || mixerPacketType == PacketTypeInjectAudio + || mixerPacketType == PacketTypeSilentAudioFrame) { nodeList->findNodeAndUpdateWithDataFromPacket(receivedPacket); } else { diff --git a/assignment-client/src/audio/AudioMixerClientData.cpp b/assignment-client/src/audio/AudioMixerClientData.cpp index 8907796094..b2da0a0aaa 100644 --- a/assignment-client/src/audio/AudioMixerClientData.cpp +++ b/assignment-client/src/audio/AudioMixerClientData.cpp @@ -41,7 +41,8 @@ AvatarAudioRingBuffer* AudioMixerClientData::getAvatarAudioRingBuffer() const { int AudioMixerClientData::parseData(const QByteArray& packet) { PacketType packetType = packetTypeForPacket(packet); if (packetType == PacketTypeMicrophoneAudioWithEcho - || packetType == PacketTypeMicrophoneAudioNoEcho) { + || packetType == PacketTypeMicrophoneAudioNoEcho + || packetType == PacketTypeSilentAudioFrame) { // grab the AvatarAudioRingBuffer from the vector (or create it if it doesn't exist) AvatarAudioRingBuffer* avatarRingBuffer = getAvatarAudioRingBuffer(); diff --git a/interface/src/Audio.cpp b/interface/src/Audio.cpp index cf51a479ab..b684cec46e 100644 --- a/interface/src/Audio.cpp +++ b/interface/src/Audio.cpp @@ -436,9 +436,8 @@ void Audio::handleAudioInput() { } } if (!_noiseGateOpen) { - for (int i = 0; i < NETWORK_BUFFER_LENGTH_SAMPLES_PER_CHANNEL; i++) { - monoAudioSamples[i] = 0; - } + memset(monoAudioSamples, 0, NETWORK_BUFFER_LENGTH_BYTES_PER_CHANNEL); + _lastInputLoudness = 0; } } @@ -451,7 +450,7 @@ void Audio::handleAudioInput() { // our input loudness is 0, since we're muted _lastInputLoudness = 0; } - + // add procedural effects to the appropriate input samples addProceduralSounds(monoAudioSamples, NETWORK_BUFFER_LENGTH_SAMPLES_PER_CHANNEL); @@ -482,9 +481,26 @@ void Audio::handleAudioInput() { // we need the amount of bytes in the buffer + 1 for type // + 12 for 3 floats for position + float for bearing + 1 attenuation byte - - PacketType packetType = Menu::getInstance()->isOptionChecked(MenuOption::EchoServerAudio) - ? PacketTypeMicrophoneAudioWithEcho : PacketTypeMicrophoneAudioNoEcho; + + int numAudioBytes = 0; + + PacketType packetType; + if (_lastInputLoudness == 0) { + packetType = PacketTypeSilentAudioFrame; + + // we need to indicate how many silent samples this is to the audio mixer + monoAudioSamples[0] = NETWORK_BUFFER_LENGTH_SAMPLES_PER_CHANNEL; + numAudioBytes = sizeof(int16_t); + + } else { + numAudioBytes = NETWORK_BUFFER_LENGTH_BYTES_PER_CHANNEL; + + if (Menu::getInstance()->isOptionChecked(MenuOption::EchoServerAudio)) { + packetType = PacketTypeMicrophoneAudioWithEcho; + } else { + packetType = PacketTypeMicrophoneAudioNoEcho; + } + } char* currentPacketPtr = monoAudioDataPacket + populatePacketHeader(monoAudioDataPacket, packetType); @@ -495,13 +511,11 @@ void Audio::handleAudioInput() { // memcpy our orientation memcpy(currentPacketPtr, &headOrientation, sizeof(headOrientation)); currentPacketPtr += sizeof(headOrientation); - - nodeList->writeDatagram(monoAudioDataPacket, - NETWORK_BUFFER_LENGTH_BYTES_PER_CHANNEL + leadingBytes, - audioMixer); + + nodeList->writeDatagram(monoAudioDataPacket, numAudioBytes + leadingBytes, audioMixer); Application::getInstance()->getBandwidthMeter()->outputStream(BandwidthMeter::AUDIO) - .updateValue(NETWORK_BUFFER_LENGTH_BYTES_PER_CHANNEL + leadingBytes); + .updateValue(numAudioBytes + leadingBytes); } delete[] inputAudioSamples; } @@ -638,7 +652,14 @@ void Audio::addProceduralSounds(int16_t* monoInput, int numSamples) { int16_t collisionSample = (int16_t) sample; + _lastInputLoudness = 0; + monoInput[i] = glm::clamp(monoInput[i] + collisionSample, MIN_SAMPLE_VALUE, MAX_SAMPLE_VALUE); + + _lastInputLoudness += fabsf(monoInput[i]); + _lastInputLoudness /= numSamples; + _lastInputLoudness /= MAX_SAMPLE_VALUE; + _localProceduralSamples[i] = glm::clamp(_localProceduralSamples[i] + collisionSample, MIN_SAMPLE_VALUE, MAX_SAMPLE_VALUE); @@ -662,7 +683,14 @@ void Audio::addProceduralSounds(int16_t* monoInput, int numSamples) { int16_t collisionSample = (int16_t) sample; + _lastInputLoudness = 0; + monoInput[i] = glm::clamp(monoInput[i] + collisionSample, MIN_SAMPLE_VALUE, MAX_SAMPLE_VALUE); + + _lastInputLoudness += fabsf(monoInput[i]); + _lastInputLoudness /= numSamples; + _lastInputLoudness /= MAX_SAMPLE_VALUE; + _localProceduralSamples[i] = glm::clamp(_localProceduralSamples[i] + collisionSample, MIN_SAMPLE_VALUE, MAX_SAMPLE_VALUE); diff --git a/libraries/audio/src/AudioRingBuffer.cpp b/libraries/audio/src/AudioRingBuffer.cpp index d07a334d81..9cef64e216 100644 --- a/libraries/audio/src/AudioRingBuffer.cpp +++ b/libraries/audio/src/AudioRingBuffer.cpp @@ -156,6 +156,21 @@ unsigned int AudioRingBuffer::samplesAvailable() const { } } +void AudioRingBuffer::addSilentFrame(int numSilentSamples) { + // memset zeroes into the buffer, accomodate a wrap around the end + // push the _endOfLastWrite to the correct spot + if (_endOfLastWrite + numSilentSamples <= _buffer + _sampleCapacity) { + memset(_endOfLastWrite, 0, numSilentSamples * sizeof(int16_t)); + _endOfLastWrite += numSilentSamples; + } else { + int numSamplesToEnd = (_buffer + _sampleCapacity) - _endOfLastWrite; + memset(_endOfLastWrite, 0, numSilentSamples * sizeof(int16_t)); + memset(_buffer, 0, (numSilentSamples - numSamplesToEnd) * sizeof(int16_t)); + + _endOfLastWrite = _buffer + (numSilentSamples - numSamplesToEnd); + } +} + bool AudioRingBuffer::isNotStarvedOrHasMinimumSamples(unsigned int numRequiredSamples) const { if (!_isStarved) { return true; diff --git a/libraries/audio/src/AudioRingBuffer.h b/libraries/audio/src/AudioRingBuffer.h index c4d3f87814..0beb45ca18 100644 --- a/libraries/audio/src/AudioRingBuffer.h +++ b/libraries/audio/src/AudioRingBuffer.h @@ -70,6 +70,8 @@ public: void setIsStarved(bool isStarved) { _isStarved = isStarved; } bool hasStarted() const { return _hasStarted; } + + void addSilentFrame(int numSilentSamples); protected: // disallow copying of AudioRingBuffer objects AudioRingBuffer(const AudioRingBuffer&); diff --git a/libraries/audio/src/PositionalAudioRingBuffer.cpp b/libraries/audio/src/PositionalAudioRingBuffer.cpp index 27b62a8a56..11e1a7c2ae 100644 --- a/libraries/audio/src/PositionalAudioRingBuffer.cpp +++ b/libraries/audio/src/PositionalAudioRingBuffer.cpp @@ -44,8 +44,20 @@ int PositionalAudioRingBuffer::parseData(const QByteArray& packet) { packetStream.skipRawData(numBytesForPacketHeader(packet)); packetStream.skipRawData(parsePositionalData(packet.mid(packetStream.device()->pos()))); - packetStream.skipRawData(writeData(packet.data() + packetStream.device()->pos(), - packet.size() - packetStream.device()->pos())); + + if (packetTypeForPacket(packet) == PacketTypeSilentAudioFrame) { + // this source had no audio to send us, but this counts as a packet + // write silence equivalent to the number of silent samples they just sent us + int16_t numSilentSamples; + packetStream.readRawData(reinterpret_cast(&numSilentSamples), sizeof(int16_t)); + + addSilentFrame(numSilentSamples); + } else { + // there is audio data to read + packetStream.skipRawData(writeData(packet.data() + packetStream.device()->pos(), + packet.size() - packetStream.device()->pos())); + } + return packetStream.device()->pos(); } diff --git a/libraries/script-engine/src/ScriptEngine.cpp b/libraries/script-engine/src/ScriptEngine.cpp index 9a672317ae..cc61633e60 100644 --- a/libraries/script-engine/src/ScriptEngine.cpp +++ b/libraries/script-engine/src/ScriptEngine.cpp @@ -267,15 +267,36 @@ void ScriptEngine::run() { if (_avatarAudioBuffer && _numAvatarAudioBufferSamples > 0) { // if have an avatar audio stream then send it out to our audio-mixer - QByteArray audioPacket = byteArrayWithPopulatedHeader(PacketTypeMicrophoneAudioNoEcho); + + bool silentFrame = true; + + // check if the all of the _numAvatarAudioBufferSamples to be sent are silence + for (int i = 0; i < _numAvatarAudioBufferSamples; ++i) { + if (_avatarAudioBuffer[i] != 0) { + silentFrame = false; + break; + } + } + + QByteArray audioPacket = byteArrayWithPopulatedHeader(silentFrame + ? PacketTypeSilentAudioFrame + : PacketTypeMicrophoneAudioNoEcho); QDataStream packetStream(&audioPacket, QIODevice::Append); // use the orientation and position of this avatar for the source of this audio packetStream.writeRawData(reinterpret_cast(&_avatarData->getPosition()), sizeof(glm::vec3)); glm::quat headOrientation = _avatarData->getHeadOrientation(); packetStream.writeRawData(reinterpret_cast(&headOrientation), sizeof(glm::quat)); - packetStream.writeRawData(reinterpret_cast(_avatarAudioBuffer), - _numAvatarAudioBufferSamples * sizeof(int16_t)); + + if (silentFrame) { + // write the number of silent samples so the audio-mixer can uphold timing + int16_t numSilentSamples = _numAvatarAudioBufferSamples; + packetStream.writeRawData(reinterpret_cast(&numSilentSamples), sizeof(int16_t)); + } else { + // write the raw audio data + packetStream.writeRawData(reinterpret_cast(_avatarAudioBuffer), + _numAvatarAudioBufferSamples * sizeof(int16_t)); + } nodeList->broadcastToNodes(audioPacket, NodeSet() << NodeType::AudioMixer); } diff --git a/libraries/shared/src/PacketHeaders.h b/libraries/shared/src/PacketHeaders.h index 0122e36a61..c6ce6bdd6b 100644 --- a/libraries/shared/src/PacketHeaders.h +++ b/libraries/shared/src/PacketHeaders.h @@ -32,7 +32,7 @@ enum PacketType { PacketTypeMicrophoneAudioNoEcho, PacketTypeMicrophoneAudioWithEcho, PacketTypeBulkAvatarData, - PacketTypeTransmitterData, // usable + PacketTypeSilentAudioFrame, PacketTypeEnvironmentData, PacketTypeDomainListRequest, PacketTypeRequestAssignment,