From 4460e23b680a610df28bd58602f993d6503d1cd5 Mon Sep 17 00:00:00 2001 From: Atlante45 Date: Wed, 5 Nov 2014 23:08:00 +0100 Subject: [PATCH 1/7] Suppress repeated Hash mismatch debug messages --- libraries/networking/src/LimitedNodeList.cpp | 9 ++++++++- 1 file changed, 8 insertions(+), 1 deletion(-) diff --git a/libraries/networking/src/LimitedNodeList.cpp b/libraries/networking/src/LimitedNodeList.cpp index 043f0621bb..0c18c82962 100644 --- a/libraries/networking/src/LimitedNodeList.cpp +++ b/libraries/networking/src/LimitedNodeList.cpp @@ -209,8 +209,15 @@ bool LimitedNodeList::packetVersionAndHashMatch(const QByteArray& packet) { if (hashFromPacketHeader(packet) == hashForPacketAndConnectionUUID(packet, sendingNode->getConnectionSecret())) { return true; } else { - qDebug() << "Packet hash mismatch on" << checkType << "- Sender" + static QMultiMap hashDebugSuppressMap; + + QUuid senderUUID = uuidFromPacketHeader(packet); + if (!hashDebugSuppressMap.contains(senderUUID, checkType)) { + qDebug() << "Packet hash mismatch on" << checkType << "- Sender" << uuidFromPacketHeader(packet); + + hashDebugSuppressMap.insert(senderUUID, checkType); + } } } else { static QString repeatedMessage From ff197a2f64bfc2da98d9f98523293cc89bf086f5 Mon Sep 17 00:00:00 2001 From: Atlante45 Date: Thu, 6 Nov 2014 12:46:37 +0100 Subject: [PATCH 2/7] Base implementation for own reverb --- interface/src/Audio.cpp | 140 +++++++++++++++++++++++++--------------- interface/src/Audio.h | 3 + 2 files changed, 91 insertions(+), 52 deletions(-) diff --git a/interface/src/Audio.cpp b/interface/src/Audio.cpp index 7d039387bb..2b38782231 100644 --- a/interface/src/Audio.cpp +++ b/interface/src/Audio.cpp @@ -510,6 +510,33 @@ void Audio::initGverb() { gverb_set_taillevel(_gverb, DB_CO(_reverbOptions->getTailLevel())); } +void Audio::updateGverbOptions() { + bool reverbChanged = false; + if (_receivedAudioStream.hasReverb()) { + + if (_zoneReverbOptions.getReverbTime() != _receivedAudioStream.getRevebTime()) { + _zoneReverbOptions.setReverbTime(_receivedAudioStream.getRevebTime()); + reverbChanged = true; + } + if (_zoneReverbOptions.getWetLevel() != _receivedAudioStream.getWetLevel()) { + _zoneReverbOptions.setWetLevel(_receivedAudioStream.getWetLevel()); + reverbChanged = true; + } + + if (_reverbOptions != &_zoneReverbOptions) { + _reverbOptions = &_zoneReverbOptions; + reverbChanged = true; + } + } else if (_reverbOptions != &_scriptReverbOptions) { + _reverbOptions = &_scriptReverbOptions; + reverbChanged = true; + } + + if (reverbChanged) { + initGverb(); + } +} + void Audio::setReverbOptions(const AudioEffectOptions* options) { // Save the new options _scriptReverbOptions.setMaxRoomSize(options->getMaxRoomSize()); @@ -557,6 +584,62 @@ void Audio::addReverb(int16_t* samplesData, int numSamples, QAudioFormat& audioF } } +void Audio::handleLocalEchoAndReverb(QByteArray inputByteArray) { + bool hasEcho = Menu::getInstance()->isOptionChecked(MenuOption::EchoLocalAudio); + bool hasLocalReverb = (_reverb || _receivedAudioStream.hasReverb()) && + !Menu::getInstance()->isOptionChecked(MenuOption::EchoServerAudio); + if (_muted || !_audioOutput || (!hasEcho && !hasLocalReverb)) { + //qDebug() << "No Echo/Reverb:" << hasEcho << _reverb << _receivedAudioStream.hasReverb(); + return; + } + + // if this person wants local loopback add that to the locally injected audio + // if there is reverb apply it to local audio and substract the origin samples + + if (!_loopbackOutputDevice && _loopbackAudioOutput) { + // we didn't have the loopback output device going so set that up now + _loopbackOutputDevice = _loopbackAudioOutput->start(); + } + + QByteArray loopBackByteArray(inputByteArray); + if (_inputFormat != _outputFormat) { + float loopbackOutputToInputRatio = (_outputFormat.sampleRate() / (float) _inputFormat.sampleRate()) * + (_outputFormat.channelCount() / _inputFormat.channelCount()); + loopBackByteArray.resize(inputByteArray.size() * loopbackOutputToInputRatio); + loopBackByteArray.fill(0); + linearResampling((int16_t*)inputByteArray.data(), (int16_t*)loopBackByteArray.data(), + inputByteArray.size() / sizeof(int16_t), loopBackByteArray.size() / sizeof(int16_t), + _inputFormat, _outputFormat); + } + + if (hasLocalReverb) { + //qDebug() << "Has Reverb"; + QByteArray loopbackCopy; + if (!hasEcho) { + loopbackCopy = loopBackByteArray; + } + + int16_t* loopbackSamples = (int16_t*) loopBackByteArray.data(); + int numLoopbackSamples = loopBackByteArray.size() / sizeof(int16_t); + updateGverbOptions(); + addReverb(loopbackSamples, numLoopbackSamples, _outputFormat); + + if (!hasEcho) { + int16_t* loopbackCopySamples = (int16_t*) loopbackCopy.data(); + + for (int i = 0; i < numLoopbackSamples; ++i) { + loopbackSamples[i] = glm::clamp((int)loopbackSamples[i] - loopbackCopySamples[i], + -32768, 32767); + } + } + } + + if (_loopbackOutputDevice) { + //qDebug() << "Writing"; + _loopbackOutputDevice->write(loopBackByteArray); + } +} + void Audio::handleAudioInput() { static char audioDataPacket[MAX_PACKET_SIZE]; @@ -601,34 +684,8 @@ void Audio::handleAudioInput() { _inputFrameBuffer.copyFrames(1, inputFrameCount, inputFrameData, true /*copy out*/); } - - if (Menu::getInstance()->isOptionChecked(MenuOption::EchoLocalAudio) && !_muted && _audioOutput) { - // if this person wants local loopback add that to the locally injected audio - - if (!_loopbackOutputDevice && _loopbackAudioOutput) { - // we didn't have the loopback output device going so set that up now - _loopbackOutputDevice = _loopbackAudioOutput->start(); - } - - if (_inputFormat == _outputFormat) { - if (_loopbackOutputDevice) { - _loopbackOutputDevice->write(inputByteArray); - } - } else { - float loopbackOutputToInputRatio = (_outputFormat.sampleRate() / (float) _inputFormat.sampleRate()) - * (_outputFormat.channelCount() / _inputFormat.channelCount()); - - QByteArray loopBackByteArray(inputByteArray.size() * loopbackOutputToInputRatio, 0); - - linearResampling((int16_t*) inputByteArray.data(), (int16_t*) loopBackByteArray.data(), - inputByteArray.size() / sizeof(int16_t), - loopBackByteArray.size() / sizeof(int16_t), _inputFormat, _outputFormat); - - if (_loopbackOutputDevice) { - _loopbackOutputDevice->write(loopBackByteArray); - } - } - } + + handleLocalEchoAndReverb(inputByteArray); _inputRingBuffer.writeData(inputByteArray.data(), inputByteArray.size()); @@ -951,34 +1008,13 @@ void Audio::processReceivedSamples(const QByteArray& inputBuffer, QByteArray& ou _desiredOutputFormat, _outputFormat); if(_reverb || _receivedAudioStream.hasReverb()) { - bool reverbChanged = false; - if (_receivedAudioStream.hasReverb()) { - - if (_zoneReverbOptions.getReverbTime() != _receivedAudioStream.getRevebTime()) { - _zoneReverbOptions.setReverbTime(_receivedAudioStream.getRevebTime()); - reverbChanged = true; - } - if (_zoneReverbOptions.getWetLevel() != _receivedAudioStream.getWetLevel()) { - _zoneReverbOptions.setWetLevel(_receivedAudioStream.getWetLevel()); - reverbChanged = true; - } - - if (_reverbOptions != &_zoneReverbOptions) { - _reverbOptions = &_zoneReverbOptions; - reverbChanged = true; - } - } else if (_reverbOptions != &_scriptReverbOptions) { - _reverbOptions = &_scriptReverbOptions; - reverbChanged = true; - } - - if (reverbChanged) { - initGverb(); - } + updateGverbOptions(); addReverb((int16_t*)outputBuffer.data(), numDeviceOutputSamples, _outputFormat); } } + + void Audio::addReceivedAudioToStream(const QByteArray& audioByteArray) { if (_audioOutput) { // Audio output must exist and be correctly set up if we're going to process received audio diff --git a/interface/src/Audio.h b/interface/src/Audio.h index fcbfb12761..13c7930478 100644 --- a/interface/src/Audio.h +++ b/interface/src/Audio.h @@ -267,8 +267,11 @@ private: // Adds Reverb void initGverb(); + void updateGverbOptions(); void addReverb(int16_t* samples, int numSamples, QAudioFormat& format); + void handleLocalEchoAndReverb(QByteArray inputByteArray); + // Add sounds that we want the user to not hear themselves, by adding on top of mic input signal void addProceduralSounds(int16_t* monoInput, int numSamples); From efc86b9f75128dc26642689ba2d55b48c526de61 Mon Sep 17 00:00:00 2001 From: Atlante45 Date: Thu, 6 Nov 2014 17:42:27 +0100 Subject: [PATCH 3/7] Remove debug --- interface/src/Audio.cpp | 4 ---- 1 file changed, 4 deletions(-) diff --git a/interface/src/Audio.cpp b/interface/src/Audio.cpp index 2b38782231..7a1bfbb1c7 100644 --- a/interface/src/Audio.cpp +++ b/interface/src/Audio.cpp @@ -589,7 +589,6 @@ void Audio::handleLocalEchoAndReverb(QByteArray inputByteArray) { bool hasLocalReverb = (_reverb || _receivedAudioStream.hasReverb()) && !Menu::getInstance()->isOptionChecked(MenuOption::EchoServerAudio); if (_muted || !_audioOutput || (!hasEcho && !hasLocalReverb)) { - //qDebug() << "No Echo/Reverb:" << hasEcho << _reverb << _receivedAudioStream.hasReverb(); return; } @@ -613,7 +612,6 @@ void Audio::handleLocalEchoAndReverb(QByteArray inputByteArray) { } if (hasLocalReverb) { - //qDebug() << "Has Reverb"; QByteArray loopbackCopy; if (!hasEcho) { loopbackCopy = loopBackByteArray; @@ -626,7 +624,6 @@ void Audio::handleLocalEchoAndReverb(QByteArray inputByteArray) { if (!hasEcho) { int16_t* loopbackCopySamples = (int16_t*) loopbackCopy.data(); - for (int i = 0; i < numLoopbackSamples; ++i) { loopbackSamples[i] = glm::clamp((int)loopbackSamples[i] - loopbackCopySamples[i], -32768, 32767); @@ -635,7 +632,6 @@ void Audio::handleLocalEchoAndReverb(QByteArray inputByteArray) { } if (_loopbackOutputDevice) { - //qDebug() << "Writing"; _loopbackOutputDevice->write(loopBackByteArray); } } From 44cb35778afa05c3a1ac6d27c6182ff95753aaf8 Mon Sep 17 00:00:00 2001 From: Atlante45 Date: Thu, 6 Nov 2014 17:42:57 +0100 Subject: [PATCH 4/7] Move AudioEnv packet send to own function Audio environment packet moved to own function and out of the if/else So it is now sent all the time, now matter if there are other people around you --- assignment-client/src/audio/AudioMixer.cpp | 113 +++++++++++---------- assignment-client/src/audio/AudioMixer.h | 3 + 2 files changed, 63 insertions(+), 53 deletions(-) diff --git a/assignment-client/src/audio/AudioMixer.cpp b/assignment-client/src/audio/AudioMixer.cpp index 6ca93a7b11..e217061826 100644 --- a/assignment-client/src/audio/AudioMixer.cpp +++ b/assignment-client/src/audio/AudioMixer.cpp @@ -463,6 +463,63 @@ int AudioMixer::prepareMixForListeningNode(Node* node) { return streamsMixed; } +void AudioMixer::sendAudioEnvironmentPacket(SharedNodePointer node) { + static char clientEnvBuffer[MAX_PACKET_SIZE]; + + // Send stream properties + bool hasReverb = false; + float reverbTime, wetLevel; + // find reverb properties + for (int i = 0; i < _zoneReverbSettings.size(); ++i) { + AudioMixerClientData* data = static_cast(node->getLinkedData()); + glm::vec3 streamPosition = data->getAvatarAudioStream()->getPosition(); + if (_audioZones[_zoneReverbSettings[i].zone].contains(streamPosition)) { + hasReverb = true; + reverbTime = _zoneReverbSettings[i].reverbTime; + wetLevel = _zoneReverbSettings[i].wetLevel; + break; + } + } + AudioMixerClientData* nodeData = static_cast(node->getLinkedData()); + AvatarAudioStream* stream = nodeData->getAvatarAudioStream(); + bool dataChanged = (stream->hasReverb() != hasReverb) || + (stream->hasReverb() && (stream->getRevebTime() != reverbTime || + stream->getWetLevel() != wetLevel)); + if (dataChanged) { + // Update stream + if (hasReverb) { + stream->setReverb(reverbTime, wetLevel); + } else { + stream->clearReverb(); + } + } + + // Send at change or every so often + float CHANCE_OF_SEND = 0.01f; + bool sendData = dataChanged || (randFloat() < CHANCE_OF_SEND); + + if (sendData) { + int numBytesEnvPacketHeader = populatePacketHeader(clientEnvBuffer, PacketTypeAudioEnvironment); + char* envDataAt = clientEnvBuffer + numBytesEnvPacketHeader; + + unsigned char bitset = 0; + if (hasReverb) { + setAtBit(bitset, HAS_REVERB_BIT); + } + + memcpy(envDataAt, &bitset, sizeof(unsigned char)); + envDataAt += sizeof(unsigned char); + + if (hasReverb) { + memcpy(envDataAt, &reverbTime, sizeof(float)); + envDataAt += sizeof(float); + memcpy(envDataAt, &wetLevel, sizeof(float)); + envDataAt += sizeof(float); + } + NodeList::getInstance()->writeDatagram(clientEnvBuffer, envDataAt - clientEnvBuffer, node); + } +} + void AudioMixer::readPendingDatagram(const QByteArray& receivedPacket, const HifiSockAddr& senderSockAddr) { NodeList* nodeList = NodeList::getInstance(); @@ -640,7 +697,6 @@ void AudioMixer::run() { timer.start(); char clientMixBuffer[MAX_PACKET_SIZE]; - char clientEnvBuffer[MAX_PACKET_SIZE]; int usecToSleep = BUFFER_SEND_INTERVAL_USECS; @@ -734,58 +790,6 @@ void AudioMixer::run() { // pack mixed audio samples memcpy(mixDataAt, _mixSamples, NETWORK_BUFFER_LENGTH_BYTES_STEREO); mixDataAt += NETWORK_BUFFER_LENGTH_BYTES_STEREO; - - // Send stream properties - bool hasReverb = false; - float reverbTime, wetLevel; - // find reverb properties - for (int i = 0; i < _zoneReverbSettings.size(); ++i) { - AudioMixerClientData* data = static_cast(node->getLinkedData()); - glm::vec3 streamPosition = data->getAvatarAudioStream()->getPosition(); - if (_audioZones[_zoneReverbSettings[i].zone].contains(streamPosition)) { - hasReverb = true; - reverbTime = _zoneReverbSettings[i].reverbTime; - wetLevel = _zoneReverbSettings[i].wetLevel; - break; - } - } - AvatarAudioStream* stream = nodeData->getAvatarAudioStream(); - bool dataChanged = (stream->hasReverb() != hasReverb) || - (stream->hasReverb() && (stream->getRevebTime() != reverbTime || - stream->getWetLevel() != wetLevel)); - if (dataChanged) { - // Update stream - if (hasReverb) { - stream->setReverb(reverbTime, wetLevel); - } else { - stream->clearReverb(); - } - } - - // Send at change or every so often - float CHANCE_OF_SEND = 0.01f; - bool sendData = dataChanged || (randFloat() < CHANCE_OF_SEND); - - if (sendData) { - int numBytesEnvPacketHeader = populatePacketHeader(clientEnvBuffer, PacketTypeAudioEnvironment); - char* envDataAt = clientEnvBuffer + numBytesEnvPacketHeader; - - unsigned char bitset = 0; - if (hasReverb) { - setAtBit(bitset, HAS_REVERB_BIT); - } - - memcpy(envDataAt, &bitset, sizeof(unsigned char)); - envDataAt += sizeof(unsigned char); - - if (hasReverb) { - memcpy(envDataAt, &reverbTime, sizeof(float)); - envDataAt += sizeof(float); - memcpy(envDataAt, &wetLevel, sizeof(float)); - envDataAt += sizeof(float); - } - nodeList->writeDatagram(clientEnvBuffer, envDataAt - clientEnvBuffer, node); - } } else { // pack header int numBytesPacketHeader = populatePacketHeader(clientMixBuffer, PacketTypeSilentAudioFrame); @@ -801,6 +805,9 @@ void AudioMixer::run() { memcpy(mixDataAt, &numSilentSamples, sizeof(quint16)); mixDataAt += sizeof(quint16); } + + // Send audio environment + sendAudioEnvironmentPacket(node); // send mixed audio packet nodeList->writeDatagram(clientMixBuffer, mixDataAt - clientMixBuffer, node); diff --git a/assignment-client/src/audio/AudioMixer.h b/assignment-client/src/audio/AudioMixer.h index ff976dec61..b4a9e2aa1c 100644 --- a/assignment-client/src/audio/AudioMixer.h +++ b/assignment-client/src/audio/AudioMixer.h @@ -49,6 +49,9 @@ private: /// prepares and sends a mix to one Node int prepareMixForListeningNode(Node* node); + + /// Send Audio Environment packet for a single node + void sendAudioEnvironmentPacket(SharedNodePointer node); // used on a per stream basis to run the filter on before mixing, large enough to handle the historical // data from a phase delay as well as an entire network buffer From 1e6cadc7c14da183069278632c2e31d8bec002d2 Mon Sep 17 00:00:00 2001 From: Atlante45 Date: Fri, 7 Nov 2014 00:06:57 +0100 Subject: [PATCH 5/7] Extra lines --- interface/src/Audio.cpp | 2 -- 1 file changed, 2 deletions(-) diff --git a/interface/src/Audio.cpp b/interface/src/Audio.cpp index 7fc563411a..6129ee4ee0 100644 --- a/interface/src/Audio.cpp +++ b/interface/src/Audio.cpp @@ -1026,8 +1026,6 @@ void Audio::processReceivedSamples(const QByteArray& inputBuffer, QByteArray& ou } } - - void Audio::addReceivedAudioToStream(const QByteArray& audioByteArray) { if (_audioOutput) { // Audio output must exist and be correctly set up if we're going to process received audio From dcfeef471250e2dfb6d320a1a6646584c59d765b Mon Sep 17 00:00:00 2001 From: Atlante45 Date: Fri, 7 Nov 2014 00:24:37 +0100 Subject: [PATCH 6/7] Reference and comments --- interface/src/Audio.cpp | 3 ++- interface/src/Audio.h | 2 +- 2 files changed, 3 insertions(+), 2 deletions(-) diff --git a/interface/src/Audio.cpp b/interface/src/Audio.cpp index 6129ee4ee0..79ebcdd043 100644 --- a/interface/src/Audio.cpp +++ b/interface/src/Audio.cpp @@ -587,8 +587,9 @@ void Audio::addReverb(int16_t* samplesData, int numSamples, QAudioFormat& audioF } } -void Audio::handleLocalEchoAndReverb(QByteArray inputByteArray) { +void Audio::handleLocalEchoAndReverb(QByteArray& inputByteArray) { bool hasEcho = Menu::getInstance()->isOptionChecked(MenuOption::EchoLocalAudio); + // If there is server echo, reverb will be applied to the recieved audio stream so no need to have it here. bool hasLocalReverb = (_reverb || _receivedAudioStream.hasReverb()) && !Menu::getInstance()->isOptionChecked(MenuOption::EchoServerAudio); if (_muted || !_audioOutput || (!hasEcho && !hasLocalReverb)) { diff --git a/interface/src/Audio.h b/interface/src/Audio.h index e58afa306f..be51511dcc 100644 --- a/interface/src/Audio.h +++ b/interface/src/Audio.h @@ -273,7 +273,7 @@ private: void updateGverbOptions(); void addReverb(int16_t* samples, int numSamples, QAudioFormat& format); - void handleLocalEchoAndReverb(QByteArray inputByteArray); + void handleLocalEchoAndReverb(QByteArray& inputByteArray); // Add sounds that we want the user to not hear themselves, by adding on top of mic input signal void addProceduralSounds(int16_t* monoInput, int numSamples); From 011e5319710fe8cc50366700ad039ff77e2e110a Mon Sep 17 00:00:00 2001 From: Atlante45 Date: Fri, 7 Nov 2014 14:41:59 +0100 Subject: [PATCH 7/7] Magic numbers and casts --- interface/src/Audio.cpp | 13 +++++++------ 1 file changed, 7 insertions(+), 6 deletions(-) diff --git a/interface/src/Audio.cpp b/interface/src/Audio.cpp index 79ebcdd043..2d7b117f51 100644 --- a/interface/src/Audio.cpp +++ b/interface/src/Audio.cpp @@ -574,11 +574,11 @@ void Audio::addReverb(int16_t* samplesData, int numSamples, QAudioFormat& audioF for (unsigned int j = sample; j < sample + audioFormat.channelCount(); j++) { if (j == sample) { // left channel - int lResult = glm::clamp((int)(samplesData[j] * dryFraction + lValue * wetFraction), -32768, 32767); + int lResult = glm::clamp((int)(samplesData[j] * dryFraction + lValue * wetFraction), MIN_SAMPLE_VALUE, MAX_SAMPLE_VALUE); samplesData[j] = (int16_t)lResult; } else if (j == (sample + 1)) { // right channel - int rResult = glm::clamp((int)(samplesData[j] * dryFraction + rValue * wetFraction), -32768, 32767); + int rResult = glm::clamp((int)(samplesData[j] * dryFraction + rValue * wetFraction), MIN_SAMPLE_VALUE, MAX_SAMPLE_VALUE); samplesData[j] = (int16_t)rResult; } else { // ignore channels above 2 @@ -610,7 +610,8 @@ void Audio::handleLocalEchoAndReverb(QByteArray& inputByteArray) { (_outputFormat.channelCount() / _inputFormat.channelCount()); loopBackByteArray.resize(inputByteArray.size() * loopbackOutputToInputRatio); loopBackByteArray.fill(0); - linearResampling((int16_t*)inputByteArray.data(), (int16_t*)loopBackByteArray.data(), + linearResampling(reinterpret_cast(inputByteArray.data()), + reinterpret_cast(loopBackByteArray.data()), inputByteArray.size() / sizeof(int16_t), loopBackByteArray.size() / sizeof(int16_t), _inputFormat, _outputFormat); } @@ -621,16 +622,16 @@ void Audio::handleLocalEchoAndReverb(QByteArray& inputByteArray) { loopbackCopy = loopBackByteArray; } - int16_t* loopbackSamples = (int16_t*) loopBackByteArray.data(); + int16_t* loopbackSamples = reinterpret_cast(loopBackByteArray.data()); int numLoopbackSamples = loopBackByteArray.size() / sizeof(int16_t); updateGverbOptions(); addReverb(loopbackSamples, numLoopbackSamples, _outputFormat); if (!hasEcho) { - int16_t* loopbackCopySamples = (int16_t*) loopbackCopy.data(); + int16_t* loopbackCopySamples = reinterpret_cast(loopbackCopy.data()); for (int i = 0; i < numLoopbackSamples; ++i) { loopbackSamples[i] = glm::clamp((int)loopbackSamples[i] - loopbackCopySamples[i], - -32768, 32767); + MIN_SAMPLE_VALUE, MAX_SAMPLE_VALUE); } } }