Merge pull request #3750 from Atlante45/reverb_for_own_voice

Reverb for own voice

assignment-client/src/audio/AudioMixer.cpp

@@ -465,6 +465,63 @@ int AudioMixer::prepareMixForListeningNode(Node* node) {
     return streamsMixed;
 }
 
+void AudioMixer::sendAudioEnvironmentPacket(SharedNodePointer node) {
+    static char clientEnvBuffer[MAX_PACKET_SIZE];
+    
+    // Send stream properties
+    bool hasReverb = false;
+    float reverbTime, wetLevel;
+    // find reverb properties
+    for (int i = 0; i < _zoneReverbSettings.size(); ++i) {
+        AudioMixerClientData* data = static_cast<AudioMixerClientData*>(node->getLinkedData());
+        glm::vec3 streamPosition = data->getAvatarAudioStream()->getPosition();
+        if (_audioZones[_zoneReverbSettings[i].zone].contains(streamPosition)) {
+            hasReverb = true;
+            reverbTime = _zoneReverbSettings[i].reverbTime;
+            wetLevel = _zoneReverbSettings[i].wetLevel;
+            break;
+        }
+    }
+    AudioMixerClientData* nodeData = static_cast<AudioMixerClientData*>(node->getLinkedData());
+    AvatarAudioStream* stream = nodeData->getAvatarAudioStream();
+    bool dataChanged = (stream->hasReverb() != hasReverb) ||
+    (stream->hasReverb() && (stream->getRevebTime() != reverbTime ||
+                             stream->getWetLevel() != wetLevel));
+    if (dataChanged) {
+        // Update stream
+        if (hasReverb) {
+            stream->setReverb(reverbTime, wetLevel);
+        } else {
+            stream->clearReverb();
+        }
+    }
+    
+    // Send at change or every so often
+    float CHANCE_OF_SEND = 0.01f;
+    bool sendData = dataChanged || (randFloat() < CHANCE_OF_SEND);
+    
+    if (sendData) {
+        int numBytesEnvPacketHeader = populatePacketHeader(clientEnvBuffer, PacketTypeAudioEnvironment);
+        char* envDataAt = clientEnvBuffer + numBytesEnvPacketHeader;
+        
+        unsigned char bitset = 0;
+        if (hasReverb) {
+            setAtBit(bitset, HAS_REVERB_BIT);
+        }
+        
+        memcpy(envDataAt, &bitset, sizeof(unsigned char));
+        envDataAt += sizeof(unsigned char);
+        
+        if (hasReverb) {
+            memcpy(envDataAt, &reverbTime, sizeof(float));
+            envDataAt += sizeof(float);
+            memcpy(envDataAt, &wetLevel, sizeof(float));
+            envDataAt += sizeof(float);
+        }
+        NodeList::getInstance()->writeDatagram(clientEnvBuffer, envDataAt - clientEnvBuffer, node);
+    }
+}
+
 void AudioMixer::readPendingDatagram(const QByteArray& receivedPacket, const HifiSockAddr& senderSockAddr) {
     NodeList* nodeList = NodeList::getInstance();
     
@@ -642,7 +699,6 @@ void AudioMixer::run() {
     timer.start();
 
     char clientMixBuffer[MAX_PACKET_SIZE];
-    char clientEnvBuffer[MAX_PACKET_SIZE];
     
     int usecToSleep = BUFFER_SEND_INTERVAL_USECS;
     
@@ -759,58 +815,6 @@ void AudioMixer::run() {
                         // pack mixed audio samples
                         memcpy(mixDataAt, _mixSamples, NETWORK_BUFFER_LENGTH_BYTES_STEREO);
                         mixDataAt += NETWORK_BUFFER_LENGTH_BYTES_STEREO;
-
-                        // Send stream properties
-                        bool hasReverb = false;
-                        float reverbTime, wetLevel;
-                        // find reverb properties
-                        for (int i = 0; i < _zoneReverbSettings.size(); ++i) {
-                            AudioMixerClientData* data = static_cast<AudioMixerClientData*>(node->getLinkedData());
-                            glm::vec3 streamPosition = data->getAvatarAudioStream()->getPosition();
-                            if (_audioZones[_zoneReverbSettings[i].zone].contains(streamPosition)) {
-                                hasReverb = true;
-                                reverbTime = _zoneReverbSettings[i].reverbTime;
-                                wetLevel = _zoneReverbSettings[i].wetLevel;
-                                break;
-                            }
-                        }
-                        AvatarAudioStream* stream = nodeData->getAvatarAudioStream();
-                        bool dataChanged = (stream->hasReverb() != hasReverb) ||
-                                            (stream->hasReverb() && (stream->getRevebTime() != reverbTime ||
-                                                                     stream->getWetLevel() != wetLevel));
-                        if (dataChanged) {                            
-                            // Update stream
-                            if (hasReverb) {
-                                stream->setReverb(reverbTime, wetLevel);
-                            } else {
-                                stream->clearReverb();
-                            }
-                        }
-                        
-                        // Send at change or every so often
-                        float CHANCE_OF_SEND = 0.01f;
-                        bool sendData = dataChanged || (randFloat() < CHANCE_OF_SEND);
-                        
-                        if (sendData) {
-                            int numBytesEnvPacketHeader = populatePacketHeader(clientEnvBuffer, PacketTypeAudioEnvironment);
-                            char* envDataAt = clientEnvBuffer + numBytesEnvPacketHeader;
-                            
-                            unsigned char bitset = 0;
-                            if (hasReverb) {
-                                setAtBit(bitset, HAS_REVERB_BIT);
-                            }
-                            
-                            memcpy(envDataAt, &bitset, sizeof(unsigned char));
-                            envDataAt += sizeof(unsigned char);
-                            
-                            if (hasReverb) {
-                                memcpy(envDataAt, &reverbTime, sizeof(float));
-                                envDataAt += sizeof(float);
-                                memcpy(envDataAt, &wetLevel, sizeof(float));
-                                envDataAt += sizeof(float);
-                            }
-                            nodeList->writeDatagram(clientEnvBuffer, envDataAt - clientEnvBuffer, node);
-                        }
                     } else {
                         // pack header
                         int numBytesPacketHeader = populatePacketHeader(clientMixBuffer, PacketTypeSilentAudioFrame);
@@ -826,6 +830,9 @@ void AudioMixer::run() {
                         memcpy(mixDataAt, &numSilentSamples, sizeof(quint16));
                         mixDataAt += sizeof(quint16);
                     }
+                    
+                    // Send audio environment
+                    sendAudioEnvironmentPacket(node);
 
                     // send mixed audio packet
                     nodeList->writeDatagram(clientMixBuffer, mixDataAt - clientMixBuffer, node);

assignment-client/src/audio/AudioMixer.h

@@ -49,6 +49,9 @@ private:
     
     /// prepares and sends a mix to one Node
     int prepareMixForListeningNode(Node* node);
+    
+    /// Send Audio Environment packet for a single node
+    void sendAudioEnvironmentPacket(SharedNodePointer node);
 
     // used on a per stream basis to run the filter on before mixing, large enough to handle the historical
     // data from a phase delay as well as an entire network buffer

interface/src/Audio.cpp

@@ -516,6 +516,33 @@ void Audio::initGverb() {
     gverb_set_taillevel(_gverb, DB_CO(_reverbOptions->getTailLevel()));
 }
 
+void Audio::updateGverbOptions() {
+    bool reverbChanged = false;
+    if (_receivedAudioStream.hasReverb()) {
+        
+        if (_zoneReverbOptions.getReverbTime() != _receivedAudioStream.getRevebTime()) {
+            _zoneReverbOptions.setReverbTime(_receivedAudioStream.getRevebTime());
+            reverbChanged = true;
+        }
+        if (_zoneReverbOptions.getWetLevel() != _receivedAudioStream.getWetLevel()) {
+            _zoneReverbOptions.setWetLevel(_receivedAudioStream.getWetLevel());
+            reverbChanged = true;
+        }
+        
+        if (_reverbOptions != &_zoneReverbOptions) {
+            _reverbOptions = &_zoneReverbOptions;
+            reverbChanged = true;
+        }
+    } else if (_reverbOptions != &_scriptReverbOptions) {
+        _reverbOptions = &_scriptReverbOptions;
+        reverbChanged = true;
+    }
+    
+    if (reverbChanged) {
+        initGverb();
+    }
+}
+
 void Audio::setReverbOptions(const AudioEffectOptions* options) {
     // Save the new options
     _scriptReverbOptions.setMaxRoomSize(options->getMaxRoomSize());
@@ -550,11 +577,11 @@ void Audio::addReverb(int16_t* samplesData, int numSamples, QAudioFormat& audioF
         for (int j = sample; j < sample + audioFormat.channelCount(); j++) {
             if (j == sample) {
                 // left channel
-                int lResult = glm::clamp((int)(samplesData[j] * dryFraction + lValue * wetFraction), -32768, 32767);
+                int lResult = glm::clamp((int)(samplesData[j] * dryFraction + lValue * wetFraction), MIN_SAMPLE_VALUE, MAX_SAMPLE_VALUE);
                 samplesData[j] = (int16_t)lResult;
             } else if (j == (sample + 1)) {
                 // right channel
-                int rResult = glm::clamp((int)(samplesData[j] * dryFraction + rValue * wetFraction), -32768, 32767);
+                int rResult = glm::clamp((int)(samplesData[j] * dryFraction + rValue * wetFraction), MIN_SAMPLE_VALUE, MAX_SAMPLE_VALUE);
                 samplesData[j] = (int16_t)rResult;
             } else {
                 // ignore channels above 2
@@ -563,6 +590,60 @@ void Audio::addReverb(int16_t* samplesData, int numSamples, QAudioFormat& audioF
     }
 }
 
+void Audio::handleLocalEchoAndReverb(QByteArray& inputByteArray) {
+    bool hasEcho = Menu::getInstance()->isOptionChecked(MenuOption::EchoLocalAudio);
+    // If there is server echo, reverb will be applied to the recieved audio stream so no need to have it here.
+    bool hasLocalReverb = (_reverb || _receivedAudioStream.hasReverb()) &&
+                          !Menu::getInstance()->isOptionChecked(MenuOption::EchoServerAudio);
+    if (_muted || !_audioOutput || (!hasEcho && !hasLocalReverb)) {
+        return;
+    }
+    
+    // if this person wants local loopback add that to the locally injected audio
+    // if there is reverb apply it to local audio and substract the origin samples
+    
+    if (!_loopbackOutputDevice && _loopbackAudioOutput) {
+        // we didn't have the loopback output device going so set that up now
+        _loopbackOutputDevice = _loopbackAudioOutput->start();
+    }
+    
+    QByteArray loopBackByteArray(inputByteArray);
+    if (_inputFormat != _outputFormat) {
+        float loopbackOutputToInputRatio = (_outputFormat.sampleRate() / (float) _inputFormat.sampleRate()) *
+                                           (_outputFormat.channelCount() / _inputFormat.channelCount());
+        loopBackByteArray.resize(inputByteArray.size() * loopbackOutputToInputRatio);
+        loopBackByteArray.fill(0);
+        linearResampling(reinterpret_cast<int16_t*>(inputByteArray.data()),
+                         reinterpret_cast<int16_t*>(loopBackByteArray.data()),
+                         inputByteArray.size() / sizeof(int16_t), loopBackByteArray.size() / sizeof(int16_t),
+                         _inputFormat, _outputFormat);
+    }
+    
+    if (hasLocalReverb) {
+        QByteArray loopbackCopy;
+        if (!hasEcho) {
+            loopbackCopy = loopBackByteArray;
+        }
+        
+        int16_t* loopbackSamples = reinterpret_cast<int16_t*>(loopBackByteArray.data());
+        int numLoopbackSamples = loopBackByteArray.size() / sizeof(int16_t);
+        updateGverbOptions();
+        addReverb(loopbackSamples, numLoopbackSamples, _outputFormat);
+        
+        if (!hasEcho) {
+            int16_t* loopbackCopySamples = reinterpret_cast<int16_t*>(loopbackCopy.data());
+            for (int i = 0; i < numLoopbackSamples; ++i) {
+                loopbackSamples[i] = glm::clamp((int)loopbackSamples[i] - loopbackCopySamples[i],
+                                                MIN_SAMPLE_VALUE, MAX_SAMPLE_VALUE);
+            }
+        }
+    }
+    
+    if (_loopbackOutputDevice) {
+        _loopbackOutputDevice->write(loopBackByteArray);
+    }
+}
+
 void Audio::handleAudioInput() {
     static char audioDataPacket[MAX_PACKET_SIZE];
 
@@ -607,34 +688,8 @@ void Audio::handleAudioInput() {
 
         _inputFrameBuffer.copyFrames(1, inputFrameCount, inputFrameData, true /*copy out*/);
     }
-        
+    
-    if (Menu::getInstance()->isOptionChecked(MenuOption::EchoLocalAudio) && !_muted && _audioOutput) {
+    handleLocalEchoAndReverb(inputByteArray);
-        // if this person wants local loopback add that to the locally injected audio
-
-        if (!_loopbackOutputDevice && _loopbackAudioOutput) {
-            // we didn't have the loopback output device going so set that up now
-            _loopbackOutputDevice = _loopbackAudioOutput->start();
-        }
-        
-        if (_inputFormat == _outputFormat) {
-            if (_loopbackOutputDevice) {
-                _loopbackOutputDevice->write(inputByteArray);
-            }
-        } else {
-            float loopbackOutputToInputRatio = (_outputFormat.sampleRate() / (float) _inputFormat.sampleRate())
-                * (_outputFormat.channelCount() / _inputFormat.channelCount());
-
-            QByteArray loopBackByteArray(inputByteArray.size() * loopbackOutputToInputRatio, 0);
-
-            linearResampling((int16_t*) inputByteArray.data(), (int16_t*) loopBackByteArray.data(),
-                             inputByteArray.size() / sizeof(int16_t),
-                             loopBackByteArray.size() / sizeof(int16_t), _inputFormat, _outputFormat);
-
-            if (_loopbackOutputDevice) {
-                _loopbackOutputDevice->write(loopBackByteArray);
-            }
-        }
-    }
 
     _inputRingBuffer.writeData(inputByteArray.data(), inputByteArray.size());
     
@@ -971,30 +1026,7 @@ void Audio::processReceivedSamples(const QByteArray& inputBuffer, QByteArray& ou
         _desiredOutputFormat, _outputFormat);
     
     if(_reverb || _receivedAudioStream.hasReverb()) {
-        bool reverbChanged = false;
+        updateGverbOptions();
-        if (_receivedAudioStream.hasReverb()) {
-            
-            if (_zoneReverbOptions.getReverbTime() != _receivedAudioStream.getRevebTime()) {
-                _zoneReverbOptions.setReverbTime(_receivedAudioStream.getRevebTime());
-                reverbChanged = true;
-            }
-            if (_zoneReverbOptions.getWetLevel() != _receivedAudioStream.getWetLevel()) {
-                _zoneReverbOptions.setWetLevel(_receivedAudioStream.getWetLevel());
-                reverbChanged = true;
-            }
-            
-            if (_reverbOptions != &_zoneReverbOptions) {
-                _reverbOptions = &_zoneReverbOptions;
-                reverbChanged = true;
-            }
-        } else if (_reverbOptions != &_scriptReverbOptions) {
-            _reverbOptions = &_scriptReverbOptions;
-            reverbChanged = true;
-        }
-        
-        if (reverbChanged) {
-            initGverb();
-        }
         addReverb((int16_t*)outputBuffer.data(), numDeviceOutputSamples, _outputFormat);
     }
 }

interface/src/Audio.h

@@ -272,8 +272,11 @@ private:
 
     // Adds Reverb
     void initGverb();
+    void updateGverbOptions();
     void addReverb(int16_t* samples, int numSamples, QAudioFormat& format);
 
+    void handleLocalEchoAndReverb(QByteArray& inputByteArray);
+    
     // Add sounds that we want the user to not hear themselves, by adding on top of mic input signal
     void addProceduralSounds(int16_t* monoInput, int numSamples);
 

libraries/networking/src/LimitedNodeList.cpp

@@ -209,8 +209,15 @@ bool LimitedNodeList::packetVersionAndHashMatch(const QByteArray& packet) {
             if (hashFromPacketHeader(packet) == hashForPacketAndConnectionUUID(packet, sendingNode->getConnectionSecret())) {
                 return true;
             } else {
-                qDebug() << "Packet hash mismatch on" << checkType << "- Sender"
+                static QMultiMap<QUuid, PacketType> hashDebugSuppressMap;
+                
+                QUuid senderUUID = uuidFromPacketHeader(packet);
+                if (!hashDebugSuppressMap.contains(senderUUID, checkType)) {
+                    qDebug() << "Packet hash mismatch on" << checkType << "- Sender"
                     << uuidFromPacketHeader(packet);
+                    
+                    hashDebugSuppressMap.insert(senderUUID, checkType);
+                }
             }
         } else {
             static QString repeatedMessage