From be184719b5d96b6333a567d943e52ef5d6d8dd8b Mon Sep 17 00:00:00 2001 From: Stephen Birarda Date: Tue, 16 Dec 2014 16:31:29 -0800 Subject: [PATCH] move the audio noise gate to its own class --- interface/src/Audio.cpp | 160 +++---------------------- interface/src/Audio.h | 25 ++-- interface/src/audio/AudioNoiseGate.cpp | 145 ++++++++++++++++++++++ interface/src/audio/AudioNoiseGate.h | 44 +++++++ 4 files changed, 217 insertions(+), 157 deletions(-) create mode 100644 interface/src/audio/AudioNoiseGate.cpp create mode 100644 interface/src/audio/AudioNoiseGate.h diff --git a/interface/src/Audio.cpp b/interface/src/Audio.cpp index 63f19ca8ff..7e84d30a27 100644 --- a/interface/src/Audio.cpp +++ b/interface/src/Audio.cpp @@ -46,8 +46,6 @@ #include "Audio.h" -static const int NUMBER_OF_NOISE_SAMPLE_FRAMES = 300; - static const int RECEIVED_AUDIO_STREAM_CAPACITY_FRAMES = 100; Audio::Audio() : @@ -68,21 +66,11 @@ Audio::Audio() : _isStereoInput(false), _averagedLatency(0.0), _lastInputLoudness(0), - _inputFrameCounter(0), - _quietestFrame(std::numeric_limits::max()), - _loudestFrame(0.0f), - _timeSinceLastClip(-1.0), - _dcOffset(0), - _noiseGateMeasuredFloor(0), - _noiseGateSampleCounter(0), - _noiseGateOpen(false), - _noiseGateEnabled(true), - _audioSourceInjectEnabled(false), - _noiseGateFramesToClose(0), - _totalInputAudioSamples(0), _muted(false), _shouldEchoLocally(false), _shouldEchoToServer(false), + _isNoiseGateEnabled(true), + _audioSourceInjectEnabled(false), _reverb(false), _reverbOptions(&_scriptReverbOptions), _gverbLocal(NULL), @@ -91,12 +79,11 @@ Audio::Audio() : _toneSourceEnabled(true), _outgoingAvatarAudioSequenceNumber(0), _audioOutputIODevice(_receivedAudioStream), - _stats(&_receivedAudioStream) + _stats(&_receivedAudioStream), + _inputGate() { // clear the array of locally injected samples memset(_localProceduralSamples, 0, AudioConstants::NETWORK_FRAME_BYTES_PER_CHANNEL); - // Create the noise sample array - _noiseSampleFrames = new float[NUMBER_OF_NOISE_SAMPLE_FRAMES]; connect(&_receivedAudioStream, &MixedProcessedAudioStream::processSamples, this, &Audio::processReceivedSamples, Qt::DirectConnection); @@ -586,6 +573,8 @@ void Audio::handleLocalEchoAndReverb(QByteArray& inputByteArray) { } } +const float CLIPPING_THRESHOLD = 0.90f; + void Audio::handleAudioInput() { static char audioDataPacket[MAX_PACKET_SIZE]; @@ -655,131 +644,24 @@ void Audio::handleAudioInput() { inputSamplesRequired, numNetworkSamples, _inputFormat, _desiredInputFormat); - // only impose the noise gate and perform tone injection if we sending mono audio - if (!_isStereoInput) { - - // - // Impose Noise Gate - // - // The Noise Gate is used to reject constant background noise by measuring the noise - // floor observed at the microphone and then opening the 'gate' to allow microphone - // signals to be transmitted when the microphone samples average level exceeds a multiple - // of the noise floor. - // - // NOISE_GATE_HEIGHT: How loud you have to speak relative to noise background to open the gate. - // Make this value lower for more sensitivity and less rejection of noise. - // NOISE_GATE_WIDTH: The number of samples in an audio frame for which the height must be exceeded - // to open the gate. - // NOISE_GATE_CLOSE_FRAME_DELAY: Once the noise is below the gate height for the frame, how many frames - // will we wait before closing the gate. - // NOISE_GATE_FRAMES_TO_AVERAGE: How many audio frames should we average together to compute noise floor. - // More means better rejection but also can reject continuous things like singing. - // NUMBER_OF_NOISE_SAMPLE_FRAMES: How often should we re-evaluate the noise floor? - - - float loudness = 0; - float thisSample = 0; - int samplesOverNoiseGate = 0; - - const float NOISE_GATE_HEIGHT = 7.0f; - const int NOISE_GATE_WIDTH = 5; - const int NOISE_GATE_CLOSE_FRAME_DELAY = 5; - const int NOISE_GATE_FRAMES_TO_AVERAGE = 5; - const float DC_OFFSET_AVERAGING = 0.99f; - const float CLIPPING_THRESHOLD = 0.90f; - - // - // Check clipping, adjust DC offset, and check if should open noise gate - // - float measuredDcOffset = 0.0f; - // Increment the time since the last clip - if (_timeSinceLastClip >= 0.0f) { - _timeSinceLastClip += (float) AudioConstants::NETWORK_FRAME_SAMPLES_PER_CHANNEL - / (float) AudioConstants::SAMPLE_RATE; - } - - for (int i = 0; i < AudioConstants::NETWORK_FRAME_SAMPLES_PER_CHANNEL; i++) { - measuredDcOffset += networkAudioSamples[i]; - networkAudioSamples[i] -= (int16_t) _dcOffset; - thisSample = fabsf(networkAudioSamples[i]); - if (thisSample >= ((float)AudioConstants::MAX_SAMPLE_VALUE * CLIPPING_THRESHOLD)) { - _timeSinceLastClip = 0.0f; - } - loudness += thisSample; - // Noise Reduction: Count peaks above the average loudness - if (_noiseGateEnabled && (thisSample > (_noiseGateMeasuredFloor * NOISE_GATE_HEIGHT))) { - samplesOverNoiseGate++; - } - } - - measuredDcOffset /= AudioConstants::NETWORK_FRAME_SAMPLES_PER_CHANNEL; - if (_dcOffset == 0.0f) { - // On first frame, copy over measured offset - _dcOffset = measuredDcOffset; - } else { - _dcOffset = DC_OFFSET_AVERAGING * _dcOffset + (1.0f - DC_OFFSET_AVERAGING) * measuredDcOffset; - } - - _lastInputLoudness = fabs(loudness / AudioConstants::NETWORK_FRAME_SAMPLES_PER_CHANNEL); - - if (_quietestFrame > _lastInputLoudness) { - _quietestFrame = _lastInputLoudness; - } - if (_loudestFrame < _lastInputLoudness) { - _loudestFrame = _lastInputLoudness; - } - - const int FRAMES_FOR_NOISE_DETECTION = 400; - if (_inputFrameCounter++ > FRAMES_FOR_NOISE_DETECTION) { - _quietestFrame = std::numeric_limits::max(); - _loudestFrame = 0.0f; - _inputFrameCounter = 0; - } - - // If Noise Gate is enabled, check and turn the gate on and off - if (!_audioSourceInjectEnabled && _noiseGateEnabled) { - float averageOfAllSampleFrames = 0.0f; - _noiseSampleFrames[_noiseGateSampleCounter++] = _lastInputLoudness; - if (_noiseGateSampleCounter == NUMBER_OF_NOISE_SAMPLE_FRAMES) { - float smallestSample = FLT_MAX; - for (int i = 0; i <= NUMBER_OF_NOISE_SAMPLE_FRAMES - NOISE_GATE_FRAMES_TO_AVERAGE; i += NOISE_GATE_FRAMES_TO_AVERAGE) { - float thisAverage = 0.0f; - for (int j = i; j < i + NOISE_GATE_FRAMES_TO_AVERAGE; j++) { - thisAverage += _noiseSampleFrames[j]; - averageOfAllSampleFrames += _noiseSampleFrames[j]; - } - thisAverage /= NOISE_GATE_FRAMES_TO_AVERAGE; - - if (thisAverage < smallestSample) { - smallestSample = thisAverage; - } - } - averageOfAllSampleFrames /= NUMBER_OF_NOISE_SAMPLE_FRAMES; - _noiseGateMeasuredFloor = smallestSample; - _noiseGateSampleCounter = 0; - - } - if (samplesOverNoiseGate > NOISE_GATE_WIDTH) { - _noiseGateOpen = true; - _noiseGateFramesToClose = NOISE_GATE_CLOSE_FRAME_DELAY; - } else { - if (--_noiseGateFramesToClose == 0) { - _noiseGateOpen = false; - } - } - if (!_noiseGateOpen) { - memset(networkAudioSamples, 0, AudioConstants::NETWORK_FRAME_BYTES_PER_CHANNEL); - _lastInputLoudness = 0; - } - } + // only impose the noise gate and perform tone injection if we are sending mono audio + if (!_isStereoInput && _isNoiseGateEnabled) { + _inputGate.gateSamples(networkAudioSamples, numNetworkSamples); + _lastInputLoudness = _inputGate.getLastLoudness(); + _timeSinceLastClip = _inputGate.getTimeSinceLastClip(); } else { float loudness = 0.0f; - for (int i = 0; i < AudioConstants::NETWORK_FRAME_SAMPLES_STEREO; i++) { - loudness += fabsf(networkAudioSamples[i]); + for (int i = 0; i < numNetworkSamples; i++) { + float thisSample = fabsf(networkAudioSamples[i]); + loudness += thisSample; + + if (thisSample > (AudioConstants::MAX_SAMPLE_VALUE * AudioNoiseGate::CLIPPING_THRESHOLD)) { + _timeSinceLastClip = 0.0f; + } } - _lastInputLoudness = fabs(loudness / AudioConstants::NETWORK_FRAME_SAMPLES_PER_CHANNEL); + _lastInputLoudness = fabs(loudness / numNetworkSamples); } } else { // our input loudness is 0, since we're muted @@ -943,10 +825,6 @@ void Audio::toggleMute() { muteToggled(); } -void Audio::toggleAudioNoiseReduction() { - _noiseGateEnabled = !_noiseGateEnabled; -} - void Audio::setIsStereoInput(bool isStereoInput) { if (isStereoInput != _isStereoInput) { _isStereoInput = isStereoInput; diff --git a/interface/src/Audio.h b/interface/src/Audio.h index 231fac5047..ca3917ec5c 100644 --- a/interface/src/Audio.h +++ b/interface/src/Audio.h @@ -32,6 +32,7 @@ #include "InterfaceConfig.h" #include "audio/AudioIOStats.h" +#include "audio/AudioNoiseGate.h" #include "AudioStreamStats.h" #include "Recorder.h" #include "RingBufferHistory.h" @@ -82,12 +83,10 @@ public: const MixedProcessedAudioStream& getReceivedAudioStream() const { return _receivedAudioStream; } - float getLastInputLoudness() const { return glm::max(_lastInputLoudness - _noiseGateMeasuredFloor, 0.0f); } + float getLastInputLoudness() const { return glm::max(_lastInputLoudness - _inputGate.getMeasuredFloor(), 0.0f); } float getTimeSinceLastClip() const { return _timeSinceLastClip; } float getAudioAverageInputLoudness() const { return _lastInputLoudness; } - void setNoiseGateEnabled(bool noiseGateEnabled) { _noiseGateEnabled = noiseGateEnabled; } - void setReceivedAudioStreamSettings(const InboundAudioStream::Settings& settings) { _receivedAudioStream.setSettings(settings); } int getDesiredJitterBufferFrames() const { return _receivedAudioStream.getDesiredJitterBufferFrames(); } @@ -116,11 +115,13 @@ public slots: void reset(); void audioMixerKilled(); void toggleMute(); - void toggleAudioNoiseReduction(); + void toggleAudioSourceInject(); void selectAudioSourcePinkNoise(); void selectAudioSourceSine440(); + void toggleAudioNoiseReduction() { _isNoiseGateEnabled = !_isNoiseGateEnabled; } + void toggleLocalEcho() { _shouldEchoLocally = !_shouldEchoLocally; } void toggleServerEcho() { _shouldEchoToServer = !_shouldEchoToServer; } @@ -179,24 +180,14 @@ private: QElapsedTimer _timeSinceLastReceived; float _averagedLatency; float _lastInputLoudness; - int _inputFrameCounter; - float _quietestFrame; - float _loudestFrame; float _timeSinceLastClip; - float _dcOffset; - float _noiseGateMeasuredFloor; - float* _noiseSampleFrames; - int _noiseGateSampleCounter; - bool _noiseGateOpen; - bool _noiseGateEnabled; - bool _audioSourceInjectEnabled; - - int _noiseGateFramesToClose; int _totalInputAudioSamples; bool _muted; bool _shouldEchoLocally; bool _shouldEchoToServer; + bool _isNoiseGateEnabled; + bool _audioSourceInjectEnabled; bool _reverb; AudioEffectOptions _scriptReverbOptions; @@ -244,6 +235,8 @@ private: WeakRecorderPointer _recorder; AudioIOStats _stats; + + AudioNoiseGate _inputGate; }; diff --git a/interface/src/audio/AudioNoiseGate.cpp b/interface/src/audio/AudioNoiseGate.cpp new file mode 100644 index 0000000000..e8f8fe88cc --- /dev/null +++ b/interface/src/audio/AudioNoiseGate.cpp @@ -0,0 +1,145 @@ +// +// AudioNoiseGate.cpp +// interface/src/audio +// +// Created by Stephen Birarda on 2014-12-16. +// Copyright 2014 High Fidelity, Inc. +// +// Distributed under the Apache License, Version 2.0. +// See the accompanying file LICENSE or http://www.apache.org/licenses/LICENSE-2.0.html +// + +#include +#include + +#include + +#include "AudioNoiseGate.h" + +const int NUMBER_OF_NOISE_SAMPLE_FRAMES = 300; +const float AudioNoiseGate::CLIPPING_THRESHOLD = 0.90f; + +AudioNoiseGate::AudioNoiseGate() { + // Create the noise sample array + _sampleFrames = new float[NUMBER_OF_NOISE_SAMPLE_FRAMES]; +} + +AudioNoiseGate::~AudioNoiseGate() { + delete[] _sampleFrames; +} + +void AudioNoiseGate::gateSamples(int16_t* samples, int numSamples) { + // + // Impose Noise Gate + // + // The Noise Gate is used to reject constant background noise by measuring the noise + // floor observed at the microphone and then opening the 'gate' to allow microphone + // signals to be transmitted when the microphone samples average level exceeds a multiple + // of the noise floor. + // + // NOISE_GATE_HEIGHT: How loud you have to speak relative to noise background to open the gate. + // Make this value lower for more sensitivity and less rejection of noise. + // NOISE_GATE_WIDTH: The number of samples in an audio frame for which the height must be exceeded + // to open the gate. + // NOISE_GATE_CLOSE_FRAME_DELAY: Once the noise is below the gate height for the frame, how many frames + // will we wait before closing the gate. + // NOISE_GATE_FRAMES_TO_AVERAGE: How many audio frames should we average together to compute noise floor. + // More means better rejection but also can reject continuous things like singing. + // NUMBER_OF_NOISE_SAMPLE_FRAMES: How often should we re-evaluate the noise floor? + + + float loudness = 0; + float thisSample = 0; + int samplesOverNoiseGate = 0; + + const float NOISE_GATE_HEIGHT = 7.0f; + const int NOISE_GATE_WIDTH = 5; + const int NOISE_GATE_CLOSE_FRAME_DELAY = 5; + const int NOISE_GATE_FRAMES_TO_AVERAGE = 5; + const float DC_OFFSET_AVERAGING = 0.99f; + + // + // Check clipping, adjust DC offset, and check if should open noise gate + // + float measuredDcOffset = 0.0f; + // Increment the time since the last clip + if (_timeSinceLastClip >= 0.0f) { + _timeSinceLastClip += (float) AudioConstants::NETWORK_FRAME_SAMPLES_PER_CHANNEL + / (float) AudioConstants::SAMPLE_RATE; + } + + for (int i = 0; i < AudioConstants::NETWORK_FRAME_SAMPLES_PER_CHANNEL; i++) { + measuredDcOffset += samples[i]; + samples[i] -= (int16_t) _dcOffset; + thisSample = fabsf(samples[i]); + + if (thisSample >= ((float)AudioConstants::MAX_SAMPLE_VALUE * CLIPPING_THRESHOLD)) { + _timeSinceLastClip = 0.0f; + } + + loudness += thisSample; + // Noise Reduction: Count peaks above the average loudness + if (thisSample > (_measuredFloor * NOISE_GATE_HEIGHT)) { + samplesOverNoiseGate++; + } + } + + measuredDcOffset /= AudioConstants::NETWORK_FRAME_SAMPLES_PER_CHANNEL; + if (_dcOffset == 0.0f) { + // On first frame, copy over measured offset + _dcOffset = measuredDcOffset; + } else { + _dcOffset = DC_OFFSET_AVERAGING * _dcOffset + (1.0f - DC_OFFSET_AVERAGING) * measuredDcOffset; + } + + _lastLoudness = fabs(loudness / AudioConstants::NETWORK_FRAME_SAMPLES_PER_CHANNEL); + + if (_quietestFrame > _lastLoudness) { + _quietestFrame = _lastLoudness; + } + if (_loudestFrame < _lastLoudness) { + _loudestFrame = _lastLoudness; + } + + const int FRAMES_FOR_NOISE_DETECTION = 400; + if (_inputFrameCounter++ > FRAMES_FOR_NOISE_DETECTION) { + _quietestFrame = std::numeric_limits::max(); + _loudestFrame = 0.0f; + _inputFrameCounter = 0; + } + + // If Noise Gate is enabled, check and turn the gate on and off + float averageOfAllSampleFrames = 0.0f; + _sampleFrames[_sampleCounter++] = _lastLoudness; + if (_sampleCounter == NUMBER_OF_NOISE_SAMPLE_FRAMES) { + float smallestSample = FLT_MAX; + for (int i = 0; i <= NUMBER_OF_NOISE_SAMPLE_FRAMES - NOISE_GATE_FRAMES_TO_AVERAGE; i += NOISE_GATE_FRAMES_TO_AVERAGE) { + float thisAverage = 0.0f; + for (int j = i; j < i + NOISE_GATE_FRAMES_TO_AVERAGE; j++) { + thisAverage += _sampleFrames[j]; + averageOfAllSampleFrames += _sampleFrames[j]; + } + thisAverage /= NOISE_GATE_FRAMES_TO_AVERAGE; + + if (thisAverage < smallestSample) { + smallestSample = thisAverage; + } + } + averageOfAllSampleFrames /= NUMBER_OF_NOISE_SAMPLE_FRAMES; + _measuredFloor = smallestSample; + _sampleCounter = 0; + + } + if (samplesOverNoiseGate > NOISE_GATE_WIDTH) { + _isOpen = true; + _framesToClose = NOISE_GATE_CLOSE_FRAME_DELAY; + } else { + if (--_framesToClose == 0) { + _isOpen = false; + } + } + if (!_isOpen) { + memset(samples, 0, AudioConstants::NETWORK_FRAME_BYTES_PER_CHANNEL); + _lastLoudness = 0; + } +} diff --git a/interface/src/audio/AudioNoiseGate.h b/interface/src/audio/AudioNoiseGate.h new file mode 100644 index 0000000000..c6d1894898 --- /dev/null +++ b/interface/src/audio/AudioNoiseGate.h @@ -0,0 +1,44 @@ +// +// AudioNoiseGate.h +// interface/src/audio +// +// Created by Stephen Birarda on 2014-12-16. +// Copyright 2014 High Fidelity, Inc. +// +// Distributed under the Apache License, Version 2.0. +// See the accompanying file LICENSE or http://www.apache.org/licenses/LICENSE-2.0.html +// + +#ifndef hifi_AudioNoiseGate_h +#define hifi_AudioNoiseGate_h + +#include + +class AudioNoiseGate { +public: + AudioNoiseGate(); + ~AudioNoiseGate(); + + void gateSamples(int16_t* samples, int numSamples); + + float getTimeSinceLastClip() const { return _timeSinceLastClip; } + float getMeasuredFloor() const { return _measuredFloor; } + float getLastLoudness() const { return _lastLoudness; } + + static const float CLIPPING_THRESHOLD; + +private: + int _inputFrameCounter; + float _lastLoudness; + float _quietestFrame; + float _loudestFrame; + float _timeSinceLastClip; + float _dcOffset; + float _measuredFloor; + float* _sampleFrames; + int _sampleCounter; + bool _isOpen; + int _framesToClose; +}; + +#endif // hifi_AudioNoiseGate_h \ No newline at end of file