From 2d0e11e2d1a54e9b330889dc66d163534458fd2b Mon Sep 17 00:00:00 2001 From: Zach Pomerantz Date: Mon, 13 Mar 2017 19:22:33 -0400 Subject: [PATCH 1/2] add AudioNoiseGate to recording --- libraries/audio-client/src/AudioClient.cpp | 197 +++++++++------------ libraries/audio-client/src/AudioClient.h | 15 +- 2 files changed, 93 insertions(+), 119 deletions(-) diff --git a/libraries/audio-client/src/AudioClient.cpp b/libraries/audio-client/src/AudioClient.cpp index c32b5600d9..4f8f4c9766 100644 --- a/libraries/audio-client/src/AudioClient.cpp +++ b/libraries/audio-client/src/AudioClient.cpp @@ -184,7 +184,6 @@ AudioClient::AudioClient() : _outgoingAvatarAudioSequenceNumber(0), _audioOutputIODevice(_localInjectorsStream, _receivedAudioStream, this), _stats(&_receivedAudioStream), - _inputGate(), _positionGetter(DEFAULT_POSITION_GETTER), _orientationGetter(DEFAULT_ORIENTATION_GETTER) { // avoid putting a lock in the device callback @@ -971,14 +970,87 @@ void AudioClient::handleLocalEchoAndReverb(QByteArray& inputByteArray) { } } -void AudioClient::handleAudioInput() { +void AudioClient::handleAudioInput(QByteArray& audioBuffer) { + if (_muted) { + _lastInputLoudness = 0.0f; + _timeSinceLastClip = 0.0f; + } else { + int16_t* samples = reinterpret_cast(audioBuffer.data()); + int numSamples = audioBuffer.size() / sizeof(AudioConstants::SAMPLE_SIZE); + bool didClip = false; + + bool shouldRemoveDCOffset = !_isPlayingBackRecording && !_isStereoInput; + if (shouldRemoveDCOffset) { + _noiseGate.removeDCOffset(samples, numSamples); + } + + bool shouldNoiseGate = (_isPlayingBackRecording || !_isStereoInput) && _isNoiseGateEnabled; + if (shouldNoiseGate) { + _noiseGate.gateSamples(samples, numSamples); + _lastInputLoudness = _noiseGate.getLastLoudness(); + didClip = _noiseGate.clippedInLastBlock(); + } else { + float loudness = 0.0f; + for (int i = 0; i < numSamples; ++i) { + int16_t sample = std::abs(samples[i]); + loudness += (float)sample; + didClip = didClip || + (sample > (AudioConstants::MAX_SAMPLE_VALUE * AudioNoiseGate::CLIPPING_THRESHOLD)); + } + _lastInputLoudness = fabs(loudness / numSamples); + } + + if (didClip) { + _timeSinceLastClip = 0.0f; + } else if (_timeSinceLastClip >= 0.0f) { + _timeSinceLastClip += (float)numSamples / (float)AudioConstants::SAMPLE_RATE; + } + + emit inputReceived({ audioBuffer.data(), numSamples }); + + if (_noiseGate.openedInLastBlock()) { + emit noiseGateOpened(); + } else if (_noiseGate.closedInLastBlock()) { + emit noiseGateClosed(); + } + } + + // the codec needs a flush frame before sending silent packets, so + // do not send one if the gate closed in this block (eventually this can be crossfaded). + auto packetType = _shouldEchoToServer ? + PacketType::MicrophoneAudioWithEcho : PacketType::MicrophoneAudioNoEcho; + if (_lastInputLoudness == 0.0f && !_noiseGate.closedInLastBlock()) { + packetType = PacketType::SilentAudioFrame; + _silentOutbound.increment(); + } else { + _audioOutbound.increment(); + } + + Transform audioTransform; + audioTransform.setTranslation(_positionGetter()); + audioTransform.setRotation(_orientationGetter()); + + QByteArray encodedBuffer; + if (_encoder) { + _encoder->encode(audioBuffer, encodedBuffer); + } else { + encodedBuffer = audioBuffer; + } + + emitAudioPacket(encodedBuffer.data(), encodedBuffer.size(), _outgoingAvatarAudioSequenceNumber, + audioTransform, avatarBoundingBoxCorner, avatarBoundingBoxScale, + packetType, _selectedCodecName); + _stats.sentPacket(); +} + +void AudioClient::handleMicAudioInput() { if (!_inputDevice || _isPlayingBackRecording) { return; } // input samples required to produce exactly NETWORK_FRAME_SAMPLES of output - const int inputSamplesRequired = (_inputToNetworkResampler ? - _inputToNetworkResampler->getMinInput(AudioConstants::NETWORK_FRAME_SAMPLES_PER_CHANNEL) : + const int inputSamplesRequired = (_inputToNetworkResampler ? + _inputToNetworkResampler->getMinInput(AudioConstants::NETWORK_FRAME_SAMPLES_PER_CHANNEL) : AudioConstants::NETWORK_FRAME_SAMPLES_PER_CHANNEL) * _inputFormat.channelCount(); const auto inputAudioSamples = std::unique_ptr(new int16_t[inputSamplesRequired]); @@ -1001,126 +1073,27 @@ void AudioClient::handleAudioInput() { static int16_t networkAudioSamples[AudioConstants::NETWORK_FRAME_SAMPLES_STEREO]; while (_inputRingBuffer.samplesAvailable() >= inputSamplesRequired) { - - if (!_muted) { - - - // Increment the time since the last clip - if (_timeSinceLastClip >= 0.0f) { - _timeSinceLastClip += (float)numNetworkSamples / (float)AudioConstants::SAMPLE_RATE; - } - + if (_muted) { + _inputRingBuffer.shiftReadPosition(inputSamplesRequired); + } else { _inputRingBuffer.readSamples(inputAudioSamples.get(), inputSamplesRequired); possibleResampling(_inputToNetworkResampler, inputAudioSamples.get(), networkAudioSamples, inputSamplesRequired, numNetworkSamples, _inputFormat.channelCount(), _desiredInputFormat.channelCount()); - - // Remove DC offset - if (!_isStereoInput) { - _inputGate.removeDCOffset(networkAudioSamples, numNetworkSamples); - } - - // only impose the noise gate and perform tone injection if we are sending mono audio - if (!_isStereoInput && _isNoiseGateEnabled) { - _inputGate.gateSamples(networkAudioSamples, numNetworkSamples); - - // if we performed the noise gate we can get values from it instead of enumerating the samples again - _lastInputLoudness = _inputGate.getLastLoudness(); - - if (_inputGate.clippedInLastBlock()) { - _timeSinceLastClip = 0.0f; - } - - } else { - float loudness = 0.0f; - - for (int i = 0; i < numNetworkSamples; i++) { - int thisSample = std::abs(networkAudioSamples[i]); - loudness += (float)thisSample; - - if (thisSample > (AudioConstants::MAX_SAMPLE_VALUE * AudioNoiseGate::CLIPPING_THRESHOLD)) { - _timeSinceLastClip = 0.0f; - } - } - - _lastInputLoudness = fabs(loudness / numNetworkSamples); - } - - emit inputReceived({ reinterpret_cast(networkAudioSamples), numNetworkBytes }); - - if (_inputGate.openedInLastBlock()) { - emit noiseGateOpened(); - } else if (_inputGate.closedInLastBlock()) { - emit noiseGateClosed(); - } - - } else { - // our input loudness is 0, since we're muted - _lastInputLoudness = 0; - _timeSinceLastClip = 0.0f; - - _inputRingBuffer.shiftReadPosition(inputSamplesRequired); } - - auto packetType = _shouldEchoToServer ? - PacketType::MicrophoneAudioWithEcho : PacketType::MicrophoneAudioNoEcho; - - // if the _inputGate closed in this last frame, then we don't actually want - // to send a silent packet, instead, we want to go ahead and encode and send - // the output from the input gate (eventually, this could be crossfaded) - // and allow the codec to properly encode down to silent/zero. If we still - // have _lastInputLoudness of 0 in our NEXT frame, we will send a silent packet - if (_lastInputLoudness == 0 && !_inputGate.closedInLastBlock()) { - packetType = PacketType::SilentAudioFrame; - _silentOutbound.increment(); - } else { - _micAudioOutbound.increment(); - } - - Transform audioTransform; - audioTransform.setTranslation(_positionGetter()); - audioTransform.setRotation(_orientationGetter()); - // FIXME find a way to properly handle both playback audio and user audio concurrently - - QByteArray decodedBuffer(reinterpret_cast(networkAudioSamples), numNetworkBytes); - QByteArray encodedBuffer; - if (_encoder) { - _encoder->encode(decodedBuffer, encodedBuffer); - } else { - encodedBuffer = decodedBuffer; - } - - emitAudioPacket(encodedBuffer.constData(), encodedBuffer.size(), _outgoingAvatarAudioSequenceNumber, - audioTransform, avatarBoundingBoxCorner, avatarBoundingBoxScale, - packetType, _selectedCodecName); - _stats.sentPacket(); - int bytesInInputRingBuffer = _inputRingBuffer.samplesAvailable() * AudioConstants::SAMPLE_SIZE; float msecsInInputRingBuffer = bytesInInputRingBuffer / (float)(_inputFormat.bytesForDuration(USECS_PER_MSEC)); _stats.updateInputMsUnplayed(msecsInInputRingBuffer); + + QByteArray audioBuffer(reinterpret_cast(networkAudioSamples), numNetworkBytes); + handleAudioInput(audioBuffer); } } -// FIXME - should this go through the noise gate and honor mute and echo? void AudioClient::handleRecordedAudioInput(const QByteArray& audio) { - Transform audioTransform; - audioTransform.setTranslation(_positionGetter()); - audioTransform.setRotation(_orientationGetter()); - - QByteArray encodedBuffer; - if (_encoder) { - _encoder->encode(audio, encodedBuffer); - } else { - encodedBuffer = audio; - } - - _micAudioOutbound.increment(); - - // FIXME check a flag to see if we should echo audio? - emitAudioPacket(encodedBuffer.data(), encodedBuffer.size(), _outgoingAvatarAudioSequenceNumber, - audioTransform, avatarBoundingBoxCorner, avatarBoundingBoxScale, - PacketType::MicrophoneAudioWithEcho, _selectedCodecName); + QByteArray audioBuffer(audio); + handleAudioInput(audioBuffer); } void AudioClient::prepareLocalAudioInjectors() { @@ -1434,7 +1407,7 @@ bool AudioClient::switchInputToAudioDevice(const QAudioDeviceInfo& inputDeviceIn lock.unlock(); if (_inputDevice) { - connect(_inputDevice, SIGNAL(readyRead()), this, SLOT(handleAudioInput())); + connect(_inputDevice, SIGNAL(readyRead()), this, SLOT(handleMicAudioInput())); supportedFormat = true; } else { qCDebug(audioclient) << "Error starting audio input -" << _audioInput->error(); diff --git a/libraries/audio-client/src/AudioClient.h b/libraries/audio-client/src/AudioClient.h index 7e9acc0586..5ed88aa63a 100644 --- a/libraries/audio-client/src/AudioClient.h +++ b/libraries/audio-client/src/AudioClient.h @@ -124,16 +124,16 @@ public: void selectAudioFormat(const QString& selectedCodecName); Q_INVOKABLE QString getSelectedAudioFormat() const { return _selectedCodecName; } - Q_INVOKABLE bool getNoiseGateOpen() const { return _inputGate.isOpen(); } - Q_INVOKABLE float getSilentOutboundPPS() const { return _silentOutbound.rate(); } - Q_INVOKABLE float getMicAudioOutboundPPS() const { return _micAudioOutbound.rate(); } + Q_INVOKABLE bool getNoiseGateOpen() const { return _noiseGate.isOpen(); } Q_INVOKABLE float getSilentInboundPPS() const { return _silentInbound.rate(); } Q_INVOKABLE float getAudioInboundPPS() const { return _audioInbound.rate(); } + Q_INVOKABLE float getSilentOutboundPPS() const { return _silentOutbound.rate(); } + Q_INVOKABLE float getMicAudioOutboundPPS() const { return _audioOutbound.rate(); } const MixedProcessedAudioStream& getReceivedAudioStream() const { return _receivedAudioStream; } MixedProcessedAudioStream& getReceivedAudioStream() { return _receivedAudioStream; } - float getLastInputLoudness() const { return glm::max(_lastInputLoudness - _inputGate.getMeasuredFloor(), 0.0f); } + float getLastInputLoudness() const { return glm::max(_lastInputLoudness - _noiseGate.getMeasuredFloor(), 0.0f); } float getTimeSinceLastClip() const { return _timeSinceLastClip; } float getAudioAverageInputLoudness() const { return _lastInputLoudness; } @@ -180,7 +180,7 @@ public slots: void handleMismatchAudioFormat(SharedNodePointer node, const QString& currentCodec, const QString& recievedCodec); void sendDownstreamAudioStatsPacket() { _stats.publish(); } - void handleAudioInput(); + void handleMicAudioInput(); void handleRecordedAudioInput(const QByteArray& audio); void reset(); void audioMixerKilled(); @@ -250,6 +250,7 @@ protected: private: void outputFormatChanged(); + void handleAudioInput(QByteArray& audioBuffer); bool mixLocalAudioInjectors(float* mixBuffer); float azimuthForSource(const glm::vec3& relativePosition); float gainForSource(float distance, float volume); @@ -371,7 +372,7 @@ private: AudioIOStats _stats; - AudioNoiseGate _inputGate; + AudioNoiseGate _noiseGate; AudioPositionGetter _positionGetter; AudioOrientationGetter _orientationGetter; @@ -395,7 +396,7 @@ private: QThread* _checkDevicesThread { nullptr }; RateCounter<> _silentOutbound; - RateCounter<> _micAudioOutbound; + RateCounter<> _audioOutbound; RateCounter<> _silentInbound; RateCounter<> _audioInbound; }; From 789bb7c8550d439748d894d54b7f8e896d319dce Mon Sep 17 00:00:00 2001 From: Zach Pomerantz Date: Tue, 14 Mar 2017 19:07:48 +0000 Subject: [PATCH 2/2] rename audio(Mic)OutboundPPS --- interface/resources/qml/Stats.qml | 2 +- interface/src/ui/Stats.cpp | 6 +++--- interface/src/ui/Stats.h | 4 ++-- libraries/audio-client/src/AudioClient.h | 2 +- 4 files changed, 7 insertions(+), 7 deletions(-) diff --git a/interface/resources/qml/Stats.qml b/interface/resources/qml/Stats.qml index 564c74b526..6f131d0473 100644 --- a/interface/resources/qml/Stats.qml +++ b/interface/resources/qml/Stats.qml @@ -198,7 +198,7 @@ Item { } StatText { visible: root.expanded; - text: "Audio Out Mic: " + root.audioMicOutboundPPS + " pps, " + + text: "Audio Out Mic: " + root.audioOutboundPPS + " pps, " + "Silent: " + root.audioSilentOutboundPPS + " pps"; } StatText { diff --git a/interface/src/ui/Stats.cpp b/interface/src/ui/Stats.cpp index 923d9f642d..e8cd5409f0 100644 --- a/interface/src/ui/Stats.cpp +++ b/interface/src/ui/Stats.cpp @@ -220,10 +220,10 @@ void Stats::updateStats(bool force) { STAT_UPDATE(audioMixerInPps, roundf(bandwidthRecorder->getAverageInputPacketsPerSecond(NodeType::AudioMixer))); STAT_UPDATE(audioMixerOutKbps, roundf(bandwidthRecorder->getAverageOutputKilobitsPerSecond(NodeType::AudioMixer))); STAT_UPDATE(audioMixerOutPps, roundf(bandwidthRecorder->getAverageOutputPacketsPerSecond(NodeType::AudioMixer))); - STAT_UPDATE(audioMicOutboundPPS, audioClient->getMicAudioOutboundPPS()); - STAT_UPDATE(audioSilentOutboundPPS, audioClient->getSilentOutboundPPS()); STAT_UPDATE(audioAudioInboundPPS, audioClient->getAudioInboundPPS()); STAT_UPDATE(audioSilentInboundPPS, audioClient->getSilentInboundPPS()); + STAT_UPDATE(audioOutboundPPS, audioClient->getAudioOutboundPPS()); + STAT_UPDATE(audioSilentOutboundPPS, audioClient->getSilentOutboundPPS()); } else { STAT_UPDATE(audioMixerKbps, -1); STAT_UPDATE(audioMixerPps, -1); @@ -231,7 +231,7 @@ void Stats::updateStats(bool force) { STAT_UPDATE(audioMixerInPps, -1); STAT_UPDATE(audioMixerOutKbps, -1); STAT_UPDATE(audioMixerOutPps, -1); - STAT_UPDATE(audioMicOutboundPPS, -1); + STAT_UPDATE(audioOutboundPPS, -1); STAT_UPDATE(audioSilentOutboundPPS, -1); STAT_UPDATE(audioAudioInboundPPS, -1); STAT_UPDATE(audioSilentInboundPPS, -1); diff --git a/interface/src/ui/Stats.h b/interface/src/ui/Stats.h index 0ce113e0a0..964c94cd4d 100644 --- a/interface/src/ui/Stats.h +++ b/interface/src/ui/Stats.h @@ -77,7 +77,7 @@ class Stats : public QQuickItem { STATS_PROPERTY(int, audioMixerOutPps, 0) STATS_PROPERTY(int, audioMixerKbps, 0) STATS_PROPERTY(int, audioMixerPps, 0) - STATS_PROPERTY(int, audioMicOutboundPPS, 0) + STATS_PROPERTY(int, audioOutboundPPS, 0) STATS_PROPERTY(int, audioSilentOutboundPPS, 0) STATS_PROPERTY(int, audioAudioInboundPPS, 0) STATS_PROPERTY(int, audioSilentInboundPPS, 0) @@ -198,7 +198,7 @@ signals: void audioMixerOutPpsChanged(); void audioMixerKbpsChanged(); void audioMixerPpsChanged(); - void audioMicOutboundPPSChanged(); + void audioOutboundPPSChanged(); void audioSilentOutboundPPSChanged(); void audioAudioInboundPPSChanged(); void audioSilentInboundPPSChanged(); diff --git a/libraries/audio-client/src/AudioClient.h b/libraries/audio-client/src/AudioClient.h index 5ed88aa63a..0d180c0f5c 100644 --- a/libraries/audio-client/src/AudioClient.h +++ b/libraries/audio-client/src/AudioClient.h @@ -128,7 +128,7 @@ public: Q_INVOKABLE float getSilentInboundPPS() const { return _silentInbound.rate(); } Q_INVOKABLE float getAudioInboundPPS() const { return _audioInbound.rate(); } Q_INVOKABLE float getSilentOutboundPPS() const { return _silentOutbound.rate(); } - Q_INVOKABLE float getMicAudioOutboundPPS() const { return _audioOutbound.rate(); } + Q_INVOKABLE float getAudioOutboundPPS() const { return _audioOutbound.rate(); } const MixedProcessedAudioStream& getReceivedAudioStream() const { return _receivedAudioStream; } MixedProcessedAudioStream& getReceivedAudioStream() { return _receivedAudioStream; }