Mirrors/overte-HifiExperiments
3
0
Fork 0
mirror of https://github.com/HifiExperiments/overte.git synced 2025-04-12 02:52:10 +02:00
Code Issues Projects Releases Packages Wiki Activity Actions

Merge pull request #15936 from sethalves/webrtc

Browse source 
acoustic echo cancellation
This commit is contained in:
Ken Cooke 2019-08-01 05:55:52 -07:00 committed by GitHub
commit 41bcbf6dbd
No known key found for this signature in database
GPG key ID: 4AEE18F83AFDEB23
13 changed files with 388 additions and 44 deletions
Show stats Download patch file Download diff file Expand all files Collapse all files

24
cmake/macros/TargetWebRTC.cmake Normal file
View file

@ -0,0 +1,24 @@
#
# Copyright 2019 High Fidelity, Inc.
#
# Distributed under the Apache License, Version 2.0.
# See the accompanying file LICENSE or http://www.apache.org/licenses/LICENSE-2.0.html
#
macro(TARGET_WEBRTC)
if (ANDROID)
# I don't yet have working libwebrtc for android
# include(SelectLibraryConfigurations)
# set(INSTALL_DIR ${HIFI_ANDROID_PRECOMPILED}/webrtc/webrtc)
# set(WEBRTC_INCLUDE_DIRS "${INSTALL_DIR}/include/webrtc")
# set(WEBRTC_LIBRARY_DEBUG ${INSTALL_DIR}/debug/lib/libwebrtc.a)
# set(WEBRTC_LIBRARY_RELEASE ${INSTALL_DIR}/lib/libwebrtc.a)
# select_library_configurations(WEBRTC)
else()
set(WEBRTC_INCLUDE_DIRS "${VCPKG_INSTALL_ROOT}/include/webrtc")
find_library(WEBRTC_LIBRARY NAMES webrtc PATHS ${VCPKG_INSTALL_ROOT}/lib/ NO_DEFAULT_PATH)
target_include_directories(${TARGET_NAME} SYSTEM PUBLIC ${WEBRTC_INCLUDE_DIRS})
target_link_libraries(${TARGET_NAME} ${WEBRTC_LIBRARY})
endif()
endmacro()

2
cmake/ports/hifi-deps/CONTROL
View file

@ -1,4 +1,4 @@
Source: hifi-deps
Version: 0.1
Description: Collected dependencies for High Fidelity applications
Build-Depends: bullet3, draco, etc2comp, glm, nvtt, openexr (!android), openssl (windows), tbb (!android&!osx), zlib
Build-Depends: bullet3, draco, etc2comp, glm, nvtt, openexr (!android), openssl (windows), tbb (!android&!osx), zlib, webrtc (!android)

3
cmake/ports/webrtc/CONTROL Normal file
View file

@ -0,0 +1,3 @@
Source: webrtc
Version: 20190626
Description: WebRTC

36
cmake/ports/webrtc/portfile.cmake Normal file
View file

@ -0,0 +1,36 @@
include(vcpkg_common_functions)
set(WEBRTC_VERSION 20190626)
set(MASTER_COPY_SOURCE_PATH ${CURRENT_BUILDTREES_DIR}/src)
if (ANDROID)
# this is handled by hifi_android.py
elseif (WIN32)
vcpkg_download_distfile(
WEBRTC_SOURCE_ARCHIVE
URLS https://hifi-public.s3.amazonaws.com/seth/webrtc-20190626-windows.zip
SHA512 c0848eddb1579b3bb0496b8785e24f30470f3c477145035fd729264a326a467b9467ae9f426aa5d72d168ad9e9bf2c279150744832736bdf39064d24b04de1a3
FILENAME webrtc-20190626-windows.zip
)
elseif (APPLE)
vcpkg_download_distfile(
WEBRTC_SOURCE_ARCHIVE
URLS https://hifi-public.s3.amazonaws.com/seth/webrtc-20190626-osx.tar.gz
SHA512 fc70cec1b5ee87395137b7090f424e2fc2300fc17d744d5ffa1cf7aa0e0f1a069a9d72ba1ad2fb4a640ebeb6c218bda24351ba0083e1ff96c4a4b5032648a9d2
FILENAME webrtc-20190626-osx.tar.gz
)
else ()
# else Linux desktop
vcpkg_download_distfile(
WEBRTC_SOURCE_ARCHIVE
URLS https://hifi-public.s3.amazonaws.com/seth/webrtc-20190626-linux.tar.gz
SHA512 07d7776551aa78cb09a3ef088a8dee7762735c168c243053b262083d90a1d258cec66dc386f6903da5c4461921a3c2db157a1ee106a2b47e7756cb424b66cc43
FILENAME webrtc-20190626-linux.tar.gz
)
endif ()
vcpkg_extract_source_archive(${WEBRTC_SOURCE_ARCHIVE})
file(COPY ${MASTER_COPY_SOURCE_PATH}/webrtc/include DESTINATION ${CURRENT_PACKAGES_DIR})
file(COPY ${MASTER_COPY_SOURCE_PATH}/webrtc/lib DESTINATION ${CURRENT_PACKAGES_DIR})
file(COPY ${MASTER_COPY_SOURCE_PATH}/webrtc/share DESTINATION ${CURRENT_PACKAGES_DIR})
file(COPY ${MASTER_COPY_SOURCE_PATH}/webrtc/debug DESTINATION ${CURRENT_PACKAGES_DIR})

4
hifi_android.py
View file

@ -94,6 +94,10 @@ ANDROID_PACKAGES = {
'checksum': 'ddcb23df336b08017042ba4786db1d9e',
'sharedLibFolder': 'lib',
'includeLibs': {'libbreakpad_client.a'}
},
'webrtc': {
'file': 'webrtc-20190626-android.tar.gz',
'checksum': 'e2dccd3d8efdcba6d428c87ba7fb2a53'
}
}

24
interface/resources/qml/hifi/audio/Audio.qml
View file

@ -166,16 +166,16 @@ Rectangle {
x: 2 * margins.paddings;
width: parent.width;
// switch heights + 2 * top margins
height: (root.switchHeight) * 3 + 48;
height: (root.switchHeight) * 6 + 48;
anchors.top: firstSeparator.bottom;
anchors.topMargin: 10;
// mute is in its own row
Item {
id: switchContainer;
x: margins.paddings;
width: parent.width / 2;
height: parent.height;
anchors.top: parent.top
anchors.left: parent.left;
HifiControlsUit.Switch {
id: muteMic;
@ -222,12 +222,29 @@ Rectangle {
}
HifiControlsUit.Switch {
id: pttSwitch
id: acousticEchoCancellationSwitch;
height: root.switchHeight;
switchWidth: root.switchWidth;
anchors.top: noiseReductionSwitch.bottom
anchors.topMargin: 24
anchors.left: parent.left
labelTextOn: "Echo Cancellation";
labelTextSize: 16;
backgroundOnColor: "#E3E3E3";
checked: AudioScriptingInterface.acousticEchoCancellation;
onCheckedChanged: {
AudioScriptingInterface.acousticEchoCancellation = checked;
checked = Qt.binding(function() { return AudioScriptingInterface.acousticEchoCancellation; });
}
}
HifiControlsUit.Switch {
id: pttSwitch
height: root.switchHeight;
switchWidth: root.switchWidth;
anchors.top: acousticEchoCancellationSwitch.bottom;
anchors.topMargin: 24
anchors.left: parent.left
labelTextOn: (bar.currentIndex === 0) ? qsTr("Push To Talk (T)") : qsTr("Push To Talk");
labelTextSize: 16;
backgroundOnColor: "#E3E3E3";
@ -298,7 +315,6 @@ Rectangle {
checked = Qt.binding(function() { return AudioScriptingInterface.isStereoInput; }); // restore binding
}
}
}
}

11
interface/resources/qml/hifi/simplifiedUI/settingsApp/audio/Audio.qml
View file

@ -222,6 +222,17 @@ Flickable {
}
}
}
SimplifiedControls.Switch {
id: acousticEchoCancellationSwitch
Layout.preferredHeight: 18
Layout.preferredWidth: parent.width
labelTextOn: "Acoustic Echo Cancellation"
checked: AudioScriptingInterface.acousticEchoCancellation
onClicked: {
AudioScriptingInterface.acousticEchoCancellation = !AudioScriptingInterface.acousticEchoCancellation;
}
}
}
}

25
interface/src/scripting/Audio.cpp
View file

@ -26,6 +26,7 @@ QString Audio::HMD { "VR" };
Setting::Handle<bool> enableNoiseReductionSetting { QStringList { Audio::AUDIO, "NoiseReduction" }, true };
Setting::Handle<bool> enableWarnWhenMutedSetting { QStringList { Audio::AUDIO, "WarnWhenMuted" }, true };
Setting::Handle<bool> enableAcousticEchoCancellationSetting { QStringList { Audio::AUDIO, "AcousticEchoCancellation" }, true };
float Audio::loudnessToLevel(float loudness) {
@ -40,12 +41,14 @@ Audio::Audio() : _devices(_contextIsHMD) {
connect(client, &AudioClient::muteToggled, this, &Audio::setMuted);
connect(client, &AudioClient::noiseReductionChanged, this, &Audio::enableNoiseReduction);
connect(client, &AudioClient::warnWhenMutedChanged, this, &Audio::enableWarnWhenMuted);
connect(client, &AudioClient::acousticEchoCancellationChanged, this, &Audio::enableAcousticEchoCancellation);
connect(client, &AudioClient::inputLoudnessChanged, this, &Audio::onInputLoudnessChanged);
connect(client, &AudioClient::inputVolumeChanged, this, &Audio::setInputVolume);
connect(this, &Audio::contextChanged, &_devices, &AudioDevices::onContextChanged);
connect(this, &Audio::pushingToTalkChanged, this, &Audio::handlePushedToTalk);
enableNoiseReduction(enableNoiseReductionSetting.get());
enableWarnWhenMuted(enableWarnWhenMutedSetting.get());
enableAcousticEchoCancellation(enableAcousticEchoCancellationSetting.get());
onContextChanged();
}
@ -277,6 +280,28 @@ void Audio::enableWarnWhenMuted(bool enable) {
}
}
bool Audio::acousticEchoCancellationEnabled() const {
return resultWithReadLock<bool>([&] {
return _enableAcousticEchoCancellation;
});
}
void Audio::enableAcousticEchoCancellation(bool enable) {
bool changed = false;
withWriteLock([&] {
if (_enableAcousticEchoCancellation != enable) {
_enableAcousticEchoCancellation = enable;
auto client = DependencyManager::get<AudioClient>().data();
QMetaObject::invokeMethod(client, "setAcousticEchoCancellation", Q_ARG(bool, enable), Q_ARG(bool, false));
enableAcousticEchoCancellationSetting.set(enable);
changed = true;
}
});
if (changed) {
emit acousticEchoCancellationChanged(enable);
}
}
float Audio::getInputVolume() const {
return resultWithReadLock<bool>([&] {
return _inputVolume;

16
interface/src/scripting/Audio.h
View file

@ -72,6 +72,9 @@ class Audio : public AudioScriptingInterface, protected ReadWriteLockable {
* @property {number} systemInjectorGain - The gain (relative volume) that system sounds are played at.
* @property {number} pushingToTalkOutputGainDesktop - The gain (relative volume) that all sounds are played at when the user is holding
* the push-to-talk key in Desktop mode.
* @property {boolean} acousticEchoCancellation - <code>true</code> if audio-echo-cancellation is enabled, otherwise
* <code>false</code>. When enabled, sound from the audio output will be suppressed when it echos back to the
* input audio signal.
*
* @comment The following properties are from AudioScriptingInterface.h.
* @property {boolean} isStereoInput - <code>true</code> if the input audio is being used in stereo, otherwise
@ -85,6 +88,8 @@ class Audio : public AudioScriptingInterface, protected ReadWriteLockable {
Q_PROPERTY(bool muted READ isMuted WRITE setMuted NOTIFY mutedChanged)
Q_PROPERTY(bool noiseReduction READ noiseReductionEnabled WRITE enableNoiseReduction NOTIFY noiseReductionChanged)
Q_PROPERTY(bool warnWhenMuted READ warnWhenMutedEnabled WRITE enableWarnWhenMuted NOTIFY warnWhenMutedChanged)
Q_PROPERTY(bool acousticEchoCancellation
READ acousticEchoCancellationEnabled WRITE enableAcousticEchoCancellation NOTIFY acousticEchoCancellationChanged)
Q_PROPERTY(float inputVolume READ getInputVolume WRITE setInputVolume NOTIFY inputVolumeChanged)
Q_PROPERTY(float inputLevel READ getInputLevel NOTIFY inputLevelChanged)
Q_PROPERTY(bool clipping READ isClipping NOTIFY clippingChanged)
@ -115,6 +120,7 @@ public:
bool isMuted() const;
bool noiseReductionEnabled() const;
bool warnWhenMutedEnabled() const;
bool acousticEchoCancellationEnabled() const;
float getInputVolume() const;
float getInputLevel() const;
bool isClipping() const;
@ -396,6 +402,14 @@ signals:
*/
void warnWhenMutedChanged(bool isEnabled);
/**jsdoc
* Triggered when acoustic echo cancellation is enabled or disabled.
* @function Audio.acousticEchoCancellationChanged
* @param {boolean} isEnabled - <code>true</code> if acoustic echo cancellation is enabled, otherwise <code>false</code>.
* @returns {Signal}
*/
void acousticEchoCancellationChanged(bool isEnabled);
/**jsdoc
* Triggered when the input audio volume changes.
* @function Audio.inputVolumeChanged
@ -494,6 +508,7 @@ private slots:
void setMuted(bool muted);
void enableNoiseReduction(bool enable);
void enableWarnWhenMuted(bool enable);
void enableAcousticEchoCancellation(bool enable);
void setInputVolume(float volume);
void onInputLoudnessChanged(float loudness, bool isClipping);
@ -512,6 +527,7 @@ private:
bool _isClipping { false };
bool _enableNoiseReduction { true }; // Match default value of AudioClient::_isNoiseGateEnabled.
bool _enableWarnWhenMuted { true };
bool _enableAcousticEchoCancellation { true }; // AudioClient::_isAECEnabled
bool _contextIsHMD { false };
AudioDevices* getDevices() { return &_devices; }
AudioDevices _devices;

5
libraries/audio-client/CMakeLists.txt
View file

@ -7,6 +7,11 @@ link_hifi_libraries(audio plugins)
include_hifi_library_headers(shared)
include_hifi_library_headers(networking)
if (ANDROID)
else ()
target_webrtc()
endif ()
# append audio includes to our list of includes to bubble
target_include_directories(${TARGET_NAME} PUBLIC "${HIFI_LIBRARY_DIR}/audio/src")

224
libraries/audio-client/src/AudioClient.cpp
View file

@ -24,7 +24,7 @@
#endif
#ifdef WIN32
#define WIN32_LEAN_AND_MEAN
#define WIN32_LEAN_AND_MEAN 1
#include <windows.h>
#include <Mmsystem.h>
#include <mmdeviceapi.h>
@ -286,6 +286,7 @@ AudioClient::AudioClient() :
_shouldEchoLocally(false),
_shouldEchoToServer(false),
_isNoiseGateEnabled(true),
_isAECEnabled(true),
_reverb(false),
_reverbOptions(&_scriptReverbOptions),
_inputToNetworkResampler(NULL),
@ -302,6 +303,7 @@ AudioClient::AudioClient() :
_isHeadsetPluggedIn(false),
#endif
_orientationGetter(DEFAULT_ORIENTATION_GETTER) {
// avoid putting a lock in the device callback
assert(_localSamplesAvailable.is_lock_free());
@ -353,6 +355,10 @@ AudioClient::AudioClient() :
configureReverb();
#if defined(WEBRTC_ENABLED)
configureWebrtc();
#endif
auto nodeList = DependencyManager::get<NodeList>();
auto& packetReceiver = nodeList->getPacketReceiver();
packetReceiver.registerListener(PacketType::AudioStreamStats, &_stats, "processStreamStatsPacket");
@ -1084,6 +1090,131 @@ void AudioClient::setReverbOptions(const AudioEffectOptions* options) {
}
}
#if defined(WEBRTC_ENABLED)
static void deinterleaveToFloat(const int16_t* src, float* const* dst, int numFrames, int numChannels) {
for (int i = 0; i < numFrames; i++) {
for (int ch = 0; ch < numChannels; ch++) {
float f = *src++;
f *= (1/32768.0f); // scale
dst[ch][i] = f; // deinterleave
}
}
}
static void interleaveToInt16(const float* const* src, int16_t* dst, int numFrames, int numChannels) {
for (int i = 0; i < numFrames; i++) {
for (int ch = 0; ch < numChannels; ch++) {
float f = src[ch][i];
f *= 32768.0f; // scale
f += (f < 0.0f) ? -0.5f : 0.5f; // round
f = std::max(std::min(f, 32767.0f), -32768.0f); // saturate
*dst++ = (int16_t)f; // interleave
}
}
}
void AudioClient::configureWebrtc() {
_apm = webrtc::AudioProcessingBuilder().Create();
webrtc::AudioProcessing::Config config;
config.pre_amplifier.enabled = false;
config.high_pass_filter.enabled = false;
config.echo_canceller.enabled = true;
config.echo_canceller.mobile_mode = false;
config.echo_canceller.use_legacy_aec = false;
config.noise_suppression.enabled = false;
config.noise_suppression.level = webrtc::AudioProcessing::Config::NoiseSuppression::kModerate;
config.voice_detection.enabled = false;
config.gain_controller1.enabled = false;
config.gain_controller2.enabled = false;
config.gain_controller2.fixed_digital.gain_db = 0.0f;
config.gain_controller2.adaptive_digital.enabled = false;
config.residual_echo_detector.enabled = true;
config.level_estimation.enabled = false;
_apm->ApplyConfig(config);
}
// rebuffer into 10ms chunks
void AudioClient::processWebrtcFarEnd(const int16_t* samples, int numFrames, int numChannels, int sampleRate) {
const webrtc::StreamConfig streamConfig = webrtc::StreamConfig(sampleRate, numChannels);
const int numChunk = (int)streamConfig.num_frames();
if (sampleRate > WEBRTC_SAMPLE_RATE_MAX) {
qCWarning(audioclient) << "WebRTC does not support" << sampleRate << "output sample rate.";
return;
}
if (numChannels > WEBRTC_CHANNELS_MAX) {
qCWarning(audioclient) << "WebRTC does not support" << numChannels << "output channels.";
return;
}
while (numFrames > 0) {
// number of frames to fill
int numFill = std::min(numFrames, numChunk - _numFifoFarEnd);
// refill fifo
memcpy(&_fifoFarEnd[_numFifoFarEnd], samples, numFill * numChannels * sizeof(int16_t));
samples += numFill * numChannels;
numFrames -= numFill;
_numFifoFarEnd += numFill;
if (_numFifoFarEnd == numChunk) {
// convert audio format
float buffer[WEBRTC_CHANNELS_MAX][WEBRTC_FRAMES_MAX];
float* const buffers[WEBRTC_CHANNELS_MAX] = { buffer[0], buffer[1] };
deinterleaveToFloat(_fifoFarEnd, buffers, numChunk, numChannels);
// process one chunk
int error = _apm->ProcessReverseStream(buffers, streamConfig, streamConfig, buffers);
if (error != _apm->kNoError) {
qCWarning(audioclient) << "WebRTC ProcessReverseStream() returned ERROR:" << error;
}
_numFifoFarEnd = 0;
}
}
}
void AudioClient::processWebrtcNearEnd(int16_t* samples, int numFrames, int numChannels, int sampleRate) {
const webrtc::StreamConfig streamConfig = webrtc::StreamConfig(sampleRate, numChannels);
const int numChunk = (int)streamConfig.num_frames();
if (sampleRate > WEBRTC_SAMPLE_RATE_MAX) {
qCWarning(audioclient) << "WebRTC does not support" << sampleRate << "input sample rate.";
return;
}
if (numChannels > WEBRTC_CHANNELS_MAX) {
qCWarning(audioclient) << "WebRTC does not support" << numChannels << "input channels.";
return;
}
if (numFrames != numChunk) {
qCWarning(audioclient) << "WebRTC requires exactly 10ms of input.";
return;
}
// convert audio format
float buffer[WEBRTC_CHANNELS_MAX][WEBRTC_FRAMES_MAX];
float* const buffers[WEBRTC_CHANNELS_MAX] = { buffer[0], buffer[1] };
deinterleaveToFloat(samples, buffers, numFrames, numChannels);
// process one chunk
int error = _apm->ProcessStream(buffers, streamConfig, streamConfig, buffers);
if (error != _apm->kNoError) {
qCWarning(audioclient) << "WebRTC ProcessStream() returned ERROR:" << error;
} else {
// modify samples in-place
interleaveToInt16(buffers, samples, numFrames, numChannels);
}
}
#endif // WEBRTC_ENABLED
void AudioClient::handleLocalEchoAndReverb(QByteArray& inputByteArray) {
// If there is server echo, reverb will be applied to the recieved audio stream so no need to have it here.
bool hasReverb = _reverb || _receivedAudioStream.hasReverb();
@ -1262,6 +1393,13 @@ void AudioClient::handleMicAudioInput() {
_inputRingBuffer.readSamples(inputAudioSamples.get(), inputSamplesRequired);
#if defined(WEBRTC_ENABLED)
if (_isAECEnabled) {
processWebrtcNearEnd(inputAudioSamples.get(), inputSamplesRequired / _inputFormat.channelCount(),
_inputFormat.channelCount(), _inputFormat.sampleRate());
}
#endif
// detect loudness and clipping on the raw input
bool isClipping = false;
float loudness = computeLoudness(inputAudioSamples.get(), inputSamplesRequired, _inputFormat.channelCount(), isClipping);
@ -1574,6 +1712,15 @@ void AudioClient::setWarnWhenMuted(bool enable, bool emitSignal) {
}
}
void AudioClient::setAcousticEchoCancellation(bool enable, bool emitSignal) {
if (_isAECEnabled != enable) {
_isAECEnabled = enable;
if (emitSignal) {
emit acousticEchoCancellationChanged(_isAECEnabled);
}
}
}
bool AudioClient::setIsStereoInput(bool isStereoInput) {
bool stereoInputChanged = false;
if (isStereoInput != _isStereoInput && _inputDeviceInfo.supportedChannelCounts().contains(2)) {
@ -2107,15 +2254,16 @@ qint64 AudioClient::AudioOutputIODevice::readData(char * data, qint64 maxSize) {
return maxSize;
}
// samples requested from OUTPUT_CHANNEL_COUNT
// max samples requested from OUTPUT_CHANNEL_COUNT
int deviceChannelCount = _audio->_outputFormat.channelCount();
int samplesRequested = (int)(maxSize / AudioConstants::SAMPLE_SIZE) * OUTPUT_CHANNEL_COUNT / deviceChannelCount;
int maxSamplesRequested = (int)(maxSize / AudioConstants::SAMPLE_SIZE) * OUTPUT_CHANNEL_COUNT / deviceChannelCount;
// restrict samplesRequested to the size of our mix/scratch buffers
samplesRequested = std::min(samplesRequested, _audio->_outputPeriod);
maxSamplesRequested = std::min(maxSamplesRequested, _audio->_outputPeriod);
int16_t* scratchBuffer = _audio->_outputScratchBuffer;
float* mixBuffer = _audio->_outputMixBuffer;
int samplesRequested = maxSamplesRequested;
int networkSamplesPopped;
if ((networkSamplesPopped = _receivedAudioStream.popSamples(samplesRequested, false)) > 0) {
qCDebug(audiostream, "Read %d samples from buffer (%d available, %d requested)", networkSamplesPopped, _receivedAudioStream.getSamplesAvailable(), samplesRequested);
@ -2160,45 +2308,45 @@ qint64 AudioClient::AudioOutputIODevice::readData(char * data, qint64 maxSize) {
});
int samplesPopped = std::max(networkSamplesPopped, injectorSamplesPopped);
int framesPopped = samplesPopped / AudioConstants::STEREO;
int bytesWritten;
if (samplesPopped > 0) {
// apply output gain
float newGain = _audio->_outputGain.load(std::memory_order_acquire);
float oldGain = _audio->_lastOutputGain;
_audio->_lastOutputGain = newGain;
applyGainSmoothing<OUTPUT_CHANNEL_COUNT>(mixBuffer, framesPopped, oldGain, newGain);
if (deviceChannelCount == OUTPUT_CHANNEL_COUNT) {
// limit the audio
_audio->_audioLimiter.render(mixBuffer, (int16_t*)data, framesPopped);
} else {
_audio->_audioLimiter.render(mixBuffer, scratchBuffer, framesPopped);
// upmix or downmix to deviceChannelCount
if (deviceChannelCount > OUTPUT_CHANNEL_COUNT) {
int extraChannels = deviceChannelCount - OUTPUT_CHANNEL_COUNT;
channelUpmix(scratchBuffer, (int16_t*)data, samplesPopped, extraChannels);
} else {
channelDownmix(scratchBuffer, (int16_t*)data, samplesPopped);
}
}
bytesWritten = framesPopped * AudioConstants::SAMPLE_SIZE * deviceChannelCount;
assert(bytesWritten <= maxSize);
} else {
// nothing on network, don't grab anything from injectors, and just return 0s
memset(data, 0, maxSize);
bytesWritten = maxSize;
if (samplesPopped == 0) {
// nothing on network, don't grab anything from injectors, and fill with silence
samplesPopped = maxSamplesRequested;
memset(mixBuffer, 0, samplesPopped * sizeof(float));
}
int framesPopped = samplesPopped / OUTPUT_CHANNEL_COUNT;
// apply output gain
float newGain = _audio->_outputGain.load(std::memory_order_acquire);
float oldGain = _audio->_lastOutputGain;
_audio->_lastOutputGain = newGain;
applyGainSmoothing<OUTPUT_CHANNEL_COUNT>(mixBuffer, framesPopped, oldGain, newGain);
// limit the audio
_audio->_audioLimiter.render(mixBuffer, scratchBuffer, framesPopped);
#if defined(WEBRTC_ENABLED)
if (_audio->_isAECEnabled) {
_audio->processWebrtcFarEnd(scratchBuffer, framesPopped, OUTPUT_CHANNEL_COUNT, _audio->_outputFormat.sampleRate());
}
#endif
// if required, upmix or downmix to deviceChannelCount
if (deviceChannelCount == OUTPUT_CHANNEL_COUNT) {
memcpy(data, scratchBuffer, samplesPopped * AudioConstants::SAMPLE_SIZE);
} else if (deviceChannelCount > OUTPUT_CHANNEL_COUNT) {
int extraChannels = deviceChannelCount - OUTPUT_CHANNEL_COUNT;
channelUpmix(scratchBuffer, (int16_t*)data, samplesPopped, extraChannels);
} else {
channelDownmix(scratchBuffer, (int16_t*)data, samplesPopped);
}
int bytesWritten = framesPopped * AudioConstants::SAMPLE_SIZE * deviceChannelCount;
assert(bytesWritten <= maxSize);
// send output buffer for recording
if (_audio->_isRecording) {
Lock lock(_recordMutex);
_audio->_audioFileWav.addRawAudioChunk(reinterpret_cast<char*>(scratchBuffer), bytesWritten);
_audio->_audioFileWav.addRawAudioChunk(data, bytesWritten);
}
int bytesAudioOutputUnplayed = _audio->_audioOutput->bufferSize() - _audio->_audioOutput->bytesFree();

22
libraries/audio-client/src/AudioClient.h
View file

@ -29,6 +29,7 @@
#include <AbstractAudioInterface.h>
#include <AudioEffectOptions.h>
#include <AudioStreamStats.h>
#include <shared/WebRTC.h>
#include <DependencyManager.h>
#include <HifiSockAddr.h>
@ -215,6 +216,9 @@ public slots:
void setWarnWhenMuted(bool isNoiseGateEnabled, bool emitSignal = true);
bool isWarnWhenMutedEnabled() const { return _warnWhenMuted; }
void setAcousticEchoCancellation(bool isAECEnabled, bool emitSignal = true);
bool isAcousticEchoCancellationEnabled() const { return _isAECEnabled; }
virtual bool getLocalEcho() override { return _shouldEchoLocally; }
virtual void setLocalEcho(bool localEcho) override { _shouldEchoLocally = localEcho; }
virtual void toggleLocalEcho() override { _shouldEchoLocally = !_shouldEchoLocally; }
@ -256,6 +260,7 @@ signals:
void muteToggled(bool muted);
void noiseReductionChanged(bool noiseReductionEnabled);
void warnWhenMutedChanged(bool warnWhenMutedEnabled);
void acousticEchoCancellationChanged(bool acousticEchoCancellationEnabled);
void mutedByMixer();
void inputReceived(const QByteArray& inputSamples);
void inputLoudnessChanged(float loudness, bool isClipping);
@ -377,6 +382,7 @@ private:
bool _shouldEchoToServer;
bool _isNoiseGateEnabled;
bool _warnWhenMuted;
bool _isAECEnabled;
bool _reverb;
AudioEffectOptions _scriptReverbOptions;
@ -414,9 +420,23 @@ private:
// Adds Reverb
void configureReverb();
void updateReverbOptions();
void handleLocalEchoAndReverb(QByteArray& inputByteArray);
#if defined(WEBRTC_ENABLED)
static const int WEBRTC_SAMPLE_RATE_MAX = 96000;
static const int WEBRTC_CHANNELS_MAX = 2;
static const int WEBRTC_FRAMES_MAX = webrtc::AudioProcessing::kChunkSizeMs * WEBRTC_SAMPLE_RATE_MAX / 1000;
webrtc::AudioProcessing* _apm { nullptr };
int16_t _fifoFarEnd[WEBRTC_CHANNELS_MAX * WEBRTC_FRAMES_MAX] {};
int _numFifoFarEnd = 0; // numFrames saved in fifo
void configureWebrtc();
void processWebrtcFarEnd(const int16_t* samples, int numFrames, int numChannels, int sampleRate);
void processWebrtcNearEnd(int16_t* samples, int numFrames, int numChannels, int sampleRate);
#endif
bool switchInputToAudioDevice(const QAudioDeviceInfo inputDeviceInfo, bool isShutdownRequest = false);
bool switchOutputToAudioDevice(const QAudioDeviceInfo outputDeviceInfo, bool isShutdownRequest = false);

36
libraries/shared/src/shared/WebRTC.h Normal file
View file

@ -0,0 +1,36 @@
//
// WebRTC.h
// libraries/shared/src/shared/
//
// Copyright 2019 High Fidelity, Inc.
//
// Distributed under the Apache License, Version 2.0.
// See the accompanying file LICENSE or http://www.apache.org/licenses/LICENSE-2.0.html
//
#ifndef hifi_WebRTC_h
#define hifi_WebRTC_h
#if defined(Q_OS_MAC)
# define WEBRTC_ENABLED 1
# define WEBRTC_POSIX 1
#elif defined(Q_OS_WIN)
# define WEBRTC_ENABLED 1
# define WEBRTC_WIN 1
# define NOMINMAX 1
# define WIN32_LEAN_AND_MEAN 1
#elif defined(Q_OS_ANDROID)
// I don't yet have a working libwebrtc for android
// # define WEBRTC_ENABLED 1
// # define WEBRTC_POSIX 1
#elif defined(Q_OS_LINUX)
# define WEBRTC_ENABLED 1
# define WEBRTC_POSIX 1
#endif
#if defined(WEBRTC_ENABLED)
# include <modules/audio_processing/include/audio_processing.h>
# include "modules/audio_processing/audio_processing_impl.h"
#endif
#endif // hifi_WebRTC_h