diff --git a/cmake/macros/TargetWebRTC.cmake b/cmake/macros/TargetWebRTC.cmake new file mode 100644 index 0000000000..d2821528df --- /dev/null +++ b/cmake/macros/TargetWebRTC.cmake @@ -0,0 +1,24 @@ +# +# Copyright 2019 High Fidelity, Inc. +# +# Distributed under the Apache License, Version 2.0. +# See the accompanying file LICENSE or http://www.apache.org/licenses/LICENSE-2.0.html +# +macro(TARGET_WEBRTC) + if (ANDROID) + # I don't yet have working libwebrtc for android + # include(SelectLibraryConfigurations) + # set(INSTALL_DIR ${HIFI_ANDROID_PRECOMPILED}/webrtc/webrtc) + # set(WEBRTC_INCLUDE_DIRS "${INSTALL_DIR}/include/webrtc") + # set(WEBRTC_LIBRARY_DEBUG ${INSTALL_DIR}/debug/lib/libwebrtc.a) + # set(WEBRTC_LIBRARY_RELEASE ${INSTALL_DIR}/lib/libwebrtc.a) + # select_library_configurations(WEBRTC) + else() + set(WEBRTC_INCLUDE_DIRS "${VCPKG_INSTALL_ROOT}/include/webrtc") + find_library(WEBRTC_LIBRARY NAMES webrtc PATHS ${VCPKG_INSTALL_ROOT}/lib/ NO_DEFAULT_PATH) + target_include_directories(${TARGET_NAME} SYSTEM PUBLIC ${WEBRTC_INCLUDE_DIRS}) + target_link_libraries(${TARGET_NAME} ${WEBRTC_LIBRARY}) + endif() + + +endmacro() diff --git a/cmake/ports/hifi-deps/CONTROL b/cmake/ports/hifi-deps/CONTROL index 5f860a1620..2441de9002 100644 --- a/cmake/ports/hifi-deps/CONTROL +++ b/cmake/ports/hifi-deps/CONTROL @@ -1,4 +1,4 @@ Source: hifi-deps Version: 0.1 Description: Collected dependencies for High Fidelity applications -Build-Depends: bullet3, draco, etc2comp, glm, nvtt, openexr (!android), openssl (windows), tbb (!android&!osx), zlib +Build-Depends: bullet3, draco, etc2comp, glm, nvtt, openexr (!android), openssl (windows), tbb (!android&!osx), zlib, webrtc (!android) diff --git a/cmake/ports/webrtc/CONTROL b/cmake/ports/webrtc/CONTROL new file mode 100644 index 0000000000..12a76920b9 --- /dev/null +++ b/cmake/ports/webrtc/CONTROL @@ -0,0 +1,3 @@ +Source: webrtc +Version: 20190626 +Description: WebRTC diff --git a/cmake/ports/webrtc/portfile.cmake b/cmake/ports/webrtc/portfile.cmake new file mode 100644 index 0000000000..3f2fb7a6ab --- /dev/null +++ b/cmake/ports/webrtc/portfile.cmake @@ -0,0 +1,36 @@ +include(vcpkg_common_functions) +set(WEBRTC_VERSION 20190626) +set(MASTER_COPY_SOURCE_PATH ${CURRENT_BUILDTREES_DIR}/src) + +if (ANDROID) + # this is handled by hifi_android.py +elseif (WIN32) + vcpkg_download_distfile( + WEBRTC_SOURCE_ARCHIVE + URLS https://hifi-public.s3.amazonaws.com/seth/webrtc-20190626-windows.zip + SHA512 c0848eddb1579b3bb0496b8785e24f30470f3c477145035fd729264a326a467b9467ae9f426aa5d72d168ad9e9bf2c279150744832736bdf39064d24b04de1a3 + FILENAME webrtc-20190626-windows.zip + ) +elseif (APPLE) + vcpkg_download_distfile( + WEBRTC_SOURCE_ARCHIVE + URLS https://hifi-public.s3.amazonaws.com/seth/webrtc-20190626-osx.tar.gz + SHA512 fc70cec1b5ee87395137b7090f424e2fc2300fc17d744d5ffa1cf7aa0e0f1a069a9d72ba1ad2fb4a640ebeb6c218bda24351ba0083e1ff96c4a4b5032648a9d2 + FILENAME webrtc-20190626-osx.tar.gz + ) +else () + # else Linux desktop + vcpkg_download_distfile( + WEBRTC_SOURCE_ARCHIVE + URLS https://hifi-public.s3.amazonaws.com/seth/webrtc-20190626-linux.tar.gz + SHA512 07d7776551aa78cb09a3ef088a8dee7762735c168c243053b262083d90a1d258cec66dc386f6903da5c4461921a3c2db157a1ee106a2b47e7756cb424b66cc43 + FILENAME webrtc-20190626-linux.tar.gz + ) +endif () + +vcpkg_extract_source_archive(${WEBRTC_SOURCE_ARCHIVE}) + +file(COPY ${MASTER_COPY_SOURCE_PATH}/webrtc/include DESTINATION ${CURRENT_PACKAGES_DIR}) +file(COPY ${MASTER_COPY_SOURCE_PATH}/webrtc/lib DESTINATION ${CURRENT_PACKAGES_DIR}) +file(COPY ${MASTER_COPY_SOURCE_PATH}/webrtc/share DESTINATION ${CURRENT_PACKAGES_DIR}) +file(COPY ${MASTER_COPY_SOURCE_PATH}/webrtc/debug DESTINATION ${CURRENT_PACKAGES_DIR}) diff --git a/hifi_android.py b/hifi_android.py index 42b472e960..0c2ea07cc7 100644 --- a/hifi_android.py +++ b/hifi_android.py @@ -94,6 +94,10 @@ ANDROID_PACKAGES = { 'checksum': 'ddcb23df336b08017042ba4786db1d9e', 'sharedLibFolder': 'lib', 'includeLibs': {'libbreakpad_client.a'} + }, + 'webrtc': { + 'file': 'webrtc-20190626-android.tar.gz', + 'checksum': 'e2dccd3d8efdcba6d428c87ba7fb2a53' } } diff --git a/interface/resources/qml/hifi/audio/Audio.qml b/interface/resources/qml/hifi/audio/Audio.qml index af517be55d..9db19e72e2 100644 --- a/interface/resources/qml/hifi/audio/Audio.qml +++ b/interface/resources/qml/hifi/audio/Audio.qml @@ -166,16 +166,16 @@ Rectangle { x: 2 * margins.paddings; width: parent.width; // switch heights + 2 * top margins - height: (root.switchHeight) * 3 + 48; + height: (root.switchHeight) * 6 + 48; anchors.top: firstSeparator.bottom; anchors.topMargin: 10; - // mute is in its own row Item { id: switchContainer; x: margins.paddings; width: parent.width / 2; height: parent.height; + anchors.top: parent.top anchors.left: parent.left; HifiControlsUit.Switch { id: muteMic; @@ -222,12 +222,29 @@ Rectangle { } HifiControlsUit.Switch { - id: pttSwitch + id: acousticEchoCancellationSwitch; height: root.switchHeight; switchWidth: root.switchWidth; anchors.top: noiseReductionSwitch.bottom anchors.topMargin: 24 anchors.left: parent.left + labelTextOn: "Echo Cancellation"; + labelTextSize: 16; + backgroundOnColor: "#E3E3E3"; + checked: AudioScriptingInterface.acousticEchoCancellation; + onCheckedChanged: { + AudioScriptingInterface.acousticEchoCancellation = checked; + checked = Qt.binding(function() { return AudioScriptingInterface.acousticEchoCancellation; }); + } + } + + HifiControlsUit.Switch { + id: pttSwitch + height: root.switchHeight; + switchWidth: root.switchWidth; + anchors.top: acousticEchoCancellationSwitch.bottom; + anchors.topMargin: 24 + anchors.left: parent.left labelTextOn: (bar.currentIndex === 0) ? qsTr("Push To Talk (T)") : qsTr("Push To Talk"); labelTextSize: 16; backgroundOnColor: "#E3E3E3"; @@ -298,7 +315,6 @@ Rectangle { checked = Qt.binding(function() { return AudioScriptingInterface.isStereoInput; }); // restore binding } } - } } diff --git a/interface/resources/qml/hifi/simplifiedUI/settingsApp/audio/Audio.qml b/interface/resources/qml/hifi/simplifiedUI/settingsApp/audio/Audio.qml index 8827bb3834..bfc0bc5200 100644 --- a/interface/resources/qml/hifi/simplifiedUI/settingsApp/audio/Audio.qml +++ b/interface/resources/qml/hifi/simplifiedUI/settingsApp/audio/Audio.qml @@ -222,6 +222,17 @@ Flickable { } } } + + SimplifiedControls.Switch { + id: acousticEchoCancellationSwitch + Layout.preferredHeight: 18 + Layout.preferredWidth: parent.width + labelTextOn: "Acoustic Echo Cancellation" + checked: AudioScriptingInterface.acousticEchoCancellation + onClicked: { + AudioScriptingInterface.acousticEchoCancellation = !AudioScriptingInterface.acousticEchoCancellation; + } + } } } diff --git a/interface/src/scripting/Audio.cpp b/interface/src/scripting/Audio.cpp index a0bea256ad..f674b533a7 100644 --- a/interface/src/scripting/Audio.cpp +++ b/interface/src/scripting/Audio.cpp @@ -26,6 +26,7 @@ QString Audio::HMD { "VR" }; Setting::Handle enableNoiseReductionSetting { QStringList { Audio::AUDIO, "NoiseReduction" }, true }; Setting::Handle enableWarnWhenMutedSetting { QStringList { Audio::AUDIO, "WarnWhenMuted" }, true }; +Setting::Handle enableAcousticEchoCancellationSetting { QStringList { Audio::AUDIO, "AcousticEchoCancellation" }, true }; float Audio::loudnessToLevel(float loudness) { @@ -40,12 +41,14 @@ Audio::Audio() : _devices(_contextIsHMD) { connect(client, &AudioClient::muteToggled, this, &Audio::setMuted); connect(client, &AudioClient::noiseReductionChanged, this, &Audio::enableNoiseReduction); connect(client, &AudioClient::warnWhenMutedChanged, this, &Audio::enableWarnWhenMuted); + connect(client, &AudioClient::acousticEchoCancellationChanged, this, &Audio::enableAcousticEchoCancellation); connect(client, &AudioClient::inputLoudnessChanged, this, &Audio::onInputLoudnessChanged); connect(client, &AudioClient::inputVolumeChanged, this, &Audio::setInputVolume); connect(this, &Audio::contextChanged, &_devices, &AudioDevices::onContextChanged); connect(this, &Audio::pushingToTalkChanged, this, &Audio::handlePushedToTalk); enableNoiseReduction(enableNoiseReductionSetting.get()); enableWarnWhenMuted(enableWarnWhenMutedSetting.get()); + enableAcousticEchoCancellation(enableAcousticEchoCancellationSetting.get()); onContextChanged(); } @@ -277,6 +280,28 @@ void Audio::enableWarnWhenMuted(bool enable) { } } +bool Audio::acousticEchoCancellationEnabled() const { + return resultWithReadLock([&] { + return _enableAcousticEchoCancellation; + }); +} + +void Audio::enableAcousticEchoCancellation(bool enable) { + bool changed = false; + withWriteLock([&] { + if (_enableAcousticEchoCancellation != enable) { + _enableAcousticEchoCancellation = enable; + auto client = DependencyManager::get().data(); + QMetaObject::invokeMethod(client, "setAcousticEchoCancellation", Q_ARG(bool, enable), Q_ARG(bool, false)); + enableAcousticEchoCancellationSetting.set(enable); + changed = true; + } + }); + if (changed) { + emit acousticEchoCancellationChanged(enable); + } +} + float Audio::getInputVolume() const { return resultWithReadLock([&] { return _inputVolume; diff --git a/interface/src/scripting/Audio.h b/interface/src/scripting/Audio.h index aab1ade95b..d2d1ee36c9 100644 --- a/interface/src/scripting/Audio.h +++ b/interface/src/scripting/Audio.h @@ -72,6 +72,9 @@ class Audio : public AudioScriptingInterface, protected ReadWriteLockable { * @property {number} systemInjectorGain - The gain (relative volume) that system sounds are played at. * @property {number} pushingToTalkOutputGainDesktop - The gain (relative volume) that all sounds are played at when the user is holding * the push-to-talk key in Desktop mode. + * @property {boolean} acousticEchoCancellation - true if audio-echo-cancellation is enabled, otherwise + * false. When enabled, sound from the audio output will be suppressed when it echos back to the + * input audio signal. * * @comment The following properties are from AudioScriptingInterface.h. * @property {boolean} isStereoInput - true if the input audio is being used in stereo, otherwise @@ -85,6 +88,8 @@ class Audio : public AudioScriptingInterface, protected ReadWriteLockable { Q_PROPERTY(bool muted READ isMuted WRITE setMuted NOTIFY mutedChanged) Q_PROPERTY(bool noiseReduction READ noiseReductionEnabled WRITE enableNoiseReduction NOTIFY noiseReductionChanged) Q_PROPERTY(bool warnWhenMuted READ warnWhenMutedEnabled WRITE enableWarnWhenMuted NOTIFY warnWhenMutedChanged) + Q_PROPERTY(bool acousticEchoCancellation + READ acousticEchoCancellationEnabled WRITE enableAcousticEchoCancellation NOTIFY acousticEchoCancellationChanged) Q_PROPERTY(float inputVolume READ getInputVolume WRITE setInputVolume NOTIFY inputVolumeChanged) Q_PROPERTY(float inputLevel READ getInputLevel NOTIFY inputLevelChanged) Q_PROPERTY(bool clipping READ isClipping NOTIFY clippingChanged) @@ -115,6 +120,7 @@ public: bool isMuted() const; bool noiseReductionEnabled() const; bool warnWhenMutedEnabled() const; + bool acousticEchoCancellationEnabled() const; float getInputVolume() const; float getInputLevel() const; bool isClipping() const; @@ -396,6 +402,14 @@ signals: */ void warnWhenMutedChanged(bool isEnabled); + /**jsdoc + * Triggered when acoustic echo cancellation is enabled or disabled. + * @function Audio.acousticEchoCancellationChanged + * @param {boolean} isEnabled - true if acoustic echo cancellation is enabled, otherwise false. + * @returns {Signal} + */ + void acousticEchoCancellationChanged(bool isEnabled); + /**jsdoc * Triggered when the input audio volume changes. * @function Audio.inputVolumeChanged @@ -494,6 +508,7 @@ private slots: void setMuted(bool muted); void enableNoiseReduction(bool enable); void enableWarnWhenMuted(bool enable); + void enableAcousticEchoCancellation(bool enable); void setInputVolume(float volume); void onInputLoudnessChanged(float loudness, bool isClipping); @@ -512,6 +527,7 @@ private: bool _isClipping { false }; bool _enableNoiseReduction { true }; // Match default value of AudioClient::_isNoiseGateEnabled. bool _enableWarnWhenMuted { true }; + bool _enableAcousticEchoCancellation { true }; // AudioClient::_isAECEnabled bool _contextIsHMD { false }; AudioDevices* getDevices() { return &_devices; } AudioDevices _devices; diff --git a/libraries/audio-client/CMakeLists.txt b/libraries/audio-client/CMakeLists.txt index 6ca7962c39..6b88292dd4 100644 --- a/libraries/audio-client/CMakeLists.txt +++ b/libraries/audio-client/CMakeLists.txt @@ -7,6 +7,11 @@ link_hifi_libraries(audio plugins) include_hifi_library_headers(shared) include_hifi_library_headers(networking) +if (ANDROID) +else () + target_webrtc() +endif () + # append audio includes to our list of includes to bubble target_include_directories(${TARGET_NAME} PUBLIC "${HIFI_LIBRARY_DIR}/audio/src") diff --git a/libraries/audio-client/src/AudioClient.cpp b/libraries/audio-client/src/AudioClient.cpp index 04ab0f7973..c16e297c28 100644 --- a/libraries/audio-client/src/AudioClient.cpp +++ b/libraries/audio-client/src/AudioClient.cpp @@ -24,7 +24,7 @@ #endif #ifdef WIN32 -#define WIN32_LEAN_AND_MEAN +#define WIN32_LEAN_AND_MEAN 1 #include #include #include @@ -286,6 +286,7 @@ AudioClient::AudioClient() : _shouldEchoLocally(false), _shouldEchoToServer(false), _isNoiseGateEnabled(true), + _isAECEnabled(true), _reverb(false), _reverbOptions(&_scriptReverbOptions), _inputToNetworkResampler(NULL), @@ -302,6 +303,7 @@ AudioClient::AudioClient() : _isHeadsetPluggedIn(false), #endif _orientationGetter(DEFAULT_ORIENTATION_GETTER) { + // avoid putting a lock in the device callback assert(_localSamplesAvailable.is_lock_free()); @@ -353,6 +355,10 @@ AudioClient::AudioClient() : configureReverb(); +#if defined(WEBRTC_ENABLED) + configureWebrtc(); +#endif + auto nodeList = DependencyManager::get(); auto& packetReceiver = nodeList->getPacketReceiver(); packetReceiver.registerListener(PacketType::AudioStreamStats, &_stats, "processStreamStatsPacket"); @@ -1084,6 +1090,131 @@ void AudioClient::setReverbOptions(const AudioEffectOptions* options) { } } +#if defined(WEBRTC_ENABLED) + +static void deinterleaveToFloat(const int16_t* src, float* const* dst, int numFrames, int numChannels) { + for (int i = 0; i < numFrames; i++) { + for (int ch = 0; ch < numChannels; ch++) { + float f = *src++; + f *= (1/32768.0f); // scale + dst[ch][i] = f; // deinterleave + } + } +} + +static void interleaveToInt16(const float* const* src, int16_t* dst, int numFrames, int numChannels) { + for (int i = 0; i < numFrames; i++) { + for (int ch = 0; ch < numChannels; ch++) { + float f = src[ch][i]; + f *= 32768.0f; // scale + f += (f < 0.0f) ? -0.5f : 0.5f; // round + f = std::max(std::min(f, 32767.0f), -32768.0f); // saturate + *dst++ = (int16_t)f; // interleave + } + } +} + +void AudioClient::configureWebrtc() { + _apm = webrtc::AudioProcessingBuilder().Create(); + + webrtc::AudioProcessing::Config config; + + config.pre_amplifier.enabled = false; + config.high_pass_filter.enabled = false; + config.echo_canceller.enabled = true; + config.echo_canceller.mobile_mode = false; + config.echo_canceller.use_legacy_aec = false; + config.noise_suppression.enabled = false; + config.noise_suppression.level = webrtc::AudioProcessing::Config::NoiseSuppression::kModerate; + config.voice_detection.enabled = false; + config.gain_controller1.enabled = false; + config.gain_controller2.enabled = false; + config.gain_controller2.fixed_digital.gain_db = 0.0f; + config.gain_controller2.adaptive_digital.enabled = false; + config.residual_echo_detector.enabled = true; + config.level_estimation.enabled = false; + + _apm->ApplyConfig(config); +} + +// rebuffer into 10ms chunks +void AudioClient::processWebrtcFarEnd(const int16_t* samples, int numFrames, int numChannels, int sampleRate) { + + const webrtc::StreamConfig streamConfig = webrtc::StreamConfig(sampleRate, numChannels); + const int numChunk = (int)streamConfig.num_frames(); + + if (sampleRate > WEBRTC_SAMPLE_RATE_MAX) { + qCWarning(audioclient) << "WebRTC does not support" << sampleRate << "output sample rate."; + return; + } + if (numChannels > WEBRTC_CHANNELS_MAX) { + qCWarning(audioclient) << "WebRTC does not support" << numChannels << "output channels."; + return; + } + + while (numFrames > 0) { + + // number of frames to fill + int numFill = std::min(numFrames, numChunk - _numFifoFarEnd); + + // refill fifo + memcpy(&_fifoFarEnd[_numFifoFarEnd], samples, numFill * numChannels * sizeof(int16_t)); + samples += numFill * numChannels; + numFrames -= numFill; + _numFifoFarEnd += numFill; + + if (_numFifoFarEnd == numChunk) { + + // convert audio format + float buffer[WEBRTC_CHANNELS_MAX][WEBRTC_FRAMES_MAX]; + float* const buffers[WEBRTC_CHANNELS_MAX] = { buffer[0], buffer[1] }; + deinterleaveToFloat(_fifoFarEnd, buffers, numChunk, numChannels); + + // process one chunk + int error = _apm->ProcessReverseStream(buffers, streamConfig, streamConfig, buffers); + if (error != _apm->kNoError) { + qCWarning(audioclient) << "WebRTC ProcessReverseStream() returned ERROR:" << error; + } + _numFifoFarEnd = 0; + } + } +} + +void AudioClient::processWebrtcNearEnd(int16_t* samples, int numFrames, int numChannels, int sampleRate) { + + const webrtc::StreamConfig streamConfig = webrtc::StreamConfig(sampleRate, numChannels); + const int numChunk = (int)streamConfig.num_frames(); + + if (sampleRate > WEBRTC_SAMPLE_RATE_MAX) { + qCWarning(audioclient) << "WebRTC does not support" << sampleRate << "input sample rate."; + return; + } + if (numChannels > WEBRTC_CHANNELS_MAX) { + qCWarning(audioclient) << "WebRTC does not support" << numChannels << "input channels."; + return; + } + if (numFrames != numChunk) { + qCWarning(audioclient) << "WebRTC requires exactly 10ms of input."; + return; + } + + // convert audio format + float buffer[WEBRTC_CHANNELS_MAX][WEBRTC_FRAMES_MAX]; + float* const buffers[WEBRTC_CHANNELS_MAX] = { buffer[0], buffer[1] }; + deinterleaveToFloat(samples, buffers, numFrames, numChannels); + + // process one chunk + int error = _apm->ProcessStream(buffers, streamConfig, streamConfig, buffers); + if (error != _apm->kNoError) { + qCWarning(audioclient) << "WebRTC ProcessStream() returned ERROR:" << error; + } else { + // modify samples in-place + interleaveToInt16(buffers, samples, numFrames, numChannels); + } +} + +#endif // WEBRTC_ENABLED + void AudioClient::handleLocalEchoAndReverb(QByteArray& inputByteArray) { // If there is server echo, reverb will be applied to the recieved audio stream so no need to have it here. bool hasReverb = _reverb || _receivedAudioStream.hasReverb(); @@ -1262,6 +1393,13 @@ void AudioClient::handleMicAudioInput() { _inputRingBuffer.readSamples(inputAudioSamples.get(), inputSamplesRequired); +#if defined(WEBRTC_ENABLED) + if (_isAECEnabled) { + processWebrtcNearEnd(inputAudioSamples.get(), inputSamplesRequired / _inputFormat.channelCount(), + _inputFormat.channelCount(), _inputFormat.sampleRate()); + } +#endif + // detect loudness and clipping on the raw input bool isClipping = false; float loudness = computeLoudness(inputAudioSamples.get(), inputSamplesRequired, _inputFormat.channelCount(), isClipping); @@ -1574,6 +1712,15 @@ void AudioClient::setWarnWhenMuted(bool enable, bool emitSignal) { } } +void AudioClient::setAcousticEchoCancellation(bool enable, bool emitSignal) { + if (_isAECEnabled != enable) { + _isAECEnabled = enable; + if (emitSignal) { + emit acousticEchoCancellationChanged(_isAECEnabled); + } + } +} + bool AudioClient::setIsStereoInput(bool isStereoInput) { bool stereoInputChanged = false; if (isStereoInput != _isStereoInput && _inputDeviceInfo.supportedChannelCounts().contains(2)) { @@ -2107,15 +2254,16 @@ qint64 AudioClient::AudioOutputIODevice::readData(char * data, qint64 maxSize) { return maxSize; } - // samples requested from OUTPUT_CHANNEL_COUNT + // max samples requested from OUTPUT_CHANNEL_COUNT int deviceChannelCount = _audio->_outputFormat.channelCount(); - int samplesRequested = (int)(maxSize / AudioConstants::SAMPLE_SIZE) * OUTPUT_CHANNEL_COUNT / deviceChannelCount; + int maxSamplesRequested = (int)(maxSize / AudioConstants::SAMPLE_SIZE) * OUTPUT_CHANNEL_COUNT / deviceChannelCount; // restrict samplesRequested to the size of our mix/scratch buffers - samplesRequested = std::min(samplesRequested, _audio->_outputPeriod); + maxSamplesRequested = std::min(maxSamplesRequested, _audio->_outputPeriod); int16_t* scratchBuffer = _audio->_outputScratchBuffer; float* mixBuffer = _audio->_outputMixBuffer; + int samplesRequested = maxSamplesRequested; int networkSamplesPopped; if ((networkSamplesPopped = _receivedAudioStream.popSamples(samplesRequested, false)) > 0) { qCDebug(audiostream, "Read %d samples from buffer (%d available, %d requested)", networkSamplesPopped, _receivedAudioStream.getSamplesAvailable(), samplesRequested); @@ -2160,45 +2308,45 @@ qint64 AudioClient::AudioOutputIODevice::readData(char * data, qint64 maxSize) { }); int samplesPopped = std::max(networkSamplesPopped, injectorSamplesPopped); - int framesPopped = samplesPopped / AudioConstants::STEREO; - int bytesWritten; - if (samplesPopped > 0) { - - // apply output gain - float newGain = _audio->_outputGain.load(std::memory_order_acquire); - float oldGain = _audio->_lastOutputGain; - _audio->_lastOutputGain = newGain; - - applyGainSmoothing(mixBuffer, framesPopped, oldGain, newGain); - - if (deviceChannelCount == OUTPUT_CHANNEL_COUNT) { - // limit the audio - _audio->_audioLimiter.render(mixBuffer, (int16_t*)data, framesPopped); - } else { - _audio->_audioLimiter.render(mixBuffer, scratchBuffer, framesPopped); - - // upmix or downmix to deviceChannelCount - if (deviceChannelCount > OUTPUT_CHANNEL_COUNT) { - int extraChannels = deviceChannelCount - OUTPUT_CHANNEL_COUNT; - channelUpmix(scratchBuffer, (int16_t*)data, samplesPopped, extraChannels); - } else { - channelDownmix(scratchBuffer, (int16_t*)data, samplesPopped); - } - } - - bytesWritten = framesPopped * AudioConstants::SAMPLE_SIZE * deviceChannelCount; - assert(bytesWritten <= maxSize); - - } else { - // nothing on network, don't grab anything from injectors, and just return 0s - memset(data, 0, maxSize); - bytesWritten = maxSize; + if (samplesPopped == 0) { + // nothing on network, don't grab anything from injectors, and fill with silence + samplesPopped = maxSamplesRequested; + memset(mixBuffer, 0, samplesPopped * sizeof(float)); } + int framesPopped = samplesPopped / OUTPUT_CHANNEL_COUNT; + + // apply output gain + float newGain = _audio->_outputGain.load(std::memory_order_acquire); + float oldGain = _audio->_lastOutputGain; + _audio->_lastOutputGain = newGain; + + applyGainSmoothing(mixBuffer, framesPopped, oldGain, newGain); + + // limit the audio + _audio->_audioLimiter.render(mixBuffer, scratchBuffer, framesPopped); + +#if defined(WEBRTC_ENABLED) + if (_audio->_isAECEnabled) { + _audio->processWebrtcFarEnd(scratchBuffer, framesPopped, OUTPUT_CHANNEL_COUNT, _audio->_outputFormat.sampleRate()); + } +#endif + + // if required, upmix or downmix to deviceChannelCount + if (deviceChannelCount == OUTPUT_CHANNEL_COUNT) { + memcpy(data, scratchBuffer, samplesPopped * AudioConstants::SAMPLE_SIZE); + } else if (deviceChannelCount > OUTPUT_CHANNEL_COUNT) { + int extraChannels = deviceChannelCount - OUTPUT_CHANNEL_COUNT; + channelUpmix(scratchBuffer, (int16_t*)data, samplesPopped, extraChannels); + } else { + channelDownmix(scratchBuffer, (int16_t*)data, samplesPopped); + } + int bytesWritten = framesPopped * AudioConstants::SAMPLE_SIZE * deviceChannelCount; + assert(bytesWritten <= maxSize); // send output buffer for recording if (_audio->_isRecording) { Lock lock(_recordMutex); - _audio->_audioFileWav.addRawAudioChunk(reinterpret_cast(scratchBuffer), bytesWritten); + _audio->_audioFileWav.addRawAudioChunk(data, bytesWritten); } int bytesAudioOutputUnplayed = _audio->_audioOutput->bufferSize() - _audio->_audioOutput->bytesFree(); diff --git a/libraries/audio-client/src/AudioClient.h b/libraries/audio-client/src/AudioClient.h index decf0f7751..ab12393ebf 100644 --- a/libraries/audio-client/src/AudioClient.h +++ b/libraries/audio-client/src/AudioClient.h @@ -29,6 +29,7 @@ #include #include #include +#include #include #include @@ -215,6 +216,9 @@ public slots: void setWarnWhenMuted(bool isNoiseGateEnabled, bool emitSignal = true); bool isWarnWhenMutedEnabled() const { return _warnWhenMuted; } + void setAcousticEchoCancellation(bool isAECEnabled, bool emitSignal = true); + bool isAcousticEchoCancellationEnabled() const { return _isAECEnabled; } + virtual bool getLocalEcho() override { return _shouldEchoLocally; } virtual void setLocalEcho(bool localEcho) override { _shouldEchoLocally = localEcho; } virtual void toggleLocalEcho() override { _shouldEchoLocally = !_shouldEchoLocally; } @@ -256,6 +260,7 @@ signals: void muteToggled(bool muted); void noiseReductionChanged(bool noiseReductionEnabled); void warnWhenMutedChanged(bool warnWhenMutedEnabled); + void acousticEchoCancellationChanged(bool acousticEchoCancellationEnabled); void mutedByMixer(); void inputReceived(const QByteArray& inputSamples); void inputLoudnessChanged(float loudness, bool isClipping); @@ -377,6 +382,7 @@ private: bool _shouldEchoToServer; bool _isNoiseGateEnabled; bool _warnWhenMuted; + bool _isAECEnabled; bool _reverb; AudioEffectOptions _scriptReverbOptions; @@ -414,9 +420,23 @@ private: // Adds Reverb void configureReverb(); void updateReverbOptions(); - void handleLocalEchoAndReverb(QByteArray& inputByteArray); +#if defined(WEBRTC_ENABLED) + static const int WEBRTC_SAMPLE_RATE_MAX = 96000; + static const int WEBRTC_CHANNELS_MAX = 2; + static const int WEBRTC_FRAMES_MAX = webrtc::AudioProcessing::kChunkSizeMs * WEBRTC_SAMPLE_RATE_MAX / 1000; + + webrtc::AudioProcessing* _apm { nullptr }; + + int16_t _fifoFarEnd[WEBRTC_CHANNELS_MAX * WEBRTC_FRAMES_MAX] {}; + int _numFifoFarEnd = 0; // numFrames saved in fifo + + void configureWebrtc(); + void processWebrtcFarEnd(const int16_t* samples, int numFrames, int numChannels, int sampleRate); + void processWebrtcNearEnd(int16_t* samples, int numFrames, int numChannels, int sampleRate); +#endif + bool switchInputToAudioDevice(const QAudioDeviceInfo inputDeviceInfo, bool isShutdownRequest = false); bool switchOutputToAudioDevice(const QAudioDeviceInfo outputDeviceInfo, bool isShutdownRequest = false); diff --git a/libraries/shared/src/shared/WebRTC.h b/libraries/shared/src/shared/WebRTC.h new file mode 100644 index 0000000000..2f0e444bff --- /dev/null +++ b/libraries/shared/src/shared/WebRTC.h @@ -0,0 +1,36 @@ +// +// WebRTC.h +// libraries/shared/src/shared/ +// +// Copyright 2019 High Fidelity, Inc. +// +// Distributed under the Apache License, Version 2.0. +// See the accompanying file LICENSE or http://www.apache.org/licenses/LICENSE-2.0.html +// + +#ifndef hifi_WebRTC_h +#define hifi_WebRTC_h + +#if defined(Q_OS_MAC) +# define WEBRTC_ENABLED 1 +# define WEBRTC_POSIX 1 +#elif defined(Q_OS_WIN) +# define WEBRTC_ENABLED 1 +# define WEBRTC_WIN 1 +# define NOMINMAX 1 +# define WIN32_LEAN_AND_MEAN 1 +#elif defined(Q_OS_ANDROID) +// I don't yet have a working libwebrtc for android +// # define WEBRTC_ENABLED 1 +// # define WEBRTC_POSIX 1 +#elif defined(Q_OS_LINUX) +# define WEBRTC_ENABLED 1 +# define WEBRTC_POSIX 1 +#endif + +#if defined(WEBRTC_ENABLED) +# include +# include "modules/audio_processing/audio_processing_impl.h" +#endif + +#endif // hifi_WebRTC_h