Merge pull request #15936 from sethalves/webrtc

acoustic echo cancellation

cmake/macros/TargetWebRTC.cmake

@@ -0,0 +1,24 @@
+#
+#  Copyright 2019 High Fidelity, Inc.
+#
+#  Distributed under the Apache License, Version 2.0.
+#  See the accompanying file LICENSE or http://www.apache.org/licenses/LICENSE-2.0.html
+#
+macro(TARGET_WEBRTC)
+    if (ANDROID)
+        # I don't yet have working libwebrtc for android
+        # include(SelectLibraryConfigurations)
+        # set(INSTALL_DIR ${HIFI_ANDROID_PRECOMPILED}/webrtc/webrtc)
+        # set(WEBRTC_INCLUDE_DIRS "${INSTALL_DIR}/include/webrtc")
+        # set(WEBRTC_LIBRARY_DEBUG ${INSTALL_DIR}/debug/lib/libwebrtc.a)
+        # set(WEBRTC_LIBRARY_RELEASE ${INSTALL_DIR}/lib/libwebrtc.a)
+        # select_library_configurations(WEBRTC)
+    else()
+        set(WEBRTC_INCLUDE_DIRS "${VCPKG_INSTALL_ROOT}/include/webrtc")
+        find_library(WEBRTC_LIBRARY NAMES webrtc PATHS ${VCPKG_INSTALL_ROOT}/lib/ NO_DEFAULT_PATH)
+        target_include_directories(${TARGET_NAME} SYSTEM PUBLIC ${WEBRTC_INCLUDE_DIRS})
+        target_link_libraries(${TARGET_NAME} ${WEBRTC_LIBRARY})
+    endif()
+
+
+endmacro()

cmake/ports/hifi-deps/CONTROL

@@ -1,4 +1,4 @@
 Source: hifi-deps
 Version: 0.1
 Description: Collected dependencies for High Fidelity applications
-Build-Depends: bullet3, draco, etc2comp, glm, nvtt, openexr (!android), openssl (windows), tbb (!android&!osx), zlib
+Build-Depends: bullet3, draco, etc2comp, glm, nvtt, openexr (!android), openssl (windows), tbb (!android&!osx), zlib, webrtc (!android)

cmake/ports/webrtc/CONTROL

@@ -0,0 +1,3 @@
+Source: webrtc
+Version: 20190626
+Description: WebRTC

cmake/ports/webrtc/portfile.cmake

@@ -0,0 +1,36 @@
+include(vcpkg_common_functions)
+set(WEBRTC_VERSION 20190626)
+set(MASTER_COPY_SOURCE_PATH ${CURRENT_BUILDTREES_DIR}/src)
+
+if (ANDROID)
+   # this is handled by hifi_android.py
+elseif (WIN32)
+    vcpkg_download_distfile(
+        WEBRTC_SOURCE_ARCHIVE
+        URLS https://hifi-public.s3.amazonaws.com/seth/webrtc-20190626-windows.zip
+        SHA512 c0848eddb1579b3bb0496b8785e24f30470f3c477145035fd729264a326a467b9467ae9f426aa5d72d168ad9e9bf2c279150744832736bdf39064d24b04de1a3
+        FILENAME webrtc-20190626-windows.zip
+    )
+elseif (APPLE)
+    vcpkg_download_distfile(
+        WEBRTC_SOURCE_ARCHIVE
+        URLS https://hifi-public.s3.amazonaws.com/seth/webrtc-20190626-osx.tar.gz
+        SHA512 fc70cec1b5ee87395137b7090f424e2fc2300fc17d744d5ffa1cf7aa0e0f1a069a9d72ba1ad2fb4a640ebeb6c218bda24351ba0083e1ff96c4a4b5032648a9d2
+        FILENAME webrtc-20190626-osx.tar.gz
+    )
+else ()
+    # else Linux desktop
+    vcpkg_download_distfile(
+        WEBRTC_SOURCE_ARCHIVE
+        URLS https://hifi-public.s3.amazonaws.com/seth/webrtc-20190626-linux.tar.gz
+        SHA512 07d7776551aa78cb09a3ef088a8dee7762735c168c243053b262083d90a1d258cec66dc386f6903da5c4461921a3c2db157a1ee106a2b47e7756cb424b66cc43
+        FILENAME webrtc-20190626-linux.tar.gz
+    )
+endif ()
+
+vcpkg_extract_source_archive(${WEBRTC_SOURCE_ARCHIVE})
+
+file(COPY ${MASTER_COPY_SOURCE_PATH}/webrtc/include DESTINATION ${CURRENT_PACKAGES_DIR})
+file(COPY ${MASTER_COPY_SOURCE_PATH}/webrtc/lib DESTINATION ${CURRENT_PACKAGES_DIR})
+file(COPY ${MASTER_COPY_SOURCE_PATH}/webrtc/share DESTINATION ${CURRENT_PACKAGES_DIR})
+file(COPY ${MASTER_COPY_SOURCE_PATH}/webrtc/debug DESTINATION ${CURRENT_PACKAGES_DIR})

hifi_android.py

@@ -94,6 +94,10 @@ ANDROID_PACKAGES = {
         'checksum': 'ddcb23df336b08017042ba4786db1d9e',
         'sharedLibFolder': 'lib',
         'includeLibs': {'libbreakpad_client.a'}
+    },
+    'webrtc': {
+        'file': 'webrtc-20190626-android.tar.gz',
+        'checksum': 'e2dccd3d8efdcba6d428c87ba7fb2a53'
     }
 }
 

interface/resources/qml/hifi/audio/Audio.qml

@@ -166,16 +166,16 @@ Rectangle {
             x: 2 * margins.paddings;
             width: parent.width;
             // switch heights + 2 * top margins
-            height: (root.switchHeight) * 3 + 48;
+            height: (root.switchHeight) * 6 + 48;
             anchors.top: firstSeparator.bottom;
             anchors.topMargin: 10;
 
-            // mute is in its own row
            Item {
                 id: switchContainer;
                 x: margins.paddings;
                 width: parent.width / 2;
                 height: parent.height;
+                anchors.top: parent.top
                 anchors.left: parent.left;
                 HifiControlsUit.Switch {
                     id: muteMic;
@@ -222,12 +222,29 @@ Rectangle {
                 }
 
                 HifiControlsUit.Switch {
-                    id: pttSwitch
+                    id: acousticEchoCancellationSwitch;
                     height: root.switchHeight;
                     switchWidth: root.switchWidth;
                     anchors.top: noiseReductionSwitch.bottom
                     anchors.topMargin: 24
                     anchors.left: parent.left
+                    labelTextOn: "Echo Cancellation";
+                    labelTextSize: 16;
+                    backgroundOnColor: "#E3E3E3";
+                    checked: AudioScriptingInterface.acousticEchoCancellation;
+                    onCheckedChanged: {
+                        AudioScriptingInterface.acousticEchoCancellation = checked;
+                        checked = Qt.binding(function() { return AudioScriptingInterface.acousticEchoCancellation; });
+                    }
+                }
+
+                HifiControlsUit.Switch {
+                    id: pttSwitch
+                    height: root.switchHeight;
+                    switchWidth: root.switchWidth;
+                    anchors.top: acousticEchoCancellationSwitch.bottom;
+                    anchors.topMargin: 24
+                    anchors.left: parent.left
                     labelTextOn: (bar.currentIndex === 0) ? qsTr("Push To Talk (T)") : qsTr("Push To Talk");
                     labelTextSize: 16;
                     backgroundOnColor: "#E3E3E3";
@@ -298,7 +315,6 @@ Rectangle {
                         checked = Qt.binding(function() { return AudioScriptingInterface.isStereoInput; }); // restore binding
                     }
                 }
-
             }
         }
 

interface/resources/qml/hifi/simplifiedUI/settingsApp/audio/Audio.qml

@@ -222,6 +222,17 @@ Flickable {
                         }
                     }
                 }
+
+                SimplifiedControls.Switch {
+                    id: acousticEchoCancellationSwitch
+                    Layout.preferredHeight: 18
+                    Layout.preferredWidth: parent.width
+                    labelTextOn: "Acoustic Echo Cancellation"
+                    checked: AudioScriptingInterface.acousticEchoCancellation
+                    onClicked: {
+                        AudioScriptingInterface.acousticEchoCancellation = !AudioScriptingInterface.acousticEchoCancellation;
+                    }
+                }
             }
         }
 

interface/src/scripting/Audio.cpp

@@ -26,6 +26,7 @@ QString Audio::HMD { "VR" };
 
 Setting::Handle<bool> enableNoiseReductionSetting { QStringList { Audio::AUDIO, "NoiseReduction" }, true };
 Setting::Handle<bool> enableWarnWhenMutedSetting { QStringList { Audio::AUDIO, "WarnWhenMuted" }, true };
+Setting::Handle<bool> enableAcousticEchoCancellationSetting { QStringList { Audio::AUDIO, "AcousticEchoCancellation" }, true };
 
 
 float Audio::loudnessToLevel(float loudness) {
@@ -40,12 +41,14 @@ Audio::Audio() : _devices(_contextIsHMD) {
     connect(client, &AudioClient::muteToggled, this, &Audio::setMuted);
     connect(client, &AudioClient::noiseReductionChanged, this, &Audio::enableNoiseReduction);
     connect(client, &AudioClient::warnWhenMutedChanged, this, &Audio::enableWarnWhenMuted);
+    connect(client, &AudioClient::acousticEchoCancellationChanged, this, &Audio::enableAcousticEchoCancellation);
     connect(client, &AudioClient::inputLoudnessChanged, this, &Audio::onInputLoudnessChanged);
     connect(client, &AudioClient::inputVolumeChanged, this, &Audio::setInputVolume);
     connect(this, &Audio::contextChanged, &_devices, &AudioDevices::onContextChanged);
     connect(this, &Audio::pushingToTalkChanged, this, &Audio::handlePushedToTalk);
     enableNoiseReduction(enableNoiseReductionSetting.get());
     enableWarnWhenMuted(enableWarnWhenMutedSetting.get());
+    enableAcousticEchoCancellation(enableAcousticEchoCancellationSetting.get());
     onContextChanged();
 }
 
@@ -277,6 +280,28 @@ void Audio::enableWarnWhenMuted(bool enable) {
     }
 }
 
+bool Audio::acousticEchoCancellationEnabled() const {
+    return resultWithReadLock<bool>([&] {
+        return _enableAcousticEchoCancellation;
+    });
+}
+
+void Audio::enableAcousticEchoCancellation(bool enable) {
+    bool changed = false;
+    withWriteLock([&] {
+        if (_enableAcousticEchoCancellation != enable) {
+            _enableAcousticEchoCancellation = enable;
+            auto client = DependencyManager::get<AudioClient>().data();
+            QMetaObject::invokeMethod(client, "setAcousticEchoCancellation", Q_ARG(bool, enable), Q_ARG(bool, false));
+            enableAcousticEchoCancellationSetting.set(enable);
+            changed = true;
+        }
+    });
+    if (changed) {
+        emit acousticEchoCancellationChanged(enable);
+    }
+}
+
 float Audio::getInputVolume() const {
     return resultWithReadLock<bool>([&] {
         return _inputVolume;

interface/src/scripting/Audio.h

@@ -72,6 +72,9 @@ class Audio : public AudioScriptingInterface, protected ReadWriteLockable {
      * @property {number} systemInjectorGain - The gain (relative volume) that system sounds are played at.
      * @property {number} pushingToTalkOutputGainDesktop - The gain (relative volume) that all sounds are played at when the user is holding
      *     the push-to-talk key in Desktop mode.
+     * @property {boolean} acousticEchoCancellation - <code>true</code> if audio-echo-cancellation is enabled, otherwise
+     *     <code>false</code>. When enabled, sound from the audio output will be suppressed when it echos back to the
+     *     input audio signal.
      *
      * @comment The following properties are from AudioScriptingInterface.h.
      * @property {boolean} isStereoInput - <code>true</code> if the input audio is being used in stereo, otherwise
@@ -85,6 +88,8 @@ class Audio : public AudioScriptingInterface, protected ReadWriteLockable {
     Q_PROPERTY(bool muted READ isMuted WRITE setMuted NOTIFY mutedChanged)
     Q_PROPERTY(bool noiseReduction READ noiseReductionEnabled WRITE enableNoiseReduction NOTIFY noiseReductionChanged)
     Q_PROPERTY(bool warnWhenMuted READ warnWhenMutedEnabled WRITE enableWarnWhenMuted NOTIFY warnWhenMutedChanged)
+    Q_PROPERTY(bool acousticEchoCancellation
+               READ acousticEchoCancellationEnabled WRITE enableAcousticEchoCancellation NOTIFY acousticEchoCancellationChanged)
     Q_PROPERTY(float inputVolume READ getInputVolume WRITE setInputVolume NOTIFY inputVolumeChanged)
     Q_PROPERTY(float inputLevel READ getInputLevel NOTIFY inputLevelChanged)
     Q_PROPERTY(bool clipping READ isClipping NOTIFY clippingChanged)
@@ -115,6 +120,7 @@ public:
     bool isMuted() const;
     bool noiseReductionEnabled() const;
     bool warnWhenMutedEnabled() const;
+    bool acousticEchoCancellationEnabled() const;
     float getInputVolume() const;
     float getInputLevel() const;
     bool isClipping() const;
@@ -396,6 +402,14 @@ signals:
      */
     void warnWhenMutedChanged(bool isEnabled);
 
+    /**jsdoc
+     * Triggered when acoustic echo cancellation is enabled or disabled.
+     * @function Audio.acousticEchoCancellationChanged
+     * @param {boolean} isEnabled - <code>true</code> if acoustic echo cancellation is enabled, otherwise <code>false</code>.
+     * @returns {Signal}
+     */
+    void acousticEchoCancellationChanged(bool isEnabled);
+
     /**jsdoc
      * Triggered when the input audio volume changes.
      * @function Audio.inputVolumeChanged
@@ -494,6 +508,7 @@ private slots:
     void setMuted(bool muted);
     void enableNoiseReduction(bool enable);
     void enableWarnWhenMuted(bool enable);
+    void enableAcousticEchoCancellation(bool enable);
     void setInputVolume(float volume);
     void onInputLoudnessChanged(float loudness, bool isClipping);
 
@@ -512,6 +527,7 @@ private:
     bool _isClipping { false };
     bool _enableNoiseReduction { true };  // Match default value of AudioClient::_isNoiseGateEnabled.
     bool _enableWarnWhenMuted { true };
+    bool _enableAcousticEchoCancellation { true }; // AudioClient::_isAECEnabled
     bool _contextIsHMD { false };
     AudioDevices* getDevices() { return &_devices; }
     AudioDevices _devices;

libraries/audio-client/CMakeLists.txt

@@ -7,6 +7,11 @@ link_hifi_libraries(audio plugins)
 include_hifi_library_headers(shared)
 include_hifi_library_headers(networking)
 
+if (ANDROID)
+else ()
+  target_webrtc()
+endif ()
+
 # append audio includes to our list of includes to bubble
 target_include_directories(${TARGET_NAME} PUBLIC "${HIFI_LIBRARY_DIR}/audio/src")
 

libraries/audio-client/src/AudioClient.cpp

@@ -24,7 +24,7 @@
 #endif
 
 #ifdef WIN32
-#define WIN32_LEAN_AND_MEAN
+#define WIN32_LEAN_AND_MEAN 1
 #include <windows.h>
 #include <Mmsystem.h>
 #include <mmdeviceapi.h>
@@ -286,6 +286,7 @@ AudioClient::AudioClient() :
     _shouldEchoLocally(false),
     _shouldEchoToServer(false),
     _isNoiseGateEnabled(true),
+    _isAECEnabled(true),
     _reverb(false),
     _reverbOptions(&_scriptReverbOptions),
     _inputToNetworkResampler(NULL),
@@ -302,6 +303,7 @@ AudioClient::AudioClient() :
     _isHeadsetPluggedIn(false),
 #endif
     _orientationGetter(DEFAULT_ORIENTATION_GETTER) {
+
     // avoid putting a lock in the device callback
     assert(_localSamplesAvailable.is_lock_free());
 
@@ -353,6 +355,10 @@ AudioClient::AudioClient() :
 
     configureReverb();
 
+#if defined(WEBRTC_ENABLED)
+    configureWebrtc();
+#endif
+
     auto nodeList = DependencyManager::get<NodeList>();
     auto& packetReceiver = nodeList->getPacketReceiver();
     packetReceiver.registerListener(PacketType::AudioStreamStats, &_stats, "processStreamStatsPacket");
@@ -1084,6 +1090,131 @@ void AudioClient::setReverbOptions(const AudioEffectOptions* options) {
     }
 }
 
+#if defined(WEBRTC_ENABLED)
+
+static void deinterleaveToFloat(const int16_t* src, float* const* dst, int numFrames, int numChannels) {
+    for (int i = 0; i < numFrames; i++) {
+        for (int ch = 0; ch < numChannels; ch++) {
+            float f = *src++;
+            f *= (1/32768.0f);  // scale
+            dst[ch][i] = f;     // deinterleave
+        }
+    }
+}
+
+static void interleaveToInt16(const float* const* src, int16_t* dst, int numFrames, int numChannels) {
+    for (int i = 0; i < numFrames; i++) {
+        for (int ch = 0; ch < numChannels; ch++) {
+            float f = src[ch][i];
+            f *= 32768.0f;                                  // scale
+            f += (f < 0.0f) ? -0.5f : 0.5f;                 // round
+            f = std::max(std::min(f, 32767.0f), -32768.0f); // saturate
+            *dst++ = (int16_t)f;                            // interleave
+        }
+    }
+}
+
+void AudioClient::configureWebrtc() {
+    _apm = webrtc::AudioProcessingBuilder().Create();
+
+    webrtc::AudioProcessing::Config config;
+
+    config.pre_amplifier.enabled = false;
+    config.high_pass_filter.enabled = false;
+    config.echo_canceller.enabled = true;
+    config.echo_canceller.mobile_mode = false;
+    config.echo_canceller.use_legacy_aec = false;
+    config.noise_suppression.enabled = false;
+    config.noise_suppression.level = webrtc::AudioProcessing::Config::NoiseSuppression::kModerate;
+    config.voice_detection.enabled = false;
+    config.gain_controller1.enabled = false;
+    config.gain_controller2.enabled = false;
+    config.gain_controller2.fixed_digital.gain_db = 0.0f;
+    config.gain_controller2.adaptive_digital.enabled = false;
+    config.residual_echo_detector.enabled = true;
+    config.level_estimation.enabled = false;
+
+    _apm->ApplyConfig(config);
+}
+
+// rebuffer into 10ms chunks
+void AudioClient::processWebrtcFarEnd(const int16_t* samples, int numFrames, int numChannels, int sampleRate) {
+
+    const webrtc::StreamConfig streamConfig = webrtc::StreamConfig(sampleRate, numChannels);
+    const int numChunk = (int)streamConfig.num_frames();
+
+    if (sampleRate > WEBRTC_SAMPLE_RATE_MAX) {
+        qCWarning(audioclient) << "WebRTC does not support" << sampleRate << "output sample rate.";
+        return;
+    }
+    if (numChannels > WEBRTC_CHANNELS_MAX) {
+        qCWarning(audioclient) << "WebRTC does not support" << numChannels << "output channels.";
+        return;
+    }
+
+    while (numFrames > 0) {
+
+        // number of frames to fill
+        int numFill = std::min(numFrames, numChunk - _numFifoFarEnd);
+
+        // refill fifo
+        memcpy(&_fifoFarEnd[_numFifoFarEnd], samples, numFill * numChannels * sizeof(int16_t));
+        samples += numFill * numChannels;
+        numFrames -= numFill;
+        _numFifoFarEnd += numFill;
+
+        if (_numFifoFarEnd == numChunk) {
+
+            // convert audio format
+            float buffer[WEBRTC_CHANNELS_MAX][WEBRTC_FRAMES_MAX];
+            float* const buffers[WEBRTC_CHANNELS_MAX] = { buffer[0], buffer[1] };
+            deinterleaveToFloat(_fifoFarEnd, buffers, numChunk, numChannels);
+
+            // process one chunk
+            int error = _apm->ProcessReverseStream(buffers, streamConfig, streamConfig, buffers);
+            if (error != _apm->kNoError) {
+                qCWarning(audioclient) << "WebRTC ProcessReverseStream() returned ERROR:" << error;
+            }
+            _numFifoFarEnd = 0;
+        }
+    }
+}
+
+void AudioClient::processWebrtcNearEnd(int16_t* samples, int numFrames, int numChannels, int sampleRate) {
+
+    const webrtc::StreamConfig streamConfig = webrtc::StreamConfig(sampleRate, numChannels);
+    const int numChunk = (int)streamConfig.num_frames();
+
+    if (sampleRate > WEBRTC_SAMPLE_RATE_MAX) {
+        qCWarning(audioclient) << "WebRTC does not support" << sampleRate << "input sample rate.";
+        return;
+    }
+    if (numChannels > WEBRTC_CHANNELS_MAX) {
+        qCWarning(audioclient) << "WebRTC does not support" << numChannels << "input channels.";
+        return;
+    }
+    if (numFrames != numChunk) {
+        qCWarning(audioclient) << "WebRTC requires exactly 10ms of input.";
+        return;
+    }
+
+    // convert audio format
+    float buffer[WEBRTC_CHANNELS_MAX][WEBRTC_FRAMES_MAX];
+    float* const buffers[WEBRTC_CHANNELS_MAX] = { buffer[0], buffer[1] };
+    deinterleaveToFloat(samples, buffers, numFrames, numChannels);
+
+    // process one chunk
+    int error = _apm->ProcessStream(buffers, streamConfig, streamConfig, buffers);
+    if (error != _apm->kNoError) {
+        qCWarning(audioclient) << "WebRTC ProcessStream() returned ERROR:" << error;
+    } else {
+        // modify samples in-place
+        interleaveToInt16(buffers, samples, numFrames, numChannels);
+    }
+}
+
+#endif // WEBRTC_ENABLED
+
 void AudioClient::handleLocalEchoAndReverb(QByteArray& inputByteArray) {
     // If there is server echo, reverb will be applied to the recieved audio stream so no need to have it here.
     bool hasReverb = _reverb || _receivedAudioStream.hasReverb();
@@ -1262,6 +1393,13 @@ void AudioClient::handleMicAudioInput() {
 
         _inputRingBuffer.readSamples(inputAudioSamples.get(), inputSamplesRequired);
 
+#if defined(WEBRTC_ENABLED)
+        if (_isAECEnabled) {
+            processWebrtcNearEnd(inputAudioSamples.get(), inputSamplesRequired / _inputFormat.channelCount(),
+                                 _inputFormat.channelCount(), _inputFormat.sampleRate());
+        }
+#endif
+
         // detect loudness and clipping on the raw input
         bool isClipping = false;
         float loudness = computeLoudness(inputAudioSamples.get(), inputSamplesRequired, _inputFormat.channelCount(), isClipping);
@@ -1574,6 +1712,15 @@ void AudioClient::setWarnWhenMuted(bool enable, bool emitSignal) {
     }
 }
 
+void AudioClient::setAcousticEchoCancellation(bool enable, bool emitSignal) {
+    if (_isAECEnabled != enable) {
+        _isAECEnabled = enable;
+        if (emitSignal) {
+            emit acousticEchoCancellationChanged(_isAECEnabled);
+        }
+    }
+}
+
 bool AudioClient::setIsStereoInput(bool isStereoInput) {
     bool stereoInputChanged = false;
     if (isStereoInput != _isStereoInput && _inputDeviceInfo.supportedChannelCounts().contains(2)) {
@@ -2107,15 +2254,16 @@ qint64 AudioClient::AudioOutputIODevice::readData(char * data, qint64 maxSize) {
         return maxSize;
     }
 
-    // samples requested from OUTPUT_CHANNEL_COUNT
+    // max samples requested from OUTPUT_CHANNEL_COUNT
     int deviceChannelCount = _audio->_outputFormat.channelCount();
-    int samplesRequested = (int)(maxSize / AudioConstants::SAMPLE_SIZE) * OUTPUT_CHANNEL_COUNT / deviceChannelCount;
+    int maxSamplesRequested = (int)(maxSize / AudioConstants::SAMPLE_SIZE) * OUTPUT_CHANNEL_COUNT / deviceChannelCount;
     // restrict samplesRequested to the size of our mix/scratch buffers
-    samplesRequested = std::min(samplesRequested, _audio->_outputPeriod);
+    maxSamplesRequested = std::min(maxSamplesRequested, _audio->_outputPeriod);
 
     int16_t* scratchBuffer = _audio->_outputScratchBuffer;
     float* mixBuffer = _audio->_outputMixBuffer;
 
+    int samplesRequested = maxSamplesRequested;
     int networkSamplesPopped;
     if ((networkSamplesPopped = _receivedAudioStream.popSamples(samplesRequested, false)) > 0) {
         qCDebug(audiostream, "Read %d samples from buffer (%d available, %d requested)", networkSamplesPopped, _receivedAudioStream.getSamplesAvailable(), samplesRequested);
@@ -2160,9 +2308,12 @@ qint64 AudioClient::AudioOutputIODevice::readData(char * data, qint64 maxSize) {
     });
 
     int samplesPopped = std::max(networkSamplesPopped, injectorSamplesPopped);
-    int framesPopped = samplesPopped / AudioConstants::STEREO;
+    if (samplesPopped == 0) {
-    int bytesWritten;
+        // nothing on network, don't grab anything from injectors, and fill with silence
-    if (samplesPopped > 0) {
+        samplesPopped = maxSamplesRequested;
+        memset(mixBuffer, 0, samplesPopped * sizeof(float));
+    }
+    int framesPopped = samplesPopped / OUTPUT_CHANNEL_COUNT;
 
     // apply output gain
     float newGain = _audio->_outputGain.load(std::memory_order_acquire);
@@ -2171,34 +2322,31 @@ qint64 AudioClient::AudioOutputIODevice::readData(char * data, qint64 maxSize) {
 
     applyGainSmoothing<OUTPUT_CHANNEL_COUNT>(mixBuffer, framesPopped, oldGain, newGain);
 
-        if (deviceChannelCount == OUTPUT_CHANNEL_COUNT) {
     // limit the audio
-            _audio->_audioLimiter.render(mixBuffer, (int16_t*)data, framesPopped);
-        } else {
     _audio->_audioLimiter.render(mixBuffer, scratchBuffer, framesPopped);
 
-            // upmix or downmix to deviceChannelCount
+#if defined(WEBRTC_ENABLED)
-            if (deviceChannelCount > OUTPUT_CHANNEL_COUNT) {
+    if (_audio->_isAECEnabled) {
+        _audio->processWebrtcFarEnd(scratchBuffer, framesPopped, OUTPUT_CHANNEL_COUNT, _audio->_outputFormat.sampleRate());
+    }
+#endif
+
+    // if required, upmix or downmix to deviceChannelCount
+    if (deviceChannelCount == OUTPUT_CHANNEL_COUNT) {
+        memcpy(data, scratchBuffer, samplesPopped * AudioConstants::SAMPLE_SIZE);
+    } else if (deviceChannelCount > OUTPUT_CHANNEL_COUNT) {
         int extraChannels = deviceChannelCount - OUTPUT_CHANNEL_COUNT;
         channelUpmix(scratchBuffer, (int16_t*)data, samplesPopped, extraChannels);
     } else {
         channelDownmix(scratchBuffer, (int16_t*)data, samplesPopped);
     }
-        }
+    int bytesWritten = framesPopped * AudioConstants::SAMPLE_SIZE * deviceChannelCount;
-
-        bytesWritten = framesPopped * AudioConstants::SAMPLE_SIZE * deviceChannelCount;
     assert(bytesWritten <= maxSize);
 
-    } else {
-        // nothing on network, don't grab anything from injectors, and just return 0s
-        memset(data, 0, maxSize);
-        bytesWritten = maxSize;
-    }
-
     // send output buffer for recording
     if (_audio->_isRecording) {
         Lock lock(_recordMutex);
-        _audio->_audioFileWav.addRawAudioChunk(reinterpret_cast<char*>(scratchBuffer), bytesWritten);
+        _audio->_audioFileWav.addRawAudioChunk(data, bytesWritten);
     }
 
     int bytesAudioOutputUnplayed = _audio->_audioOutput->bufferSize() - _audio->_audioOutput->bytesFree();

libraries/audio-client/src/AudioClient.h

@@ -29,6 +29,7 @@
 #include <AbstractAudioInterface.h>
 #include <AudioEffectOptions.h>
 #include <AudioStreamStats.h>
+#include <shared/WebRTC.h>
 
 #include <DependencyManager.h>
 #include <HifiSockAddr.h>
@@ -215,6 +216,9 @@ public slots:
     void setWarnWhenMuted(bool isNoiseGateEnabled, bool emitSignal = true);
     bool isWarnWhenMutedEnabled() const { return _warnWhenMuted; }
 
+    void setAcousticEchoCancellation(bool isAECEnabled, bool emitSignal = true);
+    bool isAcousticEchoCancellationEnabled() const { return _isAECEnabled; }
+
     virtual bool getLocalEcho() override { return _shouldEchoLocally; }
     virtual void setLocalEcho(bool localEcho) override { _shouldEchoLocally = localEcho; }
     virtual void toggleLocalEcho() override { _shouldEchoLocally = !_shouldEchoLocally; }
@@ -256,6 +260,7 @@ signals:
     void muteToggled(bool muted);
     void noiseReductionChanged(bool noiseReductionEnabled);
     void warnWhenMutedChanged(bool warnWhenMutedEnabled);
+    void acousticEchoCancellationChanged(bool acousticEchoCancellationEnabled);
     void mutedByMixer();
     void inputReceived(const QByteArray& inputSamples);
     void inputLoudnessChanged(float loudness, bool isClipping);
@@ -377,6 +382,7 @@ private:
     bool _shouldEchoToServer;
     bool _isNoiseGateEnabled;
     bool _warnWhenMuted;
+    bool _isAECEnabled;
 
     bool _reverb;
     AudioEffectOptions _scriptReverbOptions;
@@ -414,9 +420,23 @@ private:
     // Adds Reverb
     void configureReverb();
     void updateReverbOptions();
-
     void handleLocalEchoAndReverb(QByteArray& inputByteArray);
 
+#if defined(WEBRTC_ENABLED)
+    static const int WEBRTC_SAMPLE_RATE_MAX = 96000;
+    static const int WEBRTC_CHANNELS_MAX = 2;
+    static const int WEBRTC_FRAMES_MAX = webrtc::AudioProcessing::kChunkSizeMs * WEBRTC_SAMPLE_RATE_MAX / 1000;
+
+    webrtc::AudioProcessing* _apm { nullptr };
+
+    int16_t _fifoFarEnd[WEBRTC_CHANNELS_MAX * WEBRTC_FRAMES_MAX] {};
+    int _numFifoFarEnd = 0; // numFrames saved in fifo
+
+    void configureWebrtc();
+    void processWebrtcFarEnd(const int16_t* samples, int numFrames, int numChannels, int sampleRate);
+    void processWebrtcNearEnd(int16_t* samples, int numFrames, int numChannels, int sampleRate);
+#endif
+
     bool switchInputToAudioDevice(const QAudioDeviceInfo inputDeviceInfo, bool isShutdownRequest = false);
     bool switchOutputToAudioDevice(const QAudioDeviceInfo outputDeviceInfo, bool isShutdownRequest = false);
 

libraries/shared/src/shared/WebRTC.h

@@ -0,0 +1,36 @@
+//
+//  WebRTC.h
+//  libraries/shared/src/shared/
+//
+//  Copyright 2019 High Fidelity, Inc.
+//
+//  Distributed under the Apache License, Version 2.0.
+//  See the accompanying file LICENSE or http://www.apache.org/licenses/LICENSE-2.0.html
+//
+
+#ifndef hifi_WebRTC_h
+#define hifi_WebRTC_h
+
+#if defined(Q_OS_MAC)
+#  define WEBRTC_ENABLED 1
+#  define WEBRTC_POSIX 1
+#elif defined(Q_OS_WIN)
+#  define WEBRTC_ENABLED 1
+#  define WEBRTC_WIN 1
+#  define NOMINMAX 1
+#  define WIN32_LEAN_AND_MEAN 1
+#elif defined(Q_OS_ANDROID)
+// I don't yet have a working libwebrtc for android
+// #  define WEBRTC_ENABLED 1
+// #  define WEBRTC_POSIX 1
+#elif defined(Q_OS_LINUX)
+#  define WEBRTC_ENABLED 1
+#  define WEBRTC_POSIX 1
+#endif
+
+#if defined(WEBRTC_ENABLED)
+#  include <modules/audio_processing/include/audio_processing.h>
+#  include "modules/audio_processing/audio_processing_impl.h"
+#endif
+
+#endif // hifi_WebRTC_h