From 32d5ab7b1f0ce18cf4def40bf61c8b946bd48a25 Mon Sep 17 00:00:00 2001 From: Seth Alves Date: Thu, 27 Jun 2019 08:43:59 -0700 Subject: [PATCH 01/13] Make WebRTC library available --- cmake/macros/TargetWebRTC.cmake | 16 ++++++++++ cmake/ports/hifi-deps/CONTROL | 2 +- cmake/ports/webrtc/CONTROL | 3 ++ cmake/ports/webrtc/portfile.cmake | 35 ++++++++++++++++++++++ libraries/audio-client/CMakeLists.txt | 2 ++ libraries/audio-client/src/AudioClient.cpp | 9 +++++- libraries/audio-client/src/AudioClient.h | 5 ++++ libraries/shared/src/shared/WebRTC.h | 34 +++++++++++++++++++++ 8 files changed, 104 insertions(+), 2 deletions(-) create mode 100644 cmake/macros/TargetWebRTC.cmake create mode 100644 cmake/ports/webrtc/CONTROL create mode 100644 cmake/ports/webrtc/portfile.cmake create mode 100644 libraries/shared/src/shared/WebRTC.h diff --git a/cmake/macros/TargetWebRTC.cmake b/cmake/macros/TargetWebRTC.cmake new file mode 100644 index 0000000000..e117011101 --- /dev/null +++ b/cmake/macros/TargetWebRTC.cmake @@ -0,0 +1,16 @@ +# +# Copyright 2019 High Fidelity, Inc. +# +# Distributed under the Apache License, Version 2.0. +# See the accompanying file LICENSE or http://www.apache.org/licenses/LICENSE-2.0.html +# +macro(TARGET_WEBRTC) + +if (ANDROID) +else() + target_include_directories(${TARGET_NAME} SYSTEM PUBLIC "${VCPKG_INSTALL_ROOT}/include/webrtc") + find_library(WEBRTC_LIBRARY NAMES webrtc PATHS ${VCPKG_INSTALL_ROOT}/lib/ NO_DEFAULT_PATH) + target_link_libraries(${TARGET_NAME} ${WEBRTC_LIBRARY}) +endif() + +endmacro() diff --git a/cmake/ports/hifi-deps/CONTROL b/cmake/ports/hifi-deps/CONTROL index 5f860a1620..2441de9002 100644 --- a/cmake/ports/hifi-deps/CONTROL +++ b/cmake/ports/hifi-deps/CONTROL @@ -1,4 +1,4 @@ Source: hifi-deps Version: 0.1 Description: Collected dependencies for High Fidelity applications -Build-Depends: bullet3, draco, etc2comp, glm, nvtt, openexr (!android), openssl (windows), tbb (!android&!osx), zlib +Build-Depends: bullet3, draco, etc2comp, glm, nvtt, openexr (!android), openssl (windows), tbb (!android&!osx), zlib, webrtc (!android) diff --git a/cmake/ports/webrtc/CONTROL b/cmake/ports/webrtc/CONTROL new file mode 100644 index 0000000000..12a76920b9 --- /dev/null +++ b/cmake/ports/webrtc/CONTROL @@ -0,0 +1,3 @@ +Source: webrtc +Version: 20190626 +Description: WebRTC diff --git a/cmake/ports/webrtc/portfile.cmake b/cmake/ports/webrtc/portfile.cmake new file mode 100644 index 0000000000..85479d4201 --- /dev/null +++ b/cmake/ports/webrtc/portfile.cmake @@ -0,0 +1,35 @@ +include(vcpkg_common_functions) +set(WEBRTC_VERSION 20190626) +set(MASTER_COPY_SOURCE_PATH ${CURRENT_BUILDTREES_DIR}/src) + +if (WIN32) + vcpkg_download_distfile( + WEBRTC_SOURCE_ARCHIVE + URLS https://hifi-public.s3.amazonaws.com/seth/webrtc-20190626-windows.zip + SHA512 f4444a95f87800b446d95a4c381b69a8ddc36d89a08ce57bf2b03d684ce528b592a3260ef2f9dee314e38b58dd236a3495677e1c7773ce3d4057f59e80b48f32 + FILENAME webrtc-20190626-windows.zip + ) +elseif (APPLE) + vcpkg_download_distfile( + WEBRTC_SOURCE_ARCHIVE + URLS https://hifi-public.s3.amazonaws.com/seth/webrtc-20190626-osx.tar.gz + SHA512 fc70cec1b5ee87395137b7090f424e2fc2300fc17d744d5ffa1cf7aa0e0f1a069a9d72ba1ad2fb4a640ebeb6c218bda24351ba0083e1ff96c4a4b5032648a9d2 + FILENAME webrtc-20190626-osx.tar.gz + ) +elseif (ANDROID) +# then not desktop Linux +else () + vcpkg_download_distfile( + WEBRTC_SOURCE_ARCHIVE + URLS https://hifi-public.s3.amazonaws.com/seth/webrtc-20190626-linux.tar.gz + SHA512 7e41c3350e4dd3bbbedff9070ef0887db4e8d1607270ac2057cee3c8c2bf17aa2bdccb6c1bfa14526338a51b57d7896b7e1230c6cedb41f3afe5c49a5f1a7319 + FILENAME webrtc-20190626-linux.tar.gz + ) +endif () + +vcpkg_extract_source_archive(${WEBRTC_SOURCE_ARCHIVE}) + +file(COPY ${MASTER_COPY_SOURCE_PATH}/webrtc/include DESTINATION ${CURRENT_PACKAGES_DIR}) +file(COPY ${MASTER_COPY_SOURCE_PATH}/webrtc/lib DESTINATION ${CURRENT_PACKAGES_DIR}) +file(COPY ${MASTER_COPY_SOURCE_PATH}/webrtc/share DESTINATION ${CURRENT_PACKAGES_DIR}) +file(COPY ${MASTER_COPY_SOURCE_PATH}/webrtc/debug DESTINATION ${CURRENT_PACKAGES_DIR}) diff --git a/libraries/audio-client/CMakeLists.txt b/libraries/audio-client/CMakeLists.txt index 6ca7962c39..48d3d74cd6 100644 --- a/libraries/audio-client/CMakeLists.txt +++ b/libraries/audio-client/CMakeLists.txt @@ -7,6 +7,8 @@ link_hifi_libraries(audio plugins) include_hifi_library_headers(shared) include_hifi_library_headers(networking) +target_webrtc() + # append audio includes to our list of includes to bubble target_include_directories(${TARGET_NAME} PUBLIC "${HIFI_LIBRARY_DIR}/audio/src") diff --git a/libraries/audio-client/src/AudioClient.cpp b/libraries/audio-client/src/AudioClient.cpp index 04ab0f7973..3394543ddd 100644 --- a/libraries/audio-client/src/AudioClient.cpp +++ b/libraries/audio-client/src/AudioClient.cpp @@ -24,7 +24,7 @@ #endif #ifdef WIN32 -#define WIN32_LEAN_AND_MEAN +#define WIN32_LEAN_AND_MEAN 1 #include #include #include @@ -302,6 +302,13 @@ AudioClient::AudioClient() : _isHeadsetPluggedIn(false), #endif _orientationGetter(DEFAULT_ORIENTATION_GETTER) { + +#if defined(WEBRTC_ENABLED) + qDebug() << "QQQQ calling AudioProcessingBuilder"; + _apm = webrtc::AudioProcessingBuilder().Create(); + qDebug() << "QQQQ done calling AudioProcessingBuilder"; +#endif + // avoid putting a lock in the device callback assert(_localSamplesAvailable.is_lock_free()); diff --git a/libraries/audio-client/src/AudioClient.h b/libraries/audio-client/src/AudioClient.h index decf0f7751..67a064929c 100644 --- a/libraries/audio-client/src/AudioClient.h +++ b/libraries/audio-client/src/AudioClient.h @@ -29,6 +29,7 @@ #include #include #include +#include #include #include @@ -475,6 +476,10 @@ private: QTimer* _checkPeakValuesTimer { nullptr }; bool _isRecording { false }; + +#if WEBRTC_ENABLED + webrtc::AudioProcessing* _apm; +#endif }; diff --git a/libraries/shared/src/shared/WebRTC.h b/libraries/shared/src/shared/WebRTC.h new file mode 100644 index 0000000000..3085f46c0a --- /dev/null +++ b/libraries/shared/src/shared/WebRTC.h @@ -0,0 +1,34 @@ +// +// WebRTC.h +// libraries/shared/src/shared/ +// +// Copyright 2019 High Fidelity, Inc. +// +// Distributed under the Apache License, Version 2.0. +// See the accompanying file LICENSE or http://www.apache.org/licenses/LICENSE-2.0.html +// + +#ifndef hifi_WebRTC_h +#define hifi_WebRTC_h + +#if defined(Q_OS_MAC) +# define WEBRTC_ENABLED 1 +# define WEBRTC_POSIX 1 +#elif defined(WIN32) +# define WEBRTC_ENABLED 1 +# define WEBRTC_WIN 1 +# define NOMINMAX 1 +# define WIN32_LEAN_AND_MEAN 1 +#elif defined(Q_OS_ANDROID) +// no webrtc for android -- this is here so the LINUX clause doesn't get used, below +#elif defined(Q_OS_LINUX) +# define WEBRTC_ENABLED 1 +# define WEBRTC_POSIX 1 +#endif + +#if defined(WEBRTC_ENABLED) +# include +# include "modules/audio_processing/audio_processing_impl.h" +#endif + +#endif // hifi_WebRTC_h From 80aeeaa3e0cac81b85af73f686adfaf532844e5a Mon Sep 17 00:00:00 2001 From: Ken Cooke Date: Thu, 18 Jul 2019 11:43:46 -0700 Subject: [PATCH 02/13] Refactor the audio output callback, to expose the final output (far-end) needed for AEC --- libraries/audio-client/src/AudioClient.cpp | 75 +++++++++++----------- 1 file changed, 39 insertions(+), 36 deletions(-) diff --git a/libraries/audio-client/src/AudioClient.cpp b/libraries/audio-client/src/AudioClient.cpp index 3394543ddd..946b47fec5 100644 --- a/libraries/audio-client/src/AudioClient.cpp +++ b/libraries/audio-client/src/AudioClient.cpp @@ -2114,15 +2114,16 @@ qint64 AudioClient::AudioOutputIODevice::readData(char * data, qint64 maxSize) { return maxSize; } - // samples requested from OUTPUT_CHANNEL_COUNT + // max samples requested from OUTPUT_CHANNEL_COUNT int deviceChannelCount = _audio->_outputFormat.channelCount(); - int samplesRequested = (int)(maxSize / AudioConstants::SAMPLE_SIZE) * OUTPUT_CHANNEL_COUNT / deviceChannelCount; + int maxSamplesRequested = (int)(maxSize / AudioConstants::SAMPLE_SIZE) * OUTPUT_CHANNEL_COUNT / deviceChannelCount; // restrict samplesRequested to the size of our mix/scratch buffers - samplesRequested = std::min(samplesRequested, _audio->_outputPeriod); + maxSamplesRequested = std::min(maxSamplesRequested, _audio->_outputPeriod); int16_t* scratchBuffer = _audio->_outputScratchBuffer; float* mixBuffer = _audio->_outputMixBuffer; + int samplesRequested = maxSamplesRequested; int networkSamplesPopped; if ((networkSamplesPopped = _receivedAudioStream.popSamples(samplesRequested, false)) > 0) { qCDebug(audiostream, "Read %d samples from buffer (%d available, %d requested)", networkSamplesPopped, _receivedAudioStream.getSamplesAvailable(), samplesRequested); @@ -2167,40 +2168,42 @@ qint64 AudioClient::AudioOutputIODevice::readData(char * data, qint64 maxSize) { }); int samplesPopped = std::max(networkSamplesPopped, injectorSamplesPopped); - int framesPopped = samplesPopped / AudioConstants::STEREO; - int bytesWritten; - if (samplesPopped > 0) { - - // apply output gain - float newGain = _audio->_outputGain.load(std::memory_order_acquire); - float oldGain = _audio->_lastOutputGain; - _audio->_lastOutputGain = newGain; - - applyGainSmoothing(mixBuffer, framesPopped, oldGain, newGain); - - if (deviceChannelCount == OUTPUT_CHANNEL_COUNT) { - // limit the audio - _audio->_audioLimiter.render(mixBuffer, (int16_t*)data, framesPopped); - } else { - _audio->_audioLimiter.render(mixBuffer, scratchBuffer, framesPopped); - - // upmix or downmix to deviceChannelCount - if (deviceChannelCount > OUTPUT_CHANNEL_COUNT) { - int extraChannels = deviceChannelCount - OUTPUT_CHANNEL_COUNT; - channelUpmix(scratchBuffer, (int16_t*)data, samplesPopped, extraChannels); - } else { - channelDownmix(scratchBuffer, (int16_t*)data, samplesPopped); - } - } - - bytesWritten = framesPopped * AudioConstants::SAMPLE_SIZE * deviceChannelCount; - assert(bytesWritten <= maxSize); - - } else { - // nothing on network, don't grab anything from injectors, and just return 0s - memset(data, 0, maxSize); - bytesWritten = maxSize; + if (samplesPopped == 0) { + // nothing on network, don't grab anything from injectors, and fill with silence + samplesPopped = maxSamplesRequested; + memset(mixBuffer, 0, samplesPopped * sizeof(float)); } + int framesPopped = samplesPopped / OUTPUT_CHANNEL_COUNT; + + // apply output gain + float newGain = _audio->_outputGain.load(std::memory_order_acquire); + float oldGain = _audio->_lastOutputGain; + _audio->_lastOutputGain = newGain; + + applyGainSmoothing(mixBuffer, framesPopped, oldGain, newGain); + + // limit the audio + _audio->_audioLimiter.render(mixBuffer, scratchBuffer, framesPopped); + + // TODO: + // At this point, scratchBuffer contains the final (mixed, limited) output audio. + // format = interleaved int16_t + // samples = samplesPopped + // channels = OUTPUT_CHANNEL_COUNT + // sampleRate = _outputFormat.sampleRate() + // This can be used as the far-end signal for AEC. + + // if required, upmix or downmix to deviceChannelCount + if (deviceChannelCount == OUTPUT_CHANNEL_COUNT) { + memcpy(data, scratchBuffer, samplesPopped * AudioConstants::SAMPLE_SIZE); + } else if (deviceChannelCount > OUTPUT_CHANNEL_COUNT) { + int extraChannels = deviceChannelCount - OUTPUT_CHANNEL_COUNT; + channelUpmix(scratchBuffer, (int16_t*)data, samplesPopped, extraChannels); + } else { + channelDownmix(scratchBuffer, (int16_t*)data, samplesPopped); + } + int bytesWritten = framesPopped * AudioConstants::SAMPLE_SIZE * deviceChannelCount; + assert(bytesWritten <= maxSize); // send output buffer for recording if (_audio->_isRecording) { From c3b9e4806a14f889fab0918d43f493da8031c713 Mon Sep 17 00:00:00 2001 From: Ken Cooke Date: Thu, 18 Jul 2019 11:54:01 -0700 Subject: [PATCH 03/13] Fix longstanding bug in Audio Scope feature that records output to WAV --- libraries/audio-client/src/AudioClient.cpp | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/libraries/audio-client/src/AudioClient.cpp b/libraries/audio-client/src/AudioClient.cpp index 946b47fec5..9bf7a24c56 100644 --- a/libraries/audio-client/src/AudioClient.cpp +++ b/libraries/audio-client/src/AudioClient.cpp @@ -2208,7 +2208,7 @@ qint64 AudioClient::AudioOutputIODevice::readData(char * data, qint64 maxSize) { // send output buffer for recording if (_audio->_isRecording) { Lock lock(_recordMutex); - _audio->_audioFileWav.addRawAudioChunk(reinterpret_cast(scratchBuffer), bytesWritten); + _audio->_audioFileWav.addRawAudioChunk(data, bytesWritten); } int bytesAudioOutputUnplayed = _audio->_audioOutput->bufferSize() - _audio->_audioOutput->bytesFree(); From 569c76c8ba51ff25a90ebd7b1ec85206f4b0184d Mon Sep 17 00:00:00 2001 From: Ken Cooke Date: Sat, 20 Jul 2019 13:05:11 -0700 Subject: [PATCH 04/13] Baseline AEC using WebRTC, always enabled. Audio streams are hooked as close as possible to device input/output, re-buffering as needed. --- libraries/audio-client/src/AudioClient.cpp | 156 +++++++++++++++++++-- libraries/audio-client/src/AudioClient.h | 12 +- 2 files changed, 150 insertions(+), 18 deletions(-) diff --git a/libraries/audio-client/src/AudioClient.cpp b/libraries/audio-client/src/AudioClient.cpp index 9bf7a24c56..b95dd96d06 100644 --- a/libraries/audio-client/src/AudioClient.cpp +++ b/libraries/audio-client/src/AudioClient.cpp @@ -303,12 +303,6 @@ AudioClient::AudioClient() : #endif _orientationGetter(DEFAULT_ORIENTATION_GETTER) { -#if defined(WEBRTC_ENABLED) - qDebug() << "QQQQ calling AudioProcessingBuilder"; - _apm = webrtc::AudioProcessingBuilder().Create(); - qDebug() << "QQQQ done calling AudioProcessingBuilder"; -#endif - // avoid putting a lock in the device callback assert(_localSamplesAvailable.is_lock_free()); @@ -360,6 +354,10 @@ AudioClient::AudioClient() : configureReverb(); +#if defined(WEBRTC_ENABLED) + configureWebrtc(); +#endif + auto nodeList = DependencyManager::get(); auto& packetReceiver = nodeList->getPacketReceiver(); packetReceiver.registerListener(PacketType::AudioStreamStats, &_stats, "processStreamStatsPacket"); @@ -1091,6 +1089,137 @@ void AudioClient::setReverbOptions(const AudioEffectOptions* options) { } } +#if defined(WEBRTC_ENABLED) + +static const int WEBRTC_FRAMES_MAX = webrtc::AudioProcessing::kChunkSizeMs * webrtc::AudioProcessing::kMaxNativeSampleRateHz / 1000; +static const int WEBRTC_CHANNELS_MAX = 2; + +static void deinterleaveToFloat(const int16_t* src, float* const* dst, int numFrames, int numChannels) { + for (int i = 0; i < numFrames; i++) { + for (int ch = 0; ch < numChannels; ch++) { + float f = *src++; + f *= (1/32768.0f); // scale + dst[ch][i] = f; // deinterleave + } + } +} + +static void interleaveToInt16(const float* const* src, int16_t* dst, int numFrames, int numChannels) { + for (int i = 0; i < numFrames; i++) { + for (int ch = 0; ch < numChannels; ch++) { + float f = src[ch][i]; + f *= 32768.0f; // scale + f += (f < 0.0f) ? -0.5f : 0.5f; // round + f = std::max(std::min(f, 32767.0f), -32768.0f); // saturate + *dst++ = (int16_t)f; // interleave + } + } +} + +void AudioClient::configureWebrtc() { + _apm = webrtc::AudioProcessingBuilder().Create(); + + webrtc::AudioProcessing::Config config; + + config.pre_amplifier.enabled = false; + config.high_pass_filter.enabled = false; + config.echo_canceller.enabled = true; + config.echo_canceller.mobile_mode = false; + config.echo_canceller.use_legacy_aec = false; + config.noise_suppression.enabled = false; + config.noise_suppression.level = webrtc::AudioProcessing::Config::NoiseSuppression::kModerate; + config.voice_detection.enabled = false; + config.gain_controller1.enabled = false; + config.gain_controller2.enabled = false; + config.gain_controller2.fixed_digital.gain_db = 0.0f; + config.gain_controller2.adaptive_digital.enabled = false; + config.residual_echo_detector.enabled = true; + config.level_estimation.enabled = false; + + _apm->ApplyConfig(config); + + qCDebug(audioclient) << "WebRTC enabled for acoustic echo cancellation."; +} + +// rebuffer into 10ms chunks +void AudioClient::processWebrtcFarEnd(const int16_t* samples, int numFrames, int numChannels, int sampleRate) { + + // TODO: move to AudioClient.h + static int16_t _fifo[WEBRTC_CHANNELS_MAX * WEBRTC_FRAMES_MAX]; + static int _numFifo = 0; // numFrames saved in fifo + + const webrtc::StreamConfig streamConfig = webrtc::StreamConfig(sampleRate, numChannels); + const int numChunk = (int)streamConfig.num_frames(); + + if (sampleRate > webrtc::AudioProcessing::kMaxNativeSampleRateHz) { + qCWarning(audioclient) << "WebRTC does not support" << sampleRate << "output sample rate."; + return; + } + if (numChannels > WEBRTC_CHANNELS_MAX) { + qCWarning(audioclient) << "WebRTC does not support" << numChannels << "output channels."; + return; + } + + while (numFrames > 0) { + + // number of frames to fill + int numFill = std::min(numFrames, numChunk - _numFifo); + + // refill fifo + memcpy(&_fifo[_numFifo], samples, numFill * numChannels * sizeof(int16_t)); + samples += numFill * numChannels; + numFrames -= numFill; + _numFifo += numFill; + + if (_numFifo == numChunk) { + + // convert audio format + float buffer[WEBRTC_CHANNELS_MAX][WEBRTC_FRAMES_MAX]; + float* const buffers[WEBRTC_CHANNELS_MAX] = { buffer[0], buffer[1] }; + deinterleaveToFloat(_fifo, buffers, numChunk, numChannels); + + // process one chunk + if (_apm->kNoError != _apm->ProcessReverseStream(buffers, streamConfig, streamConfig, buffers)) { + qCWarning(audioclient) << "WebRTC ProcessReverseStream() returned an ERROR."; + } + _numFifo = 0; + } + } +} + +void AudioClient::processWebrtcNearEnd(int16_t* samples, int numFrames, int numChannels, int sampleRate) { + + const webrtc::StreamConfig streamConfig = webrtc::StreamConfig(sampleRate, numChannels); + const int numChunk = (int)streamConfig.num_frames(); + + if (sampleRate > webrtc::AudioProcessing::kMaxNativeSampleRateHz) { + qCWarning(audioclient) << "WebRTC does not support" << sampleRate << "input sample rate."; + return; + } + if (numChannels > WEBRTC_CHANNELS_MAX) { + qCWarning(audioclient) << "WebRTC does not support" << numChannels << "input channels."; + return; + } + if (numFrames != numChunk) { + qCWarning(audioclient) << "WebRTC requires exactly 10ms of input."; + return; + } + + // convert audio format + float buffer[WEBRTC_CHANNELS_MAX][WEBRTC_FRAMES_MAX]; + float* const buffers[WEBRTC_CHANNELS_MAX] = { buffer[0], buffer[1] }; + deinterleaveToFloat(samples, buffers, numFrames, numChannels); + + // process one chunk + if (_apm->kNoError != _apm->ProcessStream(buffers, streamConfig, streamConfig, buffers)) { + qCWarning(audioclient) << "WebRTC ProcessStream() returned an ERROR."; + } + // modify samples in-place + interleaveToInt16(buffers, samples, numFrames, numChannels); +} + +#endif // WEBRTC_ENABLED + void AudioClient::handleLocalEchoAndReverb(QByteArray& inputByteArray) { // If there is server echo, reverb will be applied to the recieved audio stream so no need to have it here. bool hasReverb = _reverb || _receivedAudioStream.hasReverb(); @@ -1269,6 +1398,11 @@ void AudioClient::handleMicAudioInput() { _inputRingBuffer.readSamples(inputAudioSamples.get(), inputSamplesRequired); +#if defined(WEBRTC_ENABLED) + processWebrtcNearEnd(inputAudioSamples.get(), inputSamplesRequired / _inputFormat.channelCount(), + _inputFormat.channelCount(), _inputFormat.sampleRate()); +#endif + // detect loudness and clipping on the raw input bool isClipping = false; float loudness = computeLoudness(inputAudioSamples.get(), inputSamplesRequired, _inputFormat.channelCount(), isClipping); @@ -2185,13 +2319,9 @@ qint64 AudioClient::AudioOutputIODevice::readData(char * data, qint64 maxSize) { // limit the audio _audio->_audioLimiter.render(mixBuffer, scratchBuffer, framesPopped); - // TODO: - // At this point, scratchBuffer contains the final (mixed, limited) output audio. - // format = interleaved int16_t - // samples = samplesPopped - // channels = OUTPUT_CHANNEL_COUNT - // sampleRate = _outputFormat.sampleRate() - // This can be used as the far-end signal for AEC. +#if defined(WEBRTC_ENABLED) + _audio->processWebrtcFarEnd(scratchBuffer, framesPopped, OUTPUT_CHANNEL_COUNT, _audio->_outputFormat.sampleRate()); +#endif // if required, upmix or downmix to deviceChannelCount if (deviceChannelCount == OUTPUT_CHANNEL_COUNT) { diff --git a/libraries/audio-client/src/AudioClient.h b/libraries/audio-client/src/AudioClient.h index 67a064929c..8cf54c1a0d 100644 --- a/libraries/audio-client/src/AudioClient.h +++ b/libraries/audio-client/src/AudioClient.h @@ -415,9 +415,15 @@ private: // Adds Reverb void configureReverb(); void updateReverbOptions(); - void handleLocalEchoAndReverb(QByteArray& inputByteArray); +#if defined(WEBRTC_ENABLED) + webrtc::AudioProcessing* _apm { nullptr }; + void configureWebrtc(); + void processWebrtcFarEnd(const int16_t* samples, int numFrames, int numChannels, int sampleRate); + void processWebrtcNearEnd(int16_t* samples, int numFrames, int numChannels, int sampleRate); +#endif + bool switchInputToAudioDevice(const QAudioDeviceInfo inputDeviceInfo, bool isShutdownRequest = false); bool switchOutputToAudioDevice(const QAudioDeviceInfo outputDeviceInfo, bool isShutdownRequest = false); @@ -476,10 +482,6 @@ private: QTimer* _checkPeakValuesTimer { nullptr }; bool _isRecording { false }; - -#if WEBRTC_ENABLED - webrtc::AudioProcessing* _apm; -#endif }; From c1cadf670e89d8324f6ac85fcb775dbc884b8f44 Mon Sep 17 00:00:00 2001 From: Seth Alves Date: Mon, 22 Jul 2019 10:43:43 -0700 Subject: [PATCH 05/13] update webrtc library, enable for android --- cmake/macros/TargetWebRTC.cmake | 17 ++++++++++++----- cmake/ports/hifi-deps/CONTROL | 2 +- cmake/ports/webrtc/portfile.cmake | 11 ++++++----- hifi_android.py | 5 +++++ libraries/shared/src/shared/WebRTC.h | 5 +++-- 5 files changed, 27 insertions(+), 13 deletions(-) diff --git a/cmake/macros/TargetWebRTC.cmake b/cmake/macros/TargetWebRTC.cmake index e117011101..fd5db5fcbe 100644 --- a/cmake/macros/TargetWebRTC.cmake +++ b/cmake/macros/TargetWebRTC.cmake @@ -5,12 +5,19 @@ # See the accompanying file LICENSE or http://www.apache.org/licenses/LICENSE-2.0.html # macro(TARGET_WEBRTC) + if (ANDROID) + include(SelectLibraryConfigurations) + set(INSTALL_DIR ${HIFI_ANDROID_PRECOMPILED}/webrtc/webrtc) + set(WEBRTC_INCLUDE_DIRS "${INSTALL_DIR}/include/webrtc") + set(WEBRTC_LIBRARY_DEBUG ${INSTALL_DIR}/debug/lib/libwebrtc.a) + set(WEBRTC_LIBRARY_RELEASE ${INSTALL_DIR}/lib/libwebrtc.a) + select_library_configurations(WEBRTC) + else() + set(WEBRTC_INCLUDE_DIRS "${VCPKG_INSTALL_ROOT}/include/webrtc") + find_library(WEBRTC_LIBRARY NAMES webrtc PATHS ${VCPKG_INSTALL_ROOT}/lib/ NO_DEFAULT_PATH) + endif() -if (ANDROID) -else() - target_include_directories(${TARGET_NAME} SYSTEM PUBLIC "${VCPKG_INSTALL_ROOT}/include/webrtc") - find_library(WEBRTC_LIBRARY NAMES webrtc PATHS ${VCPKG_INSTALL_ROOT}/lib/ NO_DEFAULT_PATH) + target_include_directories(${TARGET_NAME} SYSTEM PUBLIC ${WEBRTC_INCLUDE_DIRS}) target_link_libraries(${TARGET_NAME} ${WEBRTC_LIBRARY}) -endif() endmacro() diff --git a/cmake/ports/hifi-deps/CONTROL b/cmake/ports/hifi-deps/CONTROL index 2441de9002..0a3d194ea1 100644 --- a/cmake/ports/hifi-deps/CONTROL +++ b/cmake/ports/hifi-deps/CONTROL @@ -1,4 +1,4 @@ Source: hifi-deps Version: 0.1 Description: Collected dependencies for High Fidelity applications -Build-Depends: bullet3, draco, etc2comp, glm, nvtt, openexr (!android), openssl (windows), tbb (!android&!osx), zlib, webrtc (!android) +Build-Depends: bullet3, draco, etc2comp, glm, nvtt, openexr (!android), openssl (windows), tbb (!android&!osx), zlib, webrtc diff --git a/cmake/ports/webrtc/portfile.cmake b/cmake/ports/webrtc/portfile.cmake index 85479d4201..3f2fb7a6ab 100644 --- a/cmake/ports/webrtc/portfile.cmake +++ b/cmake/ports/webrtc/portfile.cmake @@ -2,11 +2,13 @@ include(vcpkg_common_functions) set(WEBRTC_VERSION 20190626) set(MASTER_COPY_SOURCE_PATH ${CURRENT_BUILDTREES_DIR}/src) -if (WIN32) +if (ANDROID) + # this is handled by hifi_android.py +elseif (WIN32) vcpkg_download_distfile( WEBRTC_SOURCE_ARCHIVE URLS https://hifi-public.s3.amazonaws.com/seth/webrtc-20190626-windows.zip - SHA512 f4444a95f87800b446d95a4c381b69a8ddc36d89a08ce57bf2b03d684ce528b592a3260ef2f9dee314e38b58dd236a3495677e1c7773ce3d4057f59e80b48f32 + SHA512 c0848eddb1579b3bb0496b8785e24f30470f3c477145035fd729264a326a467b9467ae9f426aa5d72d168ad9e9bf2c279150744832736bdf39064d24b04de1a3 FILENAME webrtc-20190626-windows.zip ) elseif (APPLE) @@ -16,13 +18,12 @@ elseif (APPLE) SHA512 fc70cec1b5ee87395137b7090f424e2fc2300fc17d744d5ffa1cf7aa0e0f1a069a9d72ba1ad2fb4a640ebeb6c218bda24351ba0083e1ff96c4a4b5032648a9d2 FILENAME webrtc-20190626-osx.tar.gz ) -elseif (ANDROID) -# then not desktop Linux else () + # else Linux desktop vcpkg_download_distfile( WEBRTC_SOURCE_ARCHIVE URLS https://hifi-public.s3.amazonaws.com/seth/webrtc-20190626-linux.tar.gz - SHA512 7e41c3350e4dd3bbbedff9070ef0887db4e8d1607270ac2057cee3c8c2bf17aa2bdccb6c1bfa14526338a51b57d7896b7e1230c6cedb41f3afe5c49a5f1a7319 + SHA512 07d7776551aa78cb09a3ef088a8dee7762735c168c243053b262083d90a1d258cec66dc386f6903da5c4461921a3c2db157a1ee106a2b47e7756cb424b66cc43 FILENAME webrtc-20190626-linux.tar.gz ) endif () diff --git a/hifi_android.py b/hifi_android.py index 42b472e960..926a671a6c 100644 --- a/hifi_android.py +++ b/hifi_android.py @@ -94,6 +94,11 @@ ANDROID_PACKAGES = { 'checksum': 'ddcb23df336b08017042ba4786db1d9e', 'sharedLibFolder': 'lib', 'includeLibs': {'libbreakpad_client.a'} + }, + 'webrtc': { + 'file': 'webrtc-20190626-android.tar.gz', + 'checksum': '8be046cf27e1fabff8a1029eabbe52c8', + 'versionId': '8TU1LgZswAgZpBiNYvB5s3ZLwSqNJHuj', } } diff --git a/libraries/shared/src/shared/WebRTC.h b/libraries/shared/src/shared/WebRTC.h index 3085f46c0a..6f05d38a3d 100644 --- a/libraries/shared/src/shared/WebRTC.h +++ b/libraries/shared/src/shared/WebRTC.h @@ -14,13 +14,14 @@ #if defined(Q_OS_MAC) # define WEBRTC_ENABLED 1 # define WEBRTC_POSIX 1 -#elif defined(WIN32) +#elif defined(Q_OS_WIN) # define WEBRTC_ENABLED 1 # define WEBRTC_WIN 1 # define NOMINMAX 1 # define WIN32_LEAN_AND_MEAN 1 #elif defined(Q_OS_ANDROID) -// no webrtc for android -- this is here so the LINUX clause doesn't get used, below +# define WEBRTC_ENABLED 1 +# define WEBRTC_POSIX 1 #elif defined(Q_OS_LINUX) # define WEBRTC_ENABLED 1 # define WEBRTC_POSIX 1 From 8ec4505b953ffe6c1f7e968211ab0dd9eaa73c1b Mon Sep 17 00:00:00 2001 From: Seth Alves Date: Wed, 24 Jul 2019 18:51:02 -0700 Subject: [PATCH 06/13] add an on/off control for audio echo cancelation to Audio Settings page. It's not yet hooked to anything. --- interface/resources/qml/hifi/audio/Audio.qml | 24 +++++++++++++++---- interface/src/scripting/Audio.cpp | 25 ++++++++++++++++++++ interface/src/scripting/Audio.h | 16 +++++++++++++ libraries/audio-client/src/AudioClient.cpp | 10 ++++++++ libraries/audio-client/src/AudioClient.h | 5 ++++ 5 files changed, 76 insertions(+), 4 deletions(-) diff --git a/interface/resources/qml/hifi/audio/Audio.qml b/interface/resources/qml/hifi/audio/Audio.qml index af517be55d..9db19e72e2 100644 --- a/interface/resources/qml/hifi/audio/Audio.qml +++ b/interface/resources/qml/hifi/audio/Audio.qml @@ -166,16 +166,16 @@ Rectangle { x: 2 * margins.paddings; width: parent.width; // switch heights + 2 * top margins - height: (root.switchHeight) * 3 + 48; + height: (root.switchHeight) * 6 + 48; anchors.top: firstSeparator.bottom; anchors.topMargin: 10; - // mute is in its own row Item { id: switchContainer; x: margins.paddings; width: parent.width / 2; height: parent.height; + anchors.top: parent.top anchors.left: parent.left; HifiControlsUit.Switch { id: muteMic; @@ -222,12 +222,29 @@ Rectangle { } HifiControlsUit.Switch { - id: pttSwitch + id: acousticEchoCancellationSwitch; height: root.switchHeight; switchWidth: root.switchWidth; anchors.top: noiseReductionSwitch.bottom anchors.topMargin: 24 anchors.left: parent.left + labelTextOn: "Echo Cancellation"; + labelTextSize: 16; + backgroundOnColor: "#E3E3E3"; + checked: AudioScriptingInterface.acousticEchoCancellation; + onCheckedChanged: { + AudioScriptingInterface.acousticEchoCancellation = checked; + checked = Qt.binding(function() { return AudioScriptingInterface.acousticEchoCancellation; }); + } + } + + HifiControlsUit.Switch { + id: pttSwitch + height: root.switchHeight; + switchWidth: root.switchWidth; + anchors.top: acousticEchoCancellationSwitch.bottom; + anchors.topMargin: 24 + anchors.left: parent.left labelTextOn: (bar.currentIndex === 0) ? qsTr("Push To Talk (T)") : qsTr("Push To Talk"); labelTextSize: 16; backgroundOnColor: "#E3E3E3"; @@ -298,7 +315,6 @@ Rectangle { checked = Qt.binding(function() { return AudioScriptingInterface.isStereoInput; }); // restore binding } } - } } diff --git a/interface/src/scripting/Audio.cpp b/interface/src/scripting/Audio.cpp index a0bea256ad..f674b533a7 100644 --- a/interface/src/scripting/Audio.cpp +++ b/interface/src/scripting/Audio.cpp @@ -26,6 +26,7 @@ QString Audio::HMD { "VR" }; Setting::Handle enableNoiseReductionSetting { QStringList { Audio::AUDIO, "NoiseReduction" }, true }; Setting::Handle enableWarnWhenMutedSetting { QStringList { Audio::AUDIO, "WarnWhenMuted" }, true }; +Setting::Handle enableAcousticEchoCancellationSetting { QStringList { Audio::AUDIO, "AcousticEchoCancellation" }, true }; float Audio::loudnessToLevel(float loudness) { @@ -40,12 +41,14 @@ Audio::Audio() : _devices(_contextIsHMD) { connect(client, &AudioClient::muteToggled, this, &Audio::setMuted); connect(client, &AudioClient::noiseReductionChanged, this, &Audio::enableNoiseReduction); connect(client, &AudioClient::warnWhenMutedChanged, this, &Audio::enableWarnWhenMuted); + connect(client, &AudioClient::acousticEchoCancellationChanged, this, &Audio::enableAcousticEchoCancellation); connect(client, &AudioClient::inputLoudnessChanged, this, &Audio::onInputLoudnessChanged); connect(client, &AudioClient::inputVolumeChanged, this, &Audio::setInputVolume); connect(this, &Audio::contextChanged, &_devices, &AudioDevices::onContextChanged); connect(this, &Audio::pushingToTalkChanged, this, &Audio::handlePushedToTalk); enableNoiseReduction(enableNoiseReductionSetting.get()); enableWarnWhenMuted(enableWarnWhenMutedSetting.get()); + enableAcousticEchoCancellation(enableAcousticEchoCancellationSetting.get()); onContextChanged(); } @@ -277,6 +280,28 @@ void Audio::enableWarnWhenMuted(bool enable) { } } +bool Audio::acousticEchoCancellationEnabled() const { + return resultWithReadLock([&] { + return _enableAcousticEchoCancellation; + }); +} + +void Audio::enableAcousticEchoCancellation(bool enable) { + bool changed = false; + withWriteLock([&] { + if (_enableAcousticEchoCancellation != enable) { + _enableAcousticEchoCancellation = enable; + auto client = DependencyManager::get().data(); + QMetaObject::invokeMethod(client, "setAcousticEchoCancellation", Q_ARG(bool, enable), Q_ARG(bool, false)); + enableAcousticEchoCancellationSetting.set(enable); + changed = true; + } + }); + if (changed) { + emit acousticEchoCancellationChanged(enable); + } +} + float Audio::getInputVolume() const { return resultWithReadLock([&] { return _inputVolume; diff --git a/interface/src/scripting/Audio.h b/interface/src/scripting/Audio.h index aab1ade95b..d2d1ee36c9 100644 --- a/interface/src/scripting/Audio.h +++ b/interface/src/scripting/Audio.h @@ -72,6 +72,9 @@ class Audio : public AudioScriptingInterface, protected ReadWriteLockable { * @property {number} systemInjectorGain - The gain (relative volume) that system sounds are played at. * @property {number} pushingToTalkOutputGainDesktop - The gain (relative volume) that all sounds are played at when the user is holding * the push-to-talk key in Desktop mode. + * @property {boolean} acousticEchoCancellation - true if audio-echo-cancellation is enabled, otherwise + * false. When enabled, sound from the audio output will be suppressed when it echos back to the + * input audio signal. * * @comment The following properties are from AudioScriptingInterface.h. * @property {boolean} isStereoInput - true if the input audio is being used in stereo, otherwise @@ -85,6 +88,8 @@ class Audio : public AudioScriptingInterface, protected ReadWriteLockable { Q_PROPERTY(bool muted READ isMuted WRITE setMuted NOTIFY mutedChanged) Q_PROPERTY(bool noiseReduction READ noiseReductionEnabled WRITE enableNoiseReduction NOTIFY noiseReductionChanged) Q_PROPERTY(bool warnWhenMuted READ warnWhenMutedEnabled WRITE enableWarnWhenMuted NOTIFY warnWhenMutedChanged) + Q_PROPERTY(bool acousticEchoCancellation + READ acousticEchoCancellationEnabled WRITE enableAcousticEchoCancellation NOTIFY acousticEchoCancellationChanged) Q_PROPERTY(float inputVolume READ getInputVolume WRITE setInputVolume NOTIFY inputVolumeChanged) Q_PROPERTY(float inputLevel READ getInputLevel NOTIFY inputLevelChanged) Q_PROPERTY(bool clipping READ isClipping NOTIFY clippingChanged) @@ -115,6 +120,7 @@ public: bool isMuted() const; bool noiseReductionEnabled() const; bool warnWhenMutedEnabled() const; + bool acousticEchoCancellationEnabled() const; float getInputVolume() const; float getInputLevel() const; bool isClipping() const; @@ -396,6 +402,14 @@ signals: */ void warnWhenMutedChanged(bool isEnabled); + /**jsdoc + * Triggered when acoustic echo cancellation is enabled or disabled. + * @function Audio.acousticEchoCancellationChanged + * @param {boolean} isEnabled - true if acoustic echo cancellation is enabled, otherwise false. + * @returns {Signal} + */ + void acousticEchoCancellationChanged(bool isEnabled); + /**jsdoc * Triggered when the input audio volume changes. * @function Audio.inputVolumeChanged @@ -494,6 +508,7 @@ private slots: void setMuted(bool muted); void enableNoiseReduction(bool enable); void enableWarnWhenMuted(bool enable); + void enableAcousticEchoCancellation(bool enable); void setInputVolume(float volume); void onInputLoudnessChanged(float loudness, bool isClipping); @@ -512,6 +527,7 @@ private: bool _isClipping { false }; bool _enableNoiseReduction { true }; // Match default value of AudioClient::_isNoiseGateEnabled. bool _enableWarnWhenMuted { true }; + bool _enableAcousticEchoCancellation { true }; // AudioClient::_isAECEnabled bool _contextIsHMD { false }; AudioDevices* getDevices() { return &_devices; } AudioDevices _devices; diff --git a/libraries/audio-client/src/AudioClient.cpp b/libraries/audio-client/src/AudioClient.cpp index b95dd96d06..6f5fe0e050 100644 --- a/libraries/audio-client/src/AudioClient.cpp +++ b/libraries/audio-client/src/AudioClient.cpp @@ -286,6 +286,7 @@ AudioClient::AudioClient() : _shouldEchoLocally(false), _shouldEchoToServer(false), _isNoiseGateEnabled(true), + _isAECEnabled(true), _reverb(false), _reverbOptions(&_scriptReverbOptions), _inputToNetworkResampler(NULL), @@ -1715,6 +1716,15 @@ void AudioClient::setWarnWhenMuted(bool enable, bool emitSignal) { } } +void AudioClient::setAcousticEchoCancellation(bool enable, bool emitSignal) { + if (_isAECEnabled != enable) { + _isAECEnabled = enable; + if (emitSignal) { + emit acousticEchoCancellationChanged(_isAECEnabled); + } + } +} + bool AudioClient::setIsStereoInput(bool isStereoInput) { bool stereoInputChanged = false; if (isStereoInput != _isStereoInput && _inputDeviceInfo.supportedChannelCounts().contains(2)) { diff --git a/libraries/audio-client/src/AudioClient.h b/libraries/audio-client/src/AudioClient.h index 8cf54c1a0d..638c1b61e9 100644 --- a/libraries/audio-client/src/AudioClient.h +++ b/libraries/audio-client/src/AudioClient.h @@ -216,6 +216,9 @@ public slots: void setWarnWhenMuted(bool isNoiseGateEnabled, bool emitSignal = true); bool isWarnWhenMutedEnabled() const { return _warnWhenMuted; } + void setAcousticEchoCancellation(bool isAECEnabled, bool emitSignal = true); + bool isAcousticEchoCancellationEnabled() const { return _isAECEnabled; } + virtual bool getLocalEcho() override { return _shouldEchoLocally; } virtual void setLocalEcho(bool localEcho) override { _shouldEchoLocally = localEcho; } virtual void toggleLocalEcho() override { _shouldEchoLocally = !_shouldEchoLocally; } @@ -257,6 +260,7 @@ signals: void muteToggled(bool muted); void noiseReductionChanged(bool noiseReductionEnabled); void warnWhenMutedChanged(bool warnWhenMutedEnabled); + void acousticEchoCancellationChanged(bool acousticEchoCancellationEnabled); void mutedByMixer(); void inputReceived(const QByteArray& inputSamples); void inputLoudnessChanged(float loudness, bool isClipping); @@ -378,6 +382,7 @@ private: bool _shouldEchoToServer; bool _isNoiseGateEnabled; bool _warnWhenMuted; + bool _isAECEnabled; bool _reverb; AudioEffectOptions _scriptReverbOptions; From 5c86c22076c9196af0c052f36e181aa3e71e6ce0 Mon Sep 17 00:00:00 2001 From: Ken Cooke Date: Fri, 26 Jul 2019 12:04:00 -0700 Subject: [PATCH 07/13] Move WebRTC far-end FIFO into AudioClient.h --- libraries/audio-client/src/AudioClient.cpp | 19 ++++++------------- libraries/audio-client/src/AudioClient.h | 7 +++++++ 2 files changed, 13 insertions(+), 13 deletions(-) diff --git a/libraries/audio-client/src/AudioClient.cpp b/libraries/audio-client/src/AudioClient.cpp index 6f5fe0e050..f4a01869e2 100644 --- a/libraries/audio-client/src/AudioClient.cpp +++ b/libraries/audio-client/src/AudioClient.cpp @@ -1092,9 +1092,6 @@ void AudioClient::setReverbOptions(const AudioEffectOptions* options) { #if defined(WEBRTC_ENABLED) -static const int WEBRTC_FRAMES_MAX = webrtc::AudioProcessing::kChunkSizeMs * webrtc::AudioProcessing::kMaxNativeSampleRateHz / 1000; -static const int WEBRTC_CHANNELS_MAX = 2; - static void deinterleaveToFloat(const int16_t* src, float* const* dst, int numFrames, int numChannels) { for (int i = 0; i < numFrames; i++) { for (int ch = 0; ch < numChannels; ch++) { @@ -1145,10 +1142,6 @@ void AudioClient::configureWebrtc() { // rebuffer into 10ms chunks void AudioClient::processWebrtcFarEnd(const int16_t* samples, int numFrames, int numChannels, int sampleRate) { - // TODO: move to AudioClient.h - static int16_t _fifo[WEBRTC_CHANNELS_MAX * WEBRTC_FRAMES_MAX]; - static int _numFifo = 0; // numFrames saved in fifo - const webrtc::StreamConfig streamConfig = webrtc::StreamConfig(sampleRate, numChannels); const int numChunk = (int)streamConfig.num_frames(); @@ -1164,26 +1157,26 @@ void AudioClient::processWebrtcFarEnd(const int16_t* samples, int numFrames, int while (numFrames > 0) { // number of frames to fill - int numFill = std::min(numFrames, numChunk - _numFifo); + int numFill = std::min(numFrames, numChunk - _numFifoFarEnd); // refill fifo - memcpy(&_fifo[_numFifo], samples, numFill * numChannels * sizeof(int16_t)); + memcpy(&_fifoFarEnd[_numFifoFarEnd], samples, numFill * numChannels * sizeof(int16_t)); samples += numFill * numChannels; numFrames -= numFill; - _numFifo += numFill; + _numFifoFarEnd += numFill; - if (_numFifo == numChunk) { + if (_numFifoFarEnd == numChunk) { // convert audio format float buffer[WEBRTC_CHANNELS_MAX][WEBRTC_FRAMES_MAX]; float* const buffers[WEBRTC_CHANNELS_MAX] = { buffer[0], buffer[1] }; - deinterleaveToFloat(_fifo, buffers, numChunk, numChannels); + deinterleaveToFloat(_fifoFarEnd, buffers, numChunk, numChannels); // process one chunk if (_apm->kNoError != _apm->ProcessReverseStream(buffers, streamConfig, streamConfig, buffers)) { qCWarning(audioclient) << "WebRTC ProcessReverseStream() returned an ERROR."; } - _numFifo = 0; + _numFifoFarEnd = 0; } } } diff --git a/libraries/audio-client/src/AudioClient.h b/libraries/audio-client/src/AudioClient.h index 638c1b61e9..e07a730d11 100644 --- a/libraries/audio-client/src/AudioClient.h +++ b/libraries/audio-client/src/AudioClient.h @@ -423,7 +423,14 @@ private: void handleLocalEchoAndReverb(QByteArray& inputByteArray); #if defined(WEBRTC_ENABLED) + static const int WEBRTC_FRAMES_MAX = webrtc::AudioProcessing::kChunkSizeMs * webrtc::AudioProcessing::kMaxNativeSampleRateHz / 1000; + static const int WEBRTC_CHANNELS_MAX = 2; + webrtc::AudioProcessing* _apm { nullptr }; + + int16_t _fifoFarEnd[WEBRTC_CHANNELS_MAX * WEBRTC_FRAMES_MAX] {}; + int _numFifoFarEnd = 0; // numFrames saved in fifo + void configureWebrtc(); void processWebrtcFarEnd(const int16_t* samples, int numFrames, int numChannels, int sampleRate); void processWebrtcNearEnd(int16_t* samples, int numFrames, int numChannels, int sampleRate); From bd0849fd4fe54e0aba3407372348eaa9acd632e0 Mon Sep 17 00:00:00 2001 From: Ken Cooke Date: Fri, 26 Jul 2019 12:18:30 -0700 Subject: [PATCH 08/13] Allow AEC with device sample rate up to 96KHz --- libraries/audio-client/src/AudioClient.cpp | 9 ++------- libraries/audio-client/src/AudioClient.h | 3 ++- 2 files changed, 4 insertions(+), 8 deletions(-) diff --git a/libraries/audio-client/src/AudioClient.cpp b/libraries/audio-client/src/AudioClient.cpp index f4a01869e2..f9451d196e 100644 --- a/libraries/audio-client/src/AudioClient.cpp +++ b/libraries/audio-client/src/AudioClient.cpp @@ -1145,12 +1145,10 @@ void AudioClient::processWebrtcFarEnd(const int16_t* samples, int numFrames, int const webrtc::StreamConfig streamConfig = webrtc::StreamConfig(sampleRate, numChannels); const int numChunk = (int)streamConfig.num_frames(); - if (sampleRate > webrtc::AudioProcessing::kMaxNativeSampleRateHz) { - qCWarning(audioclient) << "WebRTC does not support" << sampleRate << "output sample rate."; + if (sampleRate > WEBRTC_SAMPLE_RATE_MAX) { return; } if (numChannels > WEBRTC_CHANNELS_MAX) { - qCWarning(audioclient) << "WebRTC does not support" << numChannels << "output channels."; return; } @@ -1186,16 +1184,13 @@ void AudioClient::processWebrtcNearEnd(int16_t* samples, int numFrames, int numC const webrtc::StreamConfig streamConfig = webrtc::StreamConfig(sampleRate, numChannels); const int numChunk = (int)streamConfig.num_frames(); - if (sampleRate > webrtc::AudioProcessing::kMaxNativeSampleRateHz) { - qCWarning(audioclient) << "WebRTC does not support" << sampleRate << "input sample rate."; + if (sampleRate > WEBRTC_SAMPLE_RATE_MAX) { return; } if (numChannels > WEBRTC_CHANNELS_MAX) { - qCWarning(audioclient) << "WebRTC does not support" << numChannels << "input channels."; return; } if (numFrames != numChunk) { - qCWarning(audioclient) << "WebRTC requires exactly 10ms of input."; return; } diff --git a/libraries/audio-client/src/AudioClient.h b/libraries/audio-client/src/AudioClient.h index e07a730d11..ab12393ebf 100644 --- a/libraries/audio-client/src/AudioClient.h +++ b/libraries/audio-client/src/AudioClient.h @@ -423,8 +423,9 @@ private: void handleLocalEchoAndReverb(QByteArray& inputByteArray); #if defined(WEBRTC_ENABLED) - static const int WEBRTC_FRAMES_MAX = webrtc::AudioProcessing::kChunkSizeMs * webrtc::AudioProcessing::kMaxNativeSampleRateHz / 1000; + static const int WEBRTC_SAMPLE_RATE_MAX = 96000; static const int WEBRTC_CHANNELS_MAX = 2; + static const int WEBRTC_FRAMES_MAX = webrtc::AudioProcessing::kChunkSizeMs * WEBRTC_SAMPLE_RATE_MAX / 1000; webrtc::AudioProcessing* _apm { nullptr }; From e6e8491098a89cda9d2f9d08fffe341a1445231c Mon Sep 17 00:00:00 2001 From: Ken Cooke Date: Fri, 26 Jul 2019 12:32:38 -0700 Subject: [PATCH 09/13] Improved WebRTC error reporting --- libraries/audio-client/src/AudioClient.cpp | 20 ++++++++++++++------ 1 file changed, 14 insertions(+), 6 deletions(-) diff --git a/libraries/audio-client/src/AudioClient.cpp b/libraries/audio-client/src/AudioClient.cpp index f9451d196e..2d21958f92 100644 --- a/libraries/audio-client/src/AudioClient.cpp +++ b/libraries/audio-client/src/AudioClient.cpp @@ -1146,9 +1146,11 @@ void AudioClient::processWebrtcFarEnd(const int16_t* samples, int numFrames, int const int numChunk = (int)streamConfig.num_frames(); if (sampleRate > WEBRTC_SAMPLE_RATE_MAX) { + qCWarning(audioclient) << "WebRTC does not support" << sampleRate << "output sample rate."; return; } if (numChannels > WEBRTC_CHANNELS_MAX) { + qCWarning(audioclient) << "WebRTC does not support" << numChannels << "output channels."; return; } @@ -1171,8 +1173,9 @@ void AudioClient::processWebrtcFarEnd(const int16_t* samples, int numFrames, int deinterleaveToFloat(_fifoFarEnd, buffers, numChunk, numChannels); // process one chunk - if (_apm->kNoError != _apm->ProcessReverseStream(buffers, streamConfig, streamConfig, buffers)) { - qCWarning(audioclient) << "WebRTC ProcessReverseStream() returned an ERROR."; + int error = _apm->ProcessReverseStream(buffers, streamConfig, streamConfig, buffers); + if (error != _apm->kNoError) { + qCWarning(audioclient) << "WebRTC ProcessReverseStream() returned ERROR:" << error; } _numFifoFarEnd = 0; } @@ -1185,12 +1188,15 @@ void AudioClient::processWebrtcNearEnd(int16_t* samples, int numFrames, int numC const int numChunk = (int)streamConfig.num_frames(); if (sampleRate > WEBRTC_SAMPLE_RATE_MAX) { + qCWarning(audioclient) << "WebRTC does not support" << sampleRate << "input sample rate."; return; } if (numChannels > WEBRTC_CHANNELS_MAX) { + qCWarning(audioclient) << "WebRTC does not support" << numChannels << "input channels."; return; } if (numFrames != numChunk) { + qCWarning(audioclient) << "WebRTC requires exactly 10ms of input."; return; } @@ -1200,11 +1206,13 @@ void AudioClient::processWebrtcNearEnd(int16_t* samples, int numFrames, int numC deinterleaveToFloat(samples, buffers, numFrames, numChannels); // process one chunk - if (_apm->kNoError != _apm->ProcessStream(buffers, streamConfig, streamConfig, buffers)) { - qCWarning(audioclient) << "WebRTC ProcessStream() returned an ERROR."; + int error = _apm->ProcessStream(buffers, streamConfig, streamConfig, buffers); + if (error =! _apm->kNoError) { + qCWarning(audioclient) << "WebRTC ProcessStream() returned ERROR:" << error; + } else { + // modify samples in-place + interleaveToInt16(buffers, samples, numFrames, numChannels); } - // modify samples in-place - interleaveToInt16(buffers, samples, numFrames, numChannels); } #endif // WEBRTC_ENABLED From 8c6cee4e4dbe094b75dbf89754787a4f098434e8 Mon Sep 17 00:00:00 2001 From: Ken Cooke Date: Fri, 26 Jul 2019 12:45:21 -0700 Subject: [PATCH 10/13] Bypass WebRTC processing when not enabled. Note: WebRTC processing with all features disabled still holds locks and performs resampling in the callback. --- libraries/audio-client/src/AudioClient.cpp | 14 ++++++++------ 1 file changed, 8 insertions(+), 6 deletions(-) diff --git a/libraries/audio-client/src/AudioClient.cpp b/libraries/audio-client/src/AudioClient.cpp index 2d21958f92..c16e297c28 100644 --- a/libraries/audio-client/src/AudioClient.cpp +++ b/libraries/audio-client/src/AudioClient.cpp @@ -1135,8 +1135,6 @@ void AudioClient::configureWebrtc() { config.level_estimation.enabled = false; _apm->ApplyConfig(config); - - qCDebug(audioclient) << "WebRTC enabled for acoustic echo cancellation."; } // rebuffer into 10ms chunks @@ -1207,7 +1205,7 @@ void AudioClient::processWebrtcNearEnd(int16_t* samples, int numFrames, int numC // process one chunk int error = _apm->ProcessStream(buffers, streamConfig, streamConfig, buffers); - if (error =! _apm->kNoError) { + if (error != _apm->kNoError) { qCWarning(audioclient) << "WebRTC ProcessStream() returned ERROR:" << error; } else { // modify samples in-place @@ -1396,8 +1394,10 @@ void AudioClient::handleMicAudioInput() { _inputRingBuffer.readSamples(inputAudioSamples.get(), inputSamplesRequired); #if defined(WEBRTC_ENABLED) - processWebrtcNearEnd(inputAudioSamples.get(), inputSamplesRequired / _inputFormat.channelCount(), - _inputFormat.channelCount(), _inputFormat.sampleRate()); + if (_isAECEnabled) { + processWebrtcNearEnd(inputAudioSamples.get(), inputSamplesRequired / _inputFormat.channelCount(), + _inputFormat.channelCount(), _inputFormat.sampleRate()); + } #endif // detect loudness and clipping on the raw input @@ -2326,7 +2326,9 @@ qint64 AudioClient::AudioOutputIODevice::readData(char * data, qint64 maxSize) { _audio->_audioLimiter.render(mixBuffer, scratchBuffer, framesPopped); #if defined(WEBRTC_ENABLED) - _audio->processWebrtcFarEnd(scratchBuffer, framesPopped, OUTPUT_CHANNEL_COUNT, _audio->_outputFormat.sampleRate()); + if (_audio->_isAECEnabled) { + _audio->processWebrtcFarEnd(scratchBuffer, framesPopped, OUTPUT_CHANNEL_COUNT, _audio->_outputFormat.sampleRate()); + } #endif // if required, upmix or downmix to deviceChannelCount From e5ad87925e9135ee85abba14bb47b72789dabaa2 Mon Sep 17 00:00:00 2001 From: Seth Alves Date: Sat, 27 Jul 2019 09:26:03 -0700 Subject: [PATCH 11/13] update webrtc library for android --- hifi_android.py | 3 +-- 1 file changed, 1 insertion(+), 2 deletions(-) diff --git a/hifi_android.py b/hifi_android.py index 926a671a6c..0c2ea07cc7 100644 --- a/hifi_android.py +++ b/hifi_android.py @@ -97,8 +97,7 @@ ANDROID_PACKAGES = { }, 'webrtc': { 'file': 'webrtc-20190626-android.tar.gz', - 'checksum': '8be046cf27e1fabff8a1029eabbe52c8', - 'versionId': '8TU1LgZswAgZpBiNYvB5s3ZLwSqNJHuj', + 'checksum': 'e2dccd3d8efdcba6d428c87ba7fb2a53' } } From d8b940cc9624872deffeaa04b11ba51d89baf97d Mon Sep 17 00:00:00 2001 From: Seth Alves Date: Mon, 29 Jul 2019 10:35:44 -0700 Subject: [PATCH 12/13] add control to enable/disable acoustic echo cancellation to simplified UI --- .../qml/hifi/simplifiedUI/settingsApp/audio/Audio.qml | 11 +++++++++++ 1 file changed, 11 insertions(+) diff --git a/interface/resources/qml/hifi/simplifiedUI/settingsApp/audio/Audio.qml b/interface/resources/qml/hifi/simplifiedUI/settingsApp/audio/Audio.qml index 8827bb3834..bfc0bc5200 100644 --- a/interface/resources/qml/hifi/simplifiedUI/settingsApp/audio/Audio.qml +++ b/interface/resources/qml/hifi/simplifiedUI/settingsApp/audio/Audio.qml @@ -222,6 +222,17 @@ Flickable { } } } + + SimplifiedControls.Switch { + id: acousticEchoCancellationSwitch + Layout.preferredHeight: 18 + Layout.preferredWidth: parent.width + labelTextOn: "Acoustic Echo Cancellation" + checked: AudioScriptingInterface.acousticEchoCancellation + onClicked: { + AudioScriptingInterface.acousticEchoCancellation = !AudioScriptingInterface.acousticEchoCancellation; + } + } } } From 27b574cf14cb9b2a53026b61dde3c87596358c41 Mon Sep 17 00:00:00 2001 From: Seth Alves Date: Tue, 30 Jul 2019 14:09:04 -0700 Subject: [PATCH 13/13] disable webrtc for android --- cmake/macros/TargetWebRTC.cmake | 17 +++++++++-------- cmake/ports/hifi-deps/CONTROL | 2 +- libraries/audio-client/CMakeLists.txt | 5 ++++- libraries/shared/src/shared/WebRTC.h | 5 +++-- 4 files changed, 17 insertions(+), 12 deletions(-) diff --git a/cmake/macros/TargetWebRTC.cmake b/cmake/macros/TargetWebRTC.cmake index fd5db5fcbe..d2821528df 100644 --- a/cmake/macros/TargetWebRTC.cmake +++ b/cmake/macros/TargetWebRTC.cmake @@ -6,18 +6,19 @@ # macro(TARGET_WEBRTC) if (ANDROID) - include(SelectLibraryConfigurations) - set(INSTALL_DIR ${HIFI_ANDROID_PRECOMPILED}/webrtc/webrtc) - set(WEBRTC_INCLUDE_DIRS "${INSTALL_DIR}/include/webrtc") - set(WEBRTC_LIBRARY_DEBUG ${INSTALL_DIR}/debug/lib/libwebrtc.a) - set(WEBRTC_LIBRARY_RELEASE ${INSTALL_DIR}/lib/libwebrtc.a) - select_library_configurations(WEBRTC) + # I don't yet have working libwebrtc for android + # include(SelectLibraryConfigurations) + # set(INSTALL_DIR ${HIFI_ANDROID_PRECOMPILED}/webrtc/webrtc) + # set(WEBRTC_INCLUDE_DIRS "${INSTALL_DIR}/include/webrtc") + # set(WEBRTC_LIBRARY_DEBUG ${INSTALL_DIR}/debug/lib/libwebrtc.a) + # set(WEBRTC_LIBRARY_RELEASE ${INSTALL_DIR}/lib/libwebrtc.a) + # select_library_configurations(WEBRTC) else() set(WEBRTC_INCLUDE_DIRS "${VCPKG_INSTALL_ROOT}/include/webrtc") find_library(WEBRTC_LIBRARY NAMES webrtc PATHS ${VCPKG_INSTALL_ROOT}/lib/ NO_DEFAULT_PATH) + target_include_directories(${TARGET_NAME} SYSTEM PUBLIC ${WEBRTC_INCLUDE_DIRS}) + target_link_libraries(${TARGET_NAME} ${WEBRTC_LIBRARY}) endif() - target_include_directories(${TARGET_NAME} SYSTEM PUBLIC ${WEBRTC_INCLUDE_DIRS}) - target_link_libraries(${TARGET_NAME} ${WEBRTC_LIBRARY}) endmacro() diff --git a/cmake/ports/hifi-deps/CONTROL b/cmake/ports/hifi-deps/CONTROL index 0a3d194ea1..2441de9002 100644 --- a/cmake/ports/hifi-deps/CONTROL +++ b/cmake/ports/hifi-deps/CONTROL @@ -1,4 +1,4 @@ Source: hifi-deps Version: 0.1 Description: Collected dependencies for High Fidelity applications -Build-Depends: bullet3, draco, etc2comp, glm, nvtt, openexr (!android), openssl (windows), tbb (!android&!osx), zlib, webrtc +Build-Depends: bullet3, draco, etc2comp, glm, nvtt, openexr (!android), openssl (windows), tbb (!android&!osx), zlib, webrtc (!android) diff --git a/libraries/audio-client/CMakeLists.txt b/libraries/audio-client/CMakeLists.txt index 48d3d74cd6..6b88292dd4 100644 --- a/libraries/audio-client/CMakeLists.txt +++ b/libraries/audio-client/CMakeLists.txt @@ -7,7 +7,10 @@ link_hifi_libraries(audio plugins) include_hifi_library_headers(shared) include_hifi_library_headers(networking) -target_webrtc() +if (ANDROID) +else () + target_webrtc() +endif () # append audio includes to our list of includes to bubble target_include_directories(${TARGET_NAME} PUBLIC "${HIFI_LIBRARY_DIR}/audio/src") diff --git a/libraries/shared/src/shared/WebRTC.h b/libraries/shared/src/shared/WebRTC.h index 6f05d38a3d..2f0e444bff 100644 --- a/libraries/shared/src/shared/WebRTC.h +++ b/libraries/shared/src/shared/WebRTC.h @@ -20,8 +20,9 @@ # define NOMINMAX 1 # define WIN32_LEAN_AND_MEAN 1 #elif defined(Q_OS_ANDROID) -# define WEBRTC_ENABLED 1 -# define WEBRTC_POSIX 1 +// I don't yet have a working libwebrtc for android +// # define WEBRTC_ENABLED 1 +// # define WEBRTC_POSIX 1 #elif defined(Q_OS_LINUX) # define WEBRTC_ENABLED 1 # define WEBRTC_POSIX 1