From cabe9eab81e2a2d5dba1295d65bfcab0d49d3ca0 Mon Sep 17 00:00:00 2001 From: Ken Cooke Date: Thu, 30 May 2019 08:59:45 -0700 Subject: [PATCH 1/7] Allow local echo when audio input and output devices have different sample rates --- libraries/audio-client/src/AudioClient.cpp | 43 +++++++++++++++------- libraries/audio-client/src/AudioClient.h | 1 + 2 files changed, 30 insertions(+), 14 deletions(-) diff --git a/libraries/audio-client/src/AudioClient.cpp b/libraries/audio-client/src/AudioClient.cpp index ef0e70a31d..ea84d54ecc 100644 --- a/libraries/audio-client/src/AudioClient.cpp +++ b/libraries/audio-client/src/AudioClient.cpp @@ -291,6 +291,7 @@ AudioClient::AudioClient() : _inputToNetworkResampler(NULL), _networkToOutputResampler(NULL), _localToOutputResampler(NULL), + _loopbackResampler(NULL), _audioLimiter(AudioConstants::SAMPLE_RATE, OUTPUT_CHANNEL_COUNT), _outgoingAvatarAudioSequenceNumber(0), _audioOutputIODevice(_localInjectorsStream, _receivedAudioStream, this), @@ -762,6 +763,11 @@ void AudioClient::stop() { qCDebug(audioclient) << "AudioClient::stop(), requesting switchOutputToAudioDevice() to shut down"; switchOutputToAudioDevice(QAudioDeviceInfo(), true); + if (_loopbackResampler) { + delete _loopbackResampler; + _loopbackResampler = NULL; + } + // Stop triggering the checks QObject::disconnect(_checkPeakValuesTimer, &QTimer::timeout, nullptr, nullptr); QObject::disconnect(_checkDevicesTimer, &QTimer::timeout, nullptr, nullptr); @@ -1085,13 +1091,6 @@ void AudioClient::handleLocalEchoAndReverb(QByteArray& inputByteArray) { return; } - // NOTE: we assume the inputFormat and the outputFormat are the same, since on any modern - // multimedia OS they should be. If there is a device that this is not true for, we can - // add back support to do resampling. - if (_inputFormat.sampleRate() != _outputFormat.sampleRate()) { - return; - } - // if this person wants local loopback add that to the locally injected audio // if there is reverb apply it to local audio and substract the origin samples @@ -1108,21 +1107,30 @@ void AudioClient::handleLocalEchoAndReverb(QByteArray& inputByteArray) { } } + // if required, create loopback resampler + if (_inputFormat.sampleRate() != _outputFormat.sampleRate() && !_loopbackResampler) { + qCDebug(audioclient) << "Resampling" << _inputFormat.sampleRate() << "to" << _outputFormat.sampleRate() << "for audio loopback."; + + int channelCount = (_inputFormat.channelCount() == 2 && _outputFormat.channelCount() == 2) ? 2 : 1; + _loopbackResampler = new AudioSRC(_inputFormat.sampleRate(), _outputFormat.sampleRate(), channelCount); + } + static QByteArray loopBackByteArray; int numInputSamples = inputByteArray.size() / AudioConstants::SAMPLE_SIZE; - int numLoopbackSamples = (numInputSamples * OUTPUT_CHANNEL_COUNT) / _inputFormat.channelCount(); + int numInputFrames = numInputSamples / _inputFormat.channelCount(); + int numLoopbackFrames = (numInputFrames * _outputFormat.sampleRate() + _inputFormat.sampleRate() - 1) / _inputFormat.sampleRate(); + int numLoopbackSamples = numLoopbackFrames * OUTPUT_CHANNEL_COUNT; loopBackByteArray.resize(numLoopbackSamples * AudioConstants::SAMPLE_SIZE); int16_t* inputSamples = reinterpret_cast(inputByteArray.data()); int16_t* loopbackSamples = reinterpret_cast(loopBackByteArray.data()); - // upmix mono to stereo - if (!sampleChannelConversion(inputSamples, loopbackSamples, numInputSamples, _inputFormat.channelCount(), OUTPUT_CHANNEL_COUNT)) { - // no conversion, just copy the samples - memcpy(loopbackSamples, inputSamples, numInputSamples * AudioConstants::SAMPLE_SIZE); - } + possibleResampling(_loopbackResampler, + inputSamples, loopbackSamples, + numInputSamples, numLoopbackSamples, + _inputFormat.channelCount(), OUTPUT_CHANNEL_COUNT); // apply stereo reverb at the source, to the loopback audio if (!_shouldEchoLocally && hasReverb) { @@ -1892,15 +1900,22 @@ bool AudioClient::switchOutputToAudioDevice(const QAudioDeviceInfo outputDeviceI _outputDeviceInfo = QAudioDeviceInfo(); } + // cleanup any resamplers if (_networkToOutputResampler) { - // if we were using an input to network resampler, delete it here delete _networkToOutputResampler; _networkToOutputResampler = NULL; + } + if (_localToOutputResampler) { delete _localToOutputResampler; _localToOutputResampler = NULL; } + if (_loopbackResampler) { + delete _loopbackResampler; + _loopbackResampler = NULL; + } + if (isShutdownRequest) { qCDebug(audioclient) << "The audio output device has shut down."; return true; diff --git a/libraries/audio-client/src/AudioClient.h b/libraries/audio-client/src/AudioClient.h index e209628689..decf0f7751 100644 --- a/libraries/audio-client/src/AudioClient.h +++ b/libraries/audio-client/src/AudioClient.h @@ -390,6 +390,7 @@ private: AudioSRC* _inputToNetworkResampler; AudioSRC* _networkToOutputResampler; AudioSRC* _localToOutputResampler; + AudioSRC* _loopbackResampler; // for network audio (used by network audio thread) int16_t _networkScratchBuffer[AudioConstants::NETWORK_FRAME_SAMPLES_AMBISONIC]; From 1572c9362482464c816fc890e1b5617ed92f6b60 Mon Sep 17 00:00:00 2001 From: Ken Cooke Date: Sat, 1 Jun 2019 09:05:06 -0700 Subject: [PATCH 2/7] Fix longstanding bug where resampler called with samples instead of frames, resulting in buffer overflow --- libraries/audio-client/src/AudioClient.cpp | 18 +++++------------- 1 file changed, 5 insertions(+), 13 deletions(-) diff --git a/libraries/audio-client/src/AudioClient.cpp b/libraries/audio-client/src/AudioClient.cpp index ea84d54ecc..d75a4960ed 100644 --- a/libraries/audio-client/src/AudioClient.cpp +++ b/libraries/audio-client/src/AudioClient.cpp @@ -693,28 +693,20 @@ void possibleResampling(AudioSRC* resampler, } } else { + int numSourceFrames = numSourceSamples / sourceChannelCount; + if (sourceChannelCount != destinationChannelCount) { - int numChannelCoversionSamples = (numSourceSamples * destinationChannelCount) / sourceChannelCount; - int16_t* channelConversionSamples = new int16_t[numChannelCoversionSamples]; + int16_t* channelConversionSamples = new int16_t[numSourceFrames * destinationChannelCount]; sampleChannelConversion(sourceSamples, channelConversionSamples, numSourceSamples, sourceChannelCount, destinationChannelCount); - resampler->render(channelConversionSamples, destinationSamples, numChannelCoversionSamples); + resampler->render(channelConversionSamples, destinationSamples, numSourceFrames); delete[] channelConversionSamples; } else { - - unsigned int numAdjustedSourceSamples = numSourceSamples; - unsigned int numAdjustedDestinationSamples = numDestinationSamples; - - if (sourceChannelCount == 2 && destinationChannelCount == 2) { - numAdjustedSourceSamples /= 2; - numAdjustedDestinationSamples /= 2; - } - - resampler->render(sourceSamples, destinationSamples, numAdjustedSourceSamples); + resampler->render(sourceSamples, destinationSamples, numSourceFrames); } } } From f7236f0498e66685457564b3393639d604cb63dc Mon Sep 17 00:00:00 2001 From: Ken Cooke Date: Sat, 1 Jun 2019 10:43:34 -0700 Subject: [PATCH 3/7] Fix bug in local echo when output is multichannel --- libraries/audio-client/src/AudioClient.cpp | 6 ++---- 1 file changed, 2 insertions(+), 4 deletions(-) diff --git a/libraries/audio-client/src/AudioClient.cpp b/libraries/audio-client/src/AudioClient.cpp index d75a4960ed..4f4c5bc586 100644 --- a/libraries/audio-client/src/AudioClient.cpp +++ b/libraries/audio-client/src/AudioClient.cpp @@ -1101,10 +1101,8 @@ void AudioClient::handleLocalEchoAndReverb(QByteArray& inputByteArray) { // if required, create loopback resampler if (_inputFormat.sampleRate() != _outputFormat.sampleRate() && !_loopbackResampler) { - qCDebug(audioclient) << "Resampling" << _inputFormat.sampleRate() << "to" << _outputFormat.sampleRate() << "for audio loopback."; - - int channelCount = (_inputFormat.channelCount() == 2 && _outputFormat.channelCount() == 2) ? 2 : 1; - _loopbackResampler = new AudioSRC(_inputFormat.sampleRate(), _outputFormat.sampleRate(), channelCount); + qCDebug(audioclient) << "Resampling from" << _inputFormat.sampleRate() << "to" << _outputFormat.sampleRate() << "for audio loopback."; + _loopbackResampler = new AudioSRC(_inputFormat.sampleRate(), _outputFormat.sampleRate(), OUTPUT_CHANNEL_COUNT); } static QByteArray loopBackByteArray; From 451cf60e54be6fb7b54bd53947eb2e613f48992d Mon Sep 17 00:00:00 2001 From: Ken Cooke Date: Sat, 1 Jun 2019 11:04:52 -0700 Subject: [PATCH 4/7] Reset loopback resampler when input device is changed --- libraries/audio-client/src/AudioClient.cpp | 12 ++++++------ 1 file changed, 6 insertions(+), 6 deletions(-) diff --git a/libraries/audio-client/src/AudioClient.cpp b/libraries/audio-client/src/AudioClient.cpp index 4f4c5bc586..7c84211c41 100644 --- a/libraries/audio-client/src/AudioClient.cpp +++ b/libraries/audio-client/src/AudioClient.cpp @@ -755,11 +755,6 @@ void AudioClient::stop() { qCDebug(audioclient) << "AudioClient::stop(), requesting switchOutputToAudioDevice() to shut down"; switchOutputToAudioDevice(QAudioDeviceInfo(), true); - if (_loopbackResampler) { - delete _loopbackResampler; - _loopbackResampler = NULL; - } - // Stop triggering the checks QObject::disconnect(_checkPeakValuesTimer, &QTimer::timeout, nullptr, nullptr); QObject::disconnect(_checkDevicesTimer, &QTimer::timeout, nullptr, nullptr); @@ -1663,12 +1658,17 @@ bool AudioClient::switchInputToAudioDevice(const QAudioDeviceInfo inputDeviceInf _dummyAudioInput = NULL; } + // cleanup any resamplers if (_inputToNetworkResampler) { - // if we were using an input to network resampler, delete it here delete _inputToNetworkResampler; _inputToNetworkResampler = NULL; } + if (_loopbackResampler) { + delete _loopbackResampler; + _loopbackResampler = NULL; + } + if (_audioGate) { delete _audioGate; _audioGate = nullptr; From a56942e494355042a4700b7ec3bd91ece3e26bc8 Mon Sep 17 00:00:00 2001 From: Ken Cooke Date: Sat, 1 Jun 2019 14:27:32 -0700 Subject: [PATCH 5/7] Correctly handle time-varying buffer size due to loopback resampling --- libraries/audio-client/src/AudioClient.cpp | 44 ++++++++++++---------- 1 file changed, 25 insertions(+), 19 deletions(-) diff --git a/libraries/audio-client/src/AudioClient.cpp b/libraries/audio-client/src/AudioClient.cpp index 7c84211c41..cb8fbbdb0b 100644 --- a/libraries/audio-client/src/AudioClient.cpp +++ b/libraries/audio-client/src/AudioClient.cpp @@ -657,11 +657,11 @@ bool adjustedFormatForAudioDevice(const QAudioDeviceInfo& audioDevice, return false; // a supported format could not be found } -bool sampleChannelConversion(const int16_t* sourceSamples, int16_t* destinationSamples, unsigned int numSourceSamples, +bool sampleChannelConversion(const int16_t* sourceSamples, int16_t* destinationSamples, int numSourceSamples, const int sourceChannelCount, const int destinationChannelCount) { if (sourceChannelCount == 2 && destinationChannelCount == 1) { // loop through the stereo input audio samples and average every two samples - for (uint i = 0; i < numSourceSamples; i += 2) { + for (int i = 0; i < numSourceSamples; i += 2) { destinationSamples[i / 2] = (sourceSamples[i] / 2) + (sourceSamples[i + 1] / 2); } @@ -669,7 +669,7 @@ bool sampleChannelConversion(const int16_t* sourceSamples, int16_t* destinationS } else if (sourceChannelCount == 1 && destinationChannelCount == 2) { // loop through the mono input audio and repeat each sample twice - for (uint i = 0; i < numSourceSamples; ++i) { + for (int i = 0; i < numSourceSamples; ++i) { destinationSamples[i * 2] = destinationSamples[(i * 2) + 1] = sourceSamples[i]; } @@ -679,10 +679,13 @@ bool sampleChannelConversion(const int16_t* sourceSamples, int16_t* destinationS return false; } -void possibleResampling(AudioSRC* resampler, - const int16_t* sourceSamples, int16_t* destinationSamples, - unsigned int numSourceSamples, unsigned int numDestinationSamples, - const int sourceChannelCount, const int destinationChannelCount) { +int possibleResampling(AudioSRC* resampler, + const int16_t* sourceSamples, int16_t* destinationSamples, + int numSourceSamples, int maxDestinationSamples, + const int sourceChannelCount, const int destinationChannelCount) { + + int numSourceFrames = numSourceSamples / sourceChannelCount; + int numDestinationFrames = 0; if (numSourceSamples > 0) { if (!resampler) { @@ -691,10 +694,8 @@ void possibleResampling(AudioSRC* resampler, // no conversion, we can copy the samples directly across memcpy(destinationSamples, sourceSamples, numSourceSamples * AudioConstants::SAMPLE_SIZE); } + numDestinationFrames = numSourceFrames; } else { - - int numSourceFrames = numSourceSamples / sourceChannelCount; - if (sourceChannelCount != destinationChannelCount) { int16_t* channelConversionSamples = new int16_t[numSourceFrames * destinationChannelCount]; @@ -702,14 +703,17 @@ void possibleResampling(AudioSRC* resampler, sampleChannelConversion(sourceSamples, channelConversionSamples, numSourceSamples, sourceChannelCount, destinationChannelCount); - resampler->render(channelConversionSamples, destinationSamples, numSourceFrames); + numDestinationFrames = resampler->render(channelConversionSamples, destinationSamples, numSourceFrames); delete[] channelConversionSamples; } else { - resampler->render(sourceSamples, destinationSamples, numSourceFrames); + numDestinationFrames = resampler->render(sourceSamples, destinationSamples, numSourceFrames); } } } + + int numDestinationSamples = numDestinationFrames * destinationChannelCount; + return numDestinationSamples; } void AudioClient::start() { @@ -1104,18 +1108,20 @@ void AudioClient::handleLocalEchoAndReverb(QByteArray& inputByteArray) { int numInputSamples = inputByteArray.size() / AudioConstants::SAMPLE_SIZE; int numInputFrames = numInputSamples / _inputFormat.channelCount(); - int numLoopbackFrames = (numInputFrames * _outputFormat.sampleRate() + _inputFormat.sampleRate() - 1) / _inputFormat.sampleRate(); - int numLoopbackSamples = numLoopbackFrames * OUTPUT_CHANNEL_COUNT; + int maxLoopbackFrames = _loopbackResampler ? _loopbackResampler->getMaxOutput(numInputFrames) : numInputFrames; + int maxLoopbackSamples = maxLoopbackFrames * OUTPUT_CHANNEL_COUNT; - loopBackByteArray.resize(numLoopbackSamples * AudioConstants::SAMPLE_SIZE); + loopBackByteArray.resize(maxLoopbackSamples * AudioConstants::SAMPLE_SIZE); int16_t* inputSamples = reinterpret_cast(inputByteArray.data()); int16_t* loopbackSamples = reinterpret_cast(loopBackByteArray.data()); - possibleResampling(_loopbackResampler, - inputSamples, loopbackSamples, - numInputSamples, numLoopbackSamples, - _inputFormat.channelCount(), OUTPUT_CHANNEL_COUNT); + int numLoopbackSamples = possibleResampling(_loopbackResampler, + inputSamples, loopbackSamples, + numInputSamples, maxLoopbackSamples, + _inputFormat.channelCount(), OUTPUT_CHANNEL_COUNT); + + loopBackByteArray.resize(numLoopbackSamples * AudioConstants::SAMPLE_SIZE); // apply stereo reverb at the source, to the loopback audio if (!_shouldEchoLocally && hasReverb) { From 4a294b389945937643d58c525c91de6d355825ce Mon Sep 17 00:00:00 2001 From: Ken Cooke Date: Sat, 1 Jun 2019 14:31:14 -0700 Subject: [PATCH 6/7] Add sanity check for buffer overflow due to resampling --- libraries/audio-client/src/AudioClient.cpp | 4 ++++ 1 file changed, 4 insertions(+) diff --git a/libraries/audio-client/src/AudioClient.cpp b/libraries/audio-client/src/AudioClient.cpp index cb8fbbdb0b..8797b90860 100644 --- a/libraries/audio-client/src/AudioClient.cpp +++ b/libraries/audio-client/src/AudioClient.cpp @@ -713,6 +713,10 @@ int possibleResampling(AudioSRC* resampler, } int numDestinationSamples = numDestinationFrames * destinationChannelCount; + if (numDestinationSamples > maxDestinationSamples) { + qCWarning(audioclient) << "Resampler overflow! numDestinationSamples =" << numDestinationSamples + << "but maxDestinationSamples =" << maxDestinationSamples; + } return numDestinationSamples; } From dfab91e50a75c80493e3994df0792740dcc538a1 Mon Sep 17 00:00:00 2001 From: Ken Cooke Date: Sat, 1 Jun 2019 19:53:18 -0700 Subject: [PATCH 7/7] Reduce the performance impact of loopback resampler (AVX2 optimizations) --- libraries/audio/src/avx2/AudioSRC_avx2.cpp | 36 +++++++++++++++++++--- 1 file changed, 32 insertions(+), 4 deletions(-) diff --git a/libraries/audio/src/avx2/AudioSRC_avx2.cpp b/libraries/audio/src/avx2/AudioSRC_avx2.cpp index 0e31a58ce7..e5ac08746c 100644 --- a/libraries/audio/src/avx2/AudioSRC_avx2.cpp +++ b/libraries/audio/src/avx2/AudioSRC_avx2.cpp @@ -34,15 +34,26 @@ int AudioSRC::multirateFilter1_AVX2(const float* input0, float* output0, int inp const float* c0 = &_polyphaseFilter[_numTaps * _phase]; __m256 acc0 = _mm256_setzero_ps(); + __m256 acc1 = _mm256_setzero_ps(); - for (int j = 0; j < _numTaps; j += 8) { + int j = 0; + for (; j < _numTaps - 15; j += 16) { // unrolled x 2 //float coef = c0[j]; - __m256 coef0 = _mm256_loadu_ps(&c0[j]); + __m256 coef0 = _mm256_loadu_ps(&c0[j + 0]); + __m256 coef1 = _mm256_loadu_ps(&c0[j + 8]); //acc += input[i + j] * coef; + acc0 = _mm256_fmadd_ps(_mm256_loadu_ps(&input0[i + j + 0]), coef0, acc0); + acc1 = _mm256_fmadd_ps(_mm256_loadu_ps(&input0[i + j + 8]), coef1, acc1); + } + if (j < _numTaps) { + + __m256 coef0 = _mm256_loadu_ps(&c0[j]); + acc0 = _mm256_fmadd_ps(_mm256_loadu_ps(&input0[i + j]), coef0, acc0); } + acc0 = _mm256_add_ps(acc0, acc1); // horizontal sum acc0 = _mm256_hadd_ps(acc0, acc0); @@ -73,19 +84,36 @@ int AudioSRC::multirateFilter1_AVX2(const float* input0, float* output0, int inp const float* c1 = &_polyphaseFilter[_numTaps * (phase + 1)]; __m256 acc0 = _mm256_setzero_ps(); + __m256 acc1 = _mm256_setzero_ps(); __m256 frac = _mm256_broadcast_ss(&ftmp); - for (int j = 0; j < _numTaps; j += 8) { + int j = 0; + for (; j < _numTaps - 15; j += 16) { // unrolled x 2 //float coef = c0[j] + frac * (c1[j] - c0[j]); + __m256 coef0 = _mm256_loadu_ps(&c0[j + 0]); + __m256 coef1 = _mm256_loadu_ps(&c1[j + 0]); + __m256 coef2 = _mm256_loadu_ps(&c0[j + 8]); + __m256 coef3 = _mm256_loadu_ps(&c1[j + 8]); + coef1 = _mm256_sub_ps(coef1, coef0); + coef3 = _mm256_sub_ps(coef3, coef2); + coef0 = _mm256_fmadd_ps(coef1, frac, coef0); + coef2 = _mm256_fmadd_ps(coef3, frac, coef2); + + //acc += input[i + j] * coef; + acc0 = _mm256_fmadd_ps(_mm256_loadu_ps(&input0[i + j + 0]), coef0, acc0); + acc1 = _mm256_fmadd_ps(_mm256_loadu_ps(&input0[i + j + 8]), coef2, acc1); + } + if (j < _numTaps) { + __m256 coef0 = _mm256_loadu_ps(&c0[j]); __m256 coef1 = _mm256_loadu_ps(&c1[j]); coef1 = _mm256_sub_ps(coef1, coef0); coef0 = _mm256_fmadd_ps(coef1, frac, coef0); - //acc += input[i + j] * coef; acc0 = _mm256_fmadd_ps(_mm256_loadu_ps(&input0[i + j]), coef0, acc0); } + acc0 = _mm256_add_ps(acc0, acc1); // horizontal sum acc0 = _mm256_hadd_ps(acc0, acc0);