Mirrors/overte-JulianGro
3
0
Fork 0
mirror of https://github.com/JulianGro/overte.git synced 2025-04-07 07:02:46 +02:00
Code Issues Projects Releases Packages Wiki Activity Actions

Merge pull request #15663 from kencooke/audio-samprate-mismatch-fix

Browse source 
BUGZ-328: Some microphones are unable to hear themselves in test mic audio setting
This commit is contained in:
Ryan Huffman 2019-06-03 14:18:27 -07:00 committed by GitHub
commit ce0d2b0628
No known key found for this signature in database
GPG key ID: 4AEE18F83AFDEB23
3 changed files with 85 additions and 41 deletions
Show stats Download patch file Download diff file Expand all files Collapse all files

89
libraries/audio-client/src/AudioClient.cpp
View file

@ -291,6 +291,7 @@ AudioClient::AudioClient() :
_inputToNetworkResampler(NULL),
_networkToOutputResampler(NULL),
_localToOutputResampler(NULL),
_loopbackResampler(NULL),
_audioLimiter(AudioConstants::SAMPLE_RATE, OUTPUT_CHANNEL_COUNT),
_outgoingAvatarAudioSequenceNumber(0),
_audioOutputIODevice(_localInjectorsStream, _receivedAudioStream, this),
@ -656,11 +657,11 @@ bool adjustedFormatForAudioDevice(const QAudioDeviceInfo& audioDevice,
return false; // a supported format could not be found
}
bool sampleChannelConversion(const int16_t* sourceSamples, int16_t* destinationSamples, unsigned int numSourceSamples,
bool sampleChannelConversion(const int16_t* sourceSamples, int16_t* destinationSamples, int numSourceSamples,
const int sourceChannelCount, const int destinationChannelCount) {
if (sourceChannelCount == 2 && destinationChannelCount == 1) {
// loop through the stereo input audio samples and average every two samples
for (uint i = 0; i < numSourceSamples; i += 2) {
for (int i = 0; i < numSourceSamples; i += 2) {
destinationSamples[i / 2] = (sourceSamples[i] / 2) + (sourceSamples[i + 1] / 2);
}
@ -668,7 +669,7 @@ bool sampleChannelConversion(const int16_t* sourceSamples, int16_t* destinationS
} else if (sourceChannelCount == 1 && destinationChannelCount == 2) {
// loop through the mono input audio and repeat each sample twice
for (uint i = 0; i < numSourceSamples; ++i) {
for (int i = 0; i < numSourceSamples; ++i) {
destinationSamples[i * 2] = destinationSamples[(i * 2) + 1] = sourceSamples[i];
}
@ -678,10 +679,13 @@ bool sampleChannelConversion(const int16_t* sourceSamples, int16_t* destinationS
return false;
}
void possibleResampling(AudioSRC* resampler,
const int16_t* sourceSamples, int16_t* destinationSamples,
unsigned int numSourceSamples, unsigned int numDestinationSamples,
const int sourceChannelCount, const int destinationChannelCount) {
int possibleResampling(AudioSRC* resampler,
const int16_t* sourceSamples, int16_t* destinationSamples,
int numSourceSamples, int maxDestinationSamples,
const int sourceChannelCount, const int destinationChannelCount) {
int numSourceFrames = numSourceSamples / sourceChannelCount;
int numDestinationFrames = 0;
if (numSourceSamples > 0) {
if (!resampler) {
@ -690,33 +694,30 @@ void possibleResampling(AudioSRC* resampler,
// no conversion, we can copy the samples directly across
memcpy(destinationSamples, sourceSamples, numSourceSamples * AudioConstants::SAMPLE_SIZE);
}
numDestinationFrames = numSourceFrames;
} else {
if (sourceChannelCount != destinationChannelCount) {
int numChannelCoversionSamples = (numSourceSamples * destinationChannelCount) / sourceChannelCount;
int16_t* channelConversionSamples = new int16_t[numChannelCoversionSamples];
int16_t* channelConversionSamples = new int16_t[numSourceFrames * destinationChannelCount];
sampleChannelConversion(sourceSamples, channelConversionSamples, numSourceSamples,
sourceChannelCount, destinationChannelCount);
resampler->render(channelConversionSamples, destinationSamples, numChannelCoversionSamples);
numDestinationFrames = resampler->render(channelConversionSamples, destinationSamples, numSourceFrames);
delete[] channelConversionSamples;
} else {
unsigned int numAdjustedSourceSamples = numSourceSamples;
unsigned int numAdjustedDestinationSamples = numDestinationSamples;
if (sourceChannelCount == 2 && destinationChannelCount == 2) {
numAdjustedSourceSamples /= 2;
numAdjustedDestinationSamples /= 2;
}
resampler->render(sourceSamples, destinationSamples, numAdjustedSourceSamples);
numDestinationFrames = resampler->render(sourceSamples, destinationSamples, numSourceFrames);
}
}
}
int numDestinationSamples = numDestinationFrames * destinationChannelCount;
if (numDestinationSamples > maxDestinationSamples) {
qCWarning(audioclient) << "Resampler overflow! numDestinationSamples =" << numDestinationSamples
<< "but maxDestinationSamples =" << maxDestinationSamples;
}
return numDestinationSamples;
}
void AudioClient::start() {
@ -1085,13 +1086,6 @@ void AudioClient::handleLocalEchoAndReverb(QByteArray& inputByteArray) {
return;
}
// NOTE: we assume the inputFormat and the outputFormat are the same, since on any modern
// multimedia OS they should be. If there is a device that this is not true for, we can
// add back support to do resampling.
if (_inputFormat.sampleRate() != _outputFormat.sampleRate()) {
return;
}
// if this person wants local loopback add that to the locally injected audio
// if there is reverb apply it to local audio and substract the origin samples
@ -1108,21 +1102,30 @@ void AudioClient::handleLocalEchoAndReverb(QByteArray& inputByteArray) {
}
}
// if required, create loopback resampler
if (_inputFormat.sampleRate() != _outputFormat.sampleRate() && !_loopbackResampler) {
qCDebug(audioclient) << "Resampling from" << _inputFormat.sampleRate() << "to" << _outputFormat.sampleRate() << "for audio loopback.";
_loopbackResampler = new AudioSRC(_inputFormat.sampleRate(), _outputFormat.sampleRate(), OUTPUT_CHANNEL_COUNT);
}
static QByteArray loopBackByteArray;
int numInputSamples = inputByteArray.size() / AudioConstants::SAMPLE_SIZE;
int numLoopbackSamples = (numInputSamples * OUTPUT_CHANNEL_COUNT) / _inputFormat.channelCount();
int numInputFrames = numInputSamples / _inputFormat.channelCount();
int maxLoopbackFrames = _loopbackResampler ? _loopbackResampler->getMaxOutput(numInputFrames) : numInputFrames;
int maxLoopbackSamples = maxLoopbackFrames * OUTPUT_CHANNEL_COUNT;
loopBackByteArray.resize(numLoopbackSamples * AudioConstants::SAMPLE_SIZE);
loopBackByteArray.resize(maxLoopbackSamples * AudioConstants::SAMPLE_SIZE);
int16_t* inputSamples = reinterpret_cast<int16_t*>(inputByteArray.data());
int16_t* loopbackSamples = reinterpret_cast<int16_t*>(loopBackByteArray.data());
// upmix mono to stereo
if (!sampleChannelConversion(inputSamples, loopbackSamples, numInputSamples, _inputFormat.channelCount(), OUTPUT_CHANNEL_COUNT)) {
// no conversion, just copy the samples
memcpy(loopbackSamples, inputSamples, numInputSamples * AudioConstants::SAMPLE_SIZE);
}
int numLoopbackSamples = possibleResampling(_loopbackResampler,
inputSamples, loopbackSamples,
numInputSamples, maxLoopbackSamples,
_inputFormat.channelCount(), OUTPUT_CHANNEL_COUNT);
loopBackByteArray.resize(numLoopbackSamples * AudioConstants::SAMPLE_SIZE);
// apply stereo reverb at the source, to the loopback audio
if (!_shouldEchoLocally && hasReverb) {
@ -1665,12 +1668,17 @@ bool AudioClient::switchInputToAudioDevice(const QAudioDeviceInfo inputDeviceInf
_dummyAudioInput = NULL;
}
// cleanup any resamplers
if (_inputToNetworkResampler) {
// if we were using an input to network resampler, delete it here
delete _inputToNetworkResampler;
_inputToNetworkResampler = NULL;
}
if (_loopbackResampler) {
delete _loopbackResampler;
_loopbackResampler = NULL;
}
if (_audioGate) {
delete _audioGate;
_audioGate = nullptr;
@ -1892,15 +1900,22 @@ bool AudioClient::switchOutputToAudioDevice(const QAudioDeviceInfo outputDeviceI
_outputDeviceInfo = QAudioDeviceInfo();
}
// cleanup any resamplers
if (_networkToOutputResampler) {
// if we were using an input to network resampler, delete it here
delete _networkToOutputResampler;
_networkToOutputResampler = NULL;
}
if (_localToOutputResampler) {
delete _localToOutputResampler;
_localToOutputResampler = NULL;
}
if (_loopbackResampler) {
delete _loopbackResampler;
_loopbackResampler = NULL;
}
if (isShutdownRequest) {
qCDebug(audioclient) << "The audio output device has shut down.";
return true;

1
libraries/audio-client/src/AudioClient.h
View file

@ -390,6 +390,7 @@ private:
AudioSRC* _inputToNetworkResampler;
AudioSRC* _networkToOutputResampler;
AudioSRC* _localToOutputResampler;
AudioSRC* _loopbackResampler;
// for network audio (used by network audio thread)
int16_t _networkScratchBuffer[AudioConstants::NETWORK_FRAME_SAMPLES_AMBISONIC];

36
libraries/audio/src/avx2/AudioSRC_avx2.cpp
View file

@ -34,15 +34,26 @@ int AudioSRC::multirateFilter1_AVX2(const float* input0, float* output0, int inp
const float* c0 = &_polyphaseFilter[_numTaps * _phase];
__m256 acc0 = _mm256_setzero_ps();
__m256 acc1 = _mm256_setzero_ps();
for (int j = 0; j < _numTaps; j += 8) {
int j = 0;
for (; j < _numTaps - 15; j += 16) { // unrolled x 2
//float coef = c0[j];
__m256 coef0 = _mm256_loadu_ps(&c0[j]);
__m256 coef0 = _mm256_loadu_ps(&c0[j + 0]);
__m256 coef1 = _mm256_loadu_ps(&c0[j + 8]);
//acc += input[i + j] * coef;
acc0 = _mm256_fmadd_ps(_mm256_loadu_ps(&input0[i + j + 0]), coef0, acc0);
acc1 = _mm256_fmadd_ps(_mm256_loadu_ps(&input0[i + j + 8]), coef1, acc1);
}
if (j < _numTaps) {
__m256 coef0 = _mm256_loadu_ps(&c0[j]);
acc0 = _mm256_fmadd_ps(_mm256_loadu_ps(&input0[i + j]), coef0, acc0);
}
acc0 = _mm256_add_ps(acc0, acc1);
// horizontal sum
acc0 = _mm256_hadd_ps(acc0, acc0);
@ -73,19 +84,36 @@ int AudioSRC::multirateFilter1_AVX2(const float* input0, float* output0, int inp
const float* c1 = &_polyphaseFilter[_numTaps * (phase + 1)];
__m256 acc0 = _mm256_setzero_ps();
__m256 acc1 = _mm256_setzero_ps();
__m256 frac = _mm256_broadcast_ss(&ftmp);
for (int j = 0; j < _numTaps; j += 8) {
int j = 0;
for (; j < _numTaps - 15; j += 16) { // unrolled x 2
//float coef = c0[j] + frac * (c1[j] - c0[j]);
__m256 coef0 = _mm256_loadu_ps(&c0[j + 0]);
__m256 coef1 = _mm256_loadu_ps(&c1[j + 0]);
__m256 coef2 = _mm256_loadu_ps(&c0[j + 8]);
__m256 coef3 = _mm256_loadu_ps(&c1[j + 8]);
coef1 = _mm256_sub_ps(coef1, coef0);
coef3 = _mm256_sub_ps(coef3, coef2);
coef0 = _mm256_fmadd_ps(coef1, frac, coef0);
coef2 = _mm256_fmadd_ps(coef3, frac, coef2);
//acc += input[i + j] * coef;
acc0 = _mm256_fmadd_ps(_mm256_loadu_ps(&input0[i + j + 0]), coef0, acc0);
acc1 = _mm256_fmadd_ps(_mm256_loadu_ps(&input0[i + j + 8]), coef2, acc1);
}
if (j < _numTaps) {
__m256 coef0 = _mm256_loadu_ps(&c0[j]);
__m256 coef1 = _mm256_loadu_ps(&c1[j]);
coef1 = _mm256_sub_ps(coef1, coef0);
coef0 = _mm256_fmadd_ps(coef1, frac, coef0);
//acc += input[i + j] * coef;
acc0 = _mm256_fmadd_ps(_mm256_loadu_ps(&input0[i + j]), coef0, acc0);
}
acc0 = _mm256_add_ps(acc0, acc1);
// horizontal sum
acc0 = _mm256_hadd_ps(acc0, acc0);