From 724df38df0ffb6140c80de04f2aec4ef1fa67594 Mon Sep 17 00:00:00 2001 From: Ken Cooke Date: Tue, 13 Sep 2016 16:11:06 -0700 Subject: [PATCH 01/10] Optimize the audio pipeline. Use float mixBuffer and apply reverb at 24khz --- libraries/audio-client/src/AudioClient.cpp | 81 +++++++++------------- libraries/audio-client/src/AudioClient.h | 4 +- 2 files changed, 35 insertions(+), 50 deletions(-) diff --git a/libraries/audio-client/src/AudioClient.cpp b/libraries/audio-client/src/AudioClient.cpp index f96171af37..c681069516 100644 --- a/libraries/audio-client/src/AudioClient.cpp +++ b/libraries/audio-client/src/AudioClient.cpp @@ -683,8 +683,8 @@ bool AudioClient::switchOutputToAudioDevice(const QString& outputDeviceName) { void AudioClient::configureReverb() { ReverbParameters p; - p.sampleRate = _outputFormat.sampleRate(); + p.sampleRate = AudioConstants::SAMPLE_RATE; p.bandwidth = _reverbOptions->getBandwidth(); p.preDelay = _reverbOptions->getPreDelay(); p.lateDelay = _reverbOptions->getLateDelay(); @@ -710,6 +710,7 @@ void AudioClient::configureReverb() { _listenerReverb.setParameters(&p); // used only for adding self-reverb to loopback audio + p.sampleRate = _outputFormat.sampleRate(); p.wetDryMix = 100.0f; p.preDelay = 0.0f; p.earlyGain = -96.0f; // disable ER @@ -958,12 +959,9 @@ void AudioClient::handleRecordedAudioInput(const QByteArray& audio) { emitAudioPacket(encodedBuffer.data(), encodedBuffer.size(), _outgoingAvatarAudioSequenceNumber, audioTransform, PacketType::MicrophoneAudioWithEcho, _selectedCodecName); } -void AudioClient::mixLocalAudioInjectors(int16_t* inputBuffer) { +void AudioClient::mixLocalAudioInjectors(float* mixBuffer) { - memset(_hrtfBuffer, 0, sizeof(_hrtfBuffer)); QVector injectorsToRemove; - static const float INT16_TO_FLOAT_SCALE_FACTOR = 1/32768.0f; - bool injectorsHaveData = false; // lock the injector vector @@ -972,9 +970,7 @@ void AudioClient::mixLocalAudioInjectors(int16_t* inputBuffer) { for (AudioInjector* injector : getActiveLocalAudioInjectors()) { if (injector->getLocalBuffer()) { - qint64 samplesToRead = injector->isStereo() ? - AudioConstants::NETWORK_FRAME_BYTES_STEREO : - AudioConstants::NETWORK_FRAME_BYTES_PER_CHANNEL; + qint64 samplesToRead = injector->isStereo() ? AudioConstants::NETWORK_FRAME_BYTES_STEREO : AudioConstants::NETWORK_FRAME_BYTES_PER_CHANNEL; // get one frame from the injector (mono or stereo) memset(_scratchBuffer, 0, sizeof(_scratchBuffer)); @@ -982,9 +978,11 @@ void AudioClient::mixLocalAudioInjectors(int16_t* inputBuffer) { injectorsHaveData = true; - if (injector->isStereo() ) { - for(int i=0; iisStereo()) { + + // stereo gets directly mixed into mixBuffer + for (int i = 0; i < AudioConstants::NETWORK_FRAME_SAMPLES_STEREO; i++) { + mixBuffer[i] += (float)_scratchBuffer[i] * (1/32768.0f); } } else { @@ -995,7 +993,8 @@ void AudioClient::mixLocalAudioInjectors(int16_t* inputBuffer) { float gain = gainForSource(distance, injector->getVolume()); float azimuth = azimuthForSource(relativePosition); - injector->getLocalHRTF().render(_scratchBuffer, _hrtfBuffer, 1, azimuth, distance, gain, AudioConstants::NETWORK_FRAME_SAMPLES_PER_CHANNEL); + // mono gets spatialized into mixBuffer + injector->getLocalHRTF().render(_scratchBuffer, mixBuffer, 1, azimuth, distance, gain, AudioConstants::NETWORK_FRAME_SAMPLES_PER_CHANNEL); } } else { @@ -1013,56 +1012,42 @@ void AudioClient::mixLocalAudioInjectors(int16_t* inputBuffer) { } } - if(injectorsHaveData) { - - // mix network into the hrtfBuffer - for(int i=0; i(outputBuffer.data()); - QByteArray decodedBufferCopy = decodedBuffer; + const int16_t* decodedSamples = reinterpret_cast(decodedBuffer.data()); assert(decodedBuffer.size() == AudioConstants::NETWORK_FRAME_BYTES_STEREO); - - if(getActiveLocalAudioInjectors().size() > 0) { - mixLocalAudioInjectors((int16_t*)decodedBufferCopy.data()); - decodedSamples = reinterpret_cast(decodedBufferCopy.data()); - } else { - decodedSamples = reinterpret_cast(decodedBuffer.data()); + + outputBuffer.resize(_outputFrameSize * sizeof(int16_t)); + int16_t* outputSamples = reinterpret_cast(outputBuffer.data()); + + // convert network audio to float + for (int i = 0; i < AudioConstants::NETWORK_FRAME_SAMPLES_STEREO; i++) { + _mixBuffer[i] = (float)decodedSamples[i] * (1/32768.0f); + } + + // mix in active injectors + if (getActiveLocalAudioInjectors().size() > 0) { + mixLocalAudioInjectors(_mixBuffer); } - // copy the packet from the RB to the output - possibleResampling(_networkToOutputResampler, decodedSamples, outputSamples, - numDecodecSamples, numDeviceOutputSamples, - _desiredOutputFormat, _outputFormat); - - // apply stereo reverb at the listener, to the received audio + // apply stereo reverb bool hasReverb = _reverb || _receivedAudioStream.hasReverb(); if (hasReverb) { - assert(_outputFormat.channelCount() == 2); updateReverbOptions(); - _listenerReverb.render(outputSamples, outputSamples, numDeviceOutputSamples/2); + _listenerReverb.render(_mixBuffer, _mixBuffer, AudioConstants::NETWORK_FRAME_SAMPLES_PER_CHANNEL); } + + // apply peak limiter + _audioLimiter.render(_mixBuffer, _scratchBuffer, AudioConstants::NETWORK_FRAME_SAMPLES_PER_CHANNEL); + + // resample to output sample rate + _networkToOutputResampler->render(_scratchBuffer, outputSamples, AudioConstants::NETWORK_FRAME_SAMPLES_PER_CHANNEL); } void AudioClient::sendMuteEnvironmentPacket() { diff --git a/libraries/audio-client/src/AudioClient.h b/libraries/audio-client/src/AudioClient.h index ba6b98e1bd..53adba4a0d 100644 --- a/libraries/audio-client/src/AudioClient.h +++ b/libraries/audio-client/src/AudioClient.h @@ -227,7 +227,7 @@ protected: private: void outputFormatChanged(); - void mixLocalAudioInjectors(int16_t* inputBuffer); + void mixLocalAudioInjectors(float* mixBuffer); float azimuthForSource(const glm::vec3& relativePosition); float gainForSource(float distance, float volume); @@ -309,7 +309,7 @@ private: AudioSRC* _networkToOutputResampler; // for local hrtf-ing - float _hrtfBuffer[AudioConstants::NETWORK_FRAME_SAMPLES_STEREO]; + float _mixBuffer[AudioConstants::NETWORK_FRAME_SAMPLES_STEREO]; int16_t _scratchBuffer[AudioConstants::NETWORK_FRAME_SAMPLES_STEREO]; AudioLimiter _audioLimiter; From 3a41b285a0d1bc715a216a309e02366771e21d31 Mon Sep 17 00:00:00 2001 From: Ken Cooke Date: Tue, 13 Sep 2016 17:21:48 -0700 Subject: [PATCH 02/10] Fix local audio injectors, that got disabled somehow --- libraries/audio/src/AudioInjector.cpp | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/libraries/audio/src/AudioInjector.cpp b/libraries/audio/src/AudioInjector.cpp index 5fc07adb8d..c6368259c0 100644 --- a/libraries/audio/src/AudioInjector.cpp +++ b/libraries/audio/src/AudioInjector.cpp @@ -26,7 +26,7 @@ #include "SoundCache.h" #include "AudioSRC.h" -//int audioInjectorPtrMetaTypeId = qRegisterMetaType(); +int audioInjectorPtrMetaTypeId = qRegisterMetaType(); AudioInjectorState operator& (AudioInjectorState lhs, AudioInjectorState rhs) { return static_cast(static_cast(lhs) & static_cast(rhs)); From f1455ab1576e0db659621bf9da5624373b8e5e54 Mon Sep 17 00:00:00 2001 From: Ken Cooke Date: Tue, 13 Sep 2016 20:06:44 -0700 Subject: [PATCH 03/10] Remove unneeded buffers --- libraries/audio-client/src/AudioClient.cpp | 3 --- libraries/audio-client/src/AudioClient.h | 2 -- 2 files changed, 5 deletions(-) diff --git a/libraries/audio-client/src/AudioClient.cpp b/libraries/audio-client/src/AudioClient.cpp index c681069516..4000a048ca 100644 --- a/libraries/audio-client/src/AudioClient.cpp +++ b/libraries/audio-client/src/AudioClient.cpp @@ -146,9 +146,6 @@ AudioClient::AudioClient() : _positionGetter(DEFAULT_POSITION_GETTER), _orientationGetter(DEFAULT_ORIENTATION_GETTER) { - // clear the array of locally injected samples - memset(_localProceduralSamples, 0, AudioConstants::NETWORK_FRAME_BYTES_PER_CHANNEL); - connect(&_receivedAudioStream, &MixedProcessedAudioStream::processSamples, this, &AudioClient::processReceivedSamples, Qt::DirectConnection); connect(this, &AudioClient::changeDevice, this, [=](const QAudioDeviceInfo& outputDeviceInfo) { switchOutputToAudioDevice(outputDeviceInfo); }); diff --git a/libraries/audio-client/src/AudioClient.h b/libraries/audio-client/src/AudioClient.h index 53adba4a0d..4a75b38bbb 100644 --- a/libraries/audio-client/src/AudioClient.h +++ b/libraries/audio-client/src/AudioClient.h @@ -259,12 +259,10 @@ private: QAudioFormat _inputFormat; QIODevice* _inputDevice; int _numInputCallbackBytes; - int16_t _localProceduralSamples[AudioConstants::NETWORK_FRAME_SAMPLES_PER_CHANNEL]; QAudioOutput* _audioOutput; QAudioFormat _desiredOutputFormat; QAudioFormat _outputFormat; int _outputFrameSize; - int16_t _outputProcessingBuffer[AudioConstants::NETWORK_FRAME_SAMPLES_STEREO]; int _numOutputCallbackBytes; QAudioOutput* _loopbackAudioOutput; QIODevice* _loopbackOutputDevice; From ed9061040a82054b13e02e54e1b9447a944df62f Mon Sep 17 00:00:00 2001 From: Ken Cooke Date: Tue, 13 Sep 2016 20:16:23 -0700 Subject: [PATCH 04/10] Better Android settings --- libraries/audio-client/src/AudioClient.cpp | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/libraries/audio-client/src/AudioClient.cpp b/libraries/audio-client/src/AudioClient.cpp index 4000a048ca..b572664390 100644 --- a/libraries/audio-client/src/AudioClient.cpp +++ b/libraries/audio-client/src/AudioClient.cpp @@ -371,7 +371,8 @@ bool adjustedFormatForAudioDevice(const QAudioDeviceInfo& audioDevice, adjustedAudioFormat = desiredAudioFormat; #ifdef Q_OS_ANDROID - adjustedAudioFormat.setSampleRate(44100); + // FIXME: query the native sample rate of the device? + adjustedAudioFormat.setSampleRate(48000); #else // From 27d24eb79b1b41ab2b5398a875a93e236f3f77ea Mon Sep 17 00:00:00 2001 From: Ken Cooke Date: Tue, 13 Sep 2016 20:56:36 -0700 Subject: [PATCH 05/10] Fix compiler warning --- libraries/audio-client/src/AudioClient.cpp | 3 --- 1 file changed, 3 deletions(-) diff --git a/libraries/audio-client/src/AudioClient.cpp b/libraries/audio-client/src/AudioClient.cpp index b572664390..a6b0e28a76 100644 --- a/libraries/audio-client/src/AudioClient.cpp +++ b/libraries/audio-client/src/AudioClient.cpp @@ -960,7 +960,6 @@ void AudioClient::handleRecordedAudioInput(const QByteArray& audio) { void AudioClient::mixLocalAudioInjectors(float* mixBuffer) { QVector injectorsToRemove; - bool injectorsHaveData = false; // lock the injector vector Lock lock(_injectorsMutex); @@ -974,8 +973,6 @@ void AudioClient::mixLocalAudioInjectors(float* mixBuffer) { memset(_scratchBuffer, 0, sizeof(_scratchBuffer)); if (0 < injector->getLocalBuffer()->readData((char*)_scratchBuffer, samplesToRead)) { - injectorsHaveData = true; - if (injector->isStereo()) { // stereo gets directly mixed into mixBuffer From c555e369c851121fc27e403263cd8a1c33a082eb Mon Sep 17 00:00:00 2001 From: Ken Cooke Date: Wed, 14 Sep 2016 06:37:09 -0700 Subject: [PATCH 06/10] Handle the case when resampling is not needed --- libraries/audio-client/src/AudioClient.cpp | 14 ++++++++++---- 1 file changed, 10 insertions(+), 4 deletions(-) diff --git a/libraries/audio-client/src/AudioClient.cpp b/libraries/audio-client/src/AudioClient.cpp index a6b0e28a76..8ce442b3ca 100644 --- a/libraries/audio-client/src/AudioClient.cpp +++ b/libraries/audio-client/src/AudioClient.cpp @@ -1038,11 +1038,17 @@ void AudioClient::processReceivedSamples(const QByteArray& decodedBuffer, QByteA _listenerReverb.render(_mixBuffer, _mixBuffer, AudioConstants::NETWORK_FRAME_SAMPLES_PER_CHANNEL); } - // apply peak limiter - _audioLimiter.render(_mixBuffer, _scratchBuffer, AudioConstants::NETWORK_FRAME_SAMPLES_PER_CHANNEL); + if (_networkToOutputResampler) { - // resample to output sample rate - _networkToOutputResampler->render(_scratchBuffer, outputSamples, AudioConstants::NETWORK_FRAME_SAMPLES_PER_CHANNEL); + // resample to output sample rate + _audioLimiter.render(_mixBuffer, _scratchBuffer, AudioConstants::NETWORK_FRAME_SAMPLES_PER_CHANNEL); + _networkToOutputResampler->render(_scratchBuffer, outputSamples, AudioConstants::NETWORK_FRAME_SAMPLES_PER_CHANNEL); + + } else { + + // no resampling needed + _audioLimiter.render(_mixBuffer, outputSamples, AudioConstants::NETWORK_FRAME_SAMPLES_PER_CHANNEL); + } } void AudioClient::sendMuteEnvironmentPacket() { From 96313a421c8ca91664300f22eda9644617062003 Mon Sep 17 00:00:00 2001 From: Ken Cooke Date: Wed, 14 Sep 2016 08:22:18 -0700 Subject: [PATCH 07/10] Cleanup the confusion of using null-resampler for channel conversion --- libraries/audio-client/src/AudioClient.cpp | 11 +++++------ 1 file changed, 5 insertions(+), 6 deletions(-) diff --git a/libraries/audio-client/src/AudioClient.cpp b/libraries/audio-client/src/AudioClient.cpp index 8ce442b3ca..1c4cde8204 100644 --- a/libraries/audio-client/src/AudioClient.cpp +++ b/libraries/audio-client/src/AudioClient.cpp @@ -823,15 +823,14 @@ void AudioClient::handleLocalEchoAndReverb(QByteArray& inputByteArray) { int16_t* inputSamples = reinterpret_cast(inputByteArray.data()); int16_t* loopbackSamples = reinterpret_cast(loopBackByteArray.data()); - auto NO_RESAMPLER = nullptr; - possibleResampling(NO_RESAMPLER, - inputSamples, loopbackSamples, - numInputSamples, numLoopbackSamples, - _inputFormat, _outputFormat); + // upmix mono to stereo + if (!sampleChannelConversion(inputSamples, loopbackSamples, numInputSamples, _inputFormat, _outputFormat)) { + // no conversion, just copy the samples + memcpy(loopbackSamples, inputSamples, numInputSamples * sizeof(int16_t)); + } // apply stereo reverb at the source, to the loopback audio if (!_shouldEchoLocally && hasReverb) { - assert(_outputFormat.channelCount() == 2); updateReverbOptions(); _sourceReverb.render(loopbackSamples, loopbackSamples, numLoopbackSamples/2); } From 25920732ec5585dd39b7a093758c5aba4b7d0f02 Mon Sep 17 00:00:00 2001 From: Ken Cooke Date: Wed, 14 Sep 2016 08:23:53 -0700 Subject: [PATCH 08/10] Remove unused buffer --- libraries/audio-client/src/AudioClient.h | 1 - 1 file changed, 1 deletion(-) diff --git a/libraries/audio-client/src/AudioClient.h b/libraries/audio-client/src/AudioClient.h index 4a75b38bbb..36cda8050e 100644 --- a/libraries/audio-client/src/AudioClient.h +++ b/libraries/audio-client/src/AudioClient.h @@ -253,7 +253,6 @@ private: Gate _gate; Mutex _injectorsMutex; - QByteArray firstInputFrame; QAudioInput* _audioInput; QAudioFormat _desiredInputFormat; QAudioFormat _inputFormat; From e82d751dace2b93b9cc99d6601954557c568b015 Mon Sep 17 00:00:00 2001 From: Ken Cooke Date: Wed, 14 Sep 2016 08:25:41 -0700 Subject: [PATCH 09/10] Fix typo --- libraries/audio-client/src/AudioClient.cpp | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/libraries/audio-client/src/AudioClient.cpp b/libraries/audio-client/src/AudioClient.cpp index 1c4cde8204..0ad77f996f 100644 --- a/libraries/audio-client/src/AudioClient.cpp +++ b/libraries/audio-client/src/AudioClient.cpp @@ -927,12 +927,12 @@ void AudioClient::handleAudioInput() { audioTransform.setRotation(_orientationGetter()); // FIXME find a way to properly handle both playback audio and user audio concurrently - QByteArray decocedBuffer(reinterpret_cast(networkAudioSamples), numNetworkBytes); + QByteArray decodedBuffer(reinterpret_cast(networkAudioSamples), numNetworkBytes); QByteArray encodedBuffer; if (_encoder) { - _encoder->encode(decocedBuffer, encodedBuffer); + _encoder->encode(decodedBuffer, encodedBuffer); } else { - encodedBuffer = decocedBuffer; + encodedBuffer = decodedBuffer; } emitAudioPacket(encodedBuffer.constData(), encodedBuffer.size(), _outgoingAvatarAudioSequenceNumber, audioTransform, packetType, _selectedCodecName); From ad97a15e8b970c56e9aaa31544f044a615854c72 Mon Sep 17 00:00:00 2001 From: Ken Cooke Date: Wed, 14 Sep 2016 09:54:15 -0700 Subject: [PATCH 10/10] Handle another case when resampling is not needed --- libraries/audio-client/src/AudioClient.cpp | 6 +++++- 1 file changed, 5 insertions(+), 1 deletion(-) diff --git a/libraries/audio-client/src/AudioClient.cpp b/libraries/audio-client/src/AudioClient.cpp index 0ad77f996f..99922284dc 100644 --- a/libraries/audio-client/src/AudioClient.cpp +++ b/libraries/audio-client/src/AudioClient.cpp @@ -839,8 +839,12 @@ void AudioClient::handleLocalEchoAndReverb(QByteArray& inputByteArray) { } void AudioClient::handleAudioInput() { + // input samples required to produce exactly NETWORK_FRAME_SAMPLES of output - const int inputSamplesRequired = _inputFormat.channelCount() * _inputToNetworkResampler->getMinInput(AudioConstants::NETWORK_FRAME_SAMPLES_PER_CHANNEL); + const int inputSamplesRequired = (_inputToNetworkResampler ? + _inputToNetworkResampler->getMinInput(AudioConstants::NETWORK_FRAME_SAMPLES_PER_CHANNEL) : + AudioConstants::NETWORK_FRAME_SAMPLES_PER_CHANNEL) * _inputFormat.channelCount(); + const auto inputAudioSamples = std::unique_ptr(new int16_t[inputSamplesRequired]); QByteArray inputByteArray = _inputDevice->readAll();