Mirrors/overte-JulianGro
3
0
Fork 0
mirror of https://github.com/JulianGro/overte.git synced 2025-04-29 20:42:56 +02:00
Code Issues Projects Releases Packages Wiki Activity Actions

Optimize the audio pipeline. Use float mixBuffer and apply reverb at 24khz

Browse source
This commit is contained in:
Ken Cooke 2016-09-13 16:11:06 -07:00
parent 7c3c05a450
commit 724df38df0
2 changed files with 35 additions and 50 deletions
Show stats Download patch file Download diff file Expand all files Collapse all files

81
libraries/audio-client/src/AudioClient.cpp
View file

@ -683,8 +683,8 @@ bool AudioClient::switchOutputToAudioDevice(const QString& outputDeviceName) {
void AudioClient::configureReverb() { void AudioClient::configureReverb() {
ReverbParameters p; ReverbParameters p;
p.sampleRate = _outputFormat.sampleRate();
p.sampleRate = AudioConstants::SAMPLE_RATE;
p.bandwidth = _reverbOptions->getBandwidth(); p.bandwidth = _reverbOptions->getBandwidth();
p.preDelay = _reverbOptions->getPreDelay(); p.preDelay = _reverbOptions->getPreDelay();
p.lateDelay = _reverbOptions->getLateDelay(); p.lateDelay = _reverbOptions->getLateDelay();
@ -710,6 +710,7 @@ void AudioClient::configureReverb() {
_listenerReverb.setParameters(&p); _listenerReverb.setParameters(&p);
// used only for adding self-reverb to loopback audio // used only for adding self-reverb to loopback audio
p.sampleRate = _outputFormat.sampleRate();
p.wetDryMix = 100.0f; p.wetDryMix = 100.0f;
p.preDelay = 0.0f; p.preDelay = 0.0f;
p.earlyGain = -96.0f; // disable ER p.earlyGain = -96.0f; // disable ER
@ -958,12 +959,9 @@ void AudioClient::handleRecordedAudioInput(const QByteArray& audio) {
emitAudioPacket(encodedBuffer.data(), encodedBuffer.size(), _outgoingAvatarAudioSequenceNumber, audioTransform, PacketType::MicrophoneAudioWithEcho, _selectedCodecName); emitAudioPacket(encodedBuffer.data(), encodedBuffer.size(), _outgoingAvatarAudioSequenceNumber, audioTransform, PacketType::MicrophoneAudioWithEcho, _selectedCodecName);
} }
void AudioClient::mixLocalAudioInjectors(int16_t* inputBuffer) { void AudioClient::mixLocalAudioInjectors(float* mixBuffer) {
memset(_hrtfBuffer, 0, sizeof(_hrtfBuffer));
QVector<AudioInjector*> injectorsToRemove; QVector<AudioInjector*> injectorsToRemove;
static const float INT16_TO_FLOAT_SCALE_FACTOR = 1/32768.0f;
bool injectorsHaveData = false; bool injectorsHaveData = false;
// lock the injector vector // lock the injector vector
@ -972,9 +970,7 @@ void AudioClient::mixLocalAudioInjectors(int16_t* inputBuffer) {
for (AudioInjector* injector : getActiveLocalAudioInjectors()) { for (AudioInjector* injector : getActiveLocalAudioInjectors()) {
if (injector->getLocalBuffer()) { if (injector->getLocalBuffer()) {
qint64 samplesToRead = injector->isStereo() ? qint64 samplesToRead = injector->isStereo() ? AudioConstants::NETWORK_FRAME_BYTES_STEREO : AudioConstants::NETWORK_FRAME_BYTES_PER_CHANNEL;
AudioConstants::NETWORK_FRAME_BYTES_STEREO :
AudioConstants::NETWORK_FRAME_BYTES_PER_CHANNEL;
// get one frame from the injector (mono or stereo) // get one frame from the injector (mono or stereo)
memset(_scratchBuffer, 0, sizeof(_scratchBuffer)); memset(_scratchBuffer, 0, sizeof(_scratchBuffer));
@ -982,9 +978,11 @@ void AudioClient::mixLocalAudioInjectors(int16_t* inputBuffer) {
injectorsHaveData = true; injectorsHaveData = true;
if (injector->isStereo() ) { if (injector->isStereo()) {
for(int i=0; i<AudioConstants::NETWORK_FRAME_SAMPLES_STEREO; i++) {
_hrtfBuffer[i] += (float)(_scratchBuffer[i]) * INT16_TO_FLOAT_SCALE_FACTOR; // stereo gets directly mixed into mixBuffer
for (int i = 0; i < AudioConstants::NETWORK_FRAME_SAMPLES_STEREO; i++) {
mixBuffer[i] += (float)_scratchBuffer[i] * (1/32768.0f);
} }
} else { } else {
@ -995,7 +993,8 @@ void AudioClient::mixLocalAudioInjectors(int16_t* inputBuffer) {
float gain = gainForSource(distance, injector->getVolume()); float gain = gainForSource(distance, injector->getVolume());
float azimuth = azimuthForSource(relativePosition); float azimuth = azimuthForSource(relativePosition);
injector->getLocalHRTF().render(_scratchBuffer, _hrtfBuffer, 1, azimuth, distance, gain, AudioConstants::NETWORK_FRAME_SAMPLES_PER_CHANNEL); // mono gets spatialized into mixBuffer
injector->getLocalHRTF().render(_scratchBuffer, mixBuffer, 1, azimuth, distance, gain, AudioConstants::NETWORK_FRAME_SAMPLES_PER_CHANNEL);
} }
} else { } else {
@ -1013,56 +1012,42 @@ void AudioClient::mixLocalAudioInjectors(int16_t* inputBuffer) {
} }
} }
if(injectorsHaveData) { for (AudioInjector* injector : injectorsToRemove) {
// mix network into the hrtfBuffer
for(int i=0; i<AudioConstants::NETWORK_FRAME_SAMPLES_STEREO; i++) {
_hrtfBuffer[i] += (float)(inputBuffer[i]) * INT16_TO_FLOAT_SCALE_FACTOR;
}
// now, use limiter to write back to the inputBuffer
_audioLimiter.render(_hrtfBuffer, inputBuffer, AudioConstants::NETWORK_FRAME_SAMPLES_PER_CHANNEL);
}
for(AudioInjector* injector : injectorsToRemove) {
qDebug() << "removing injector"; qDebug() << "removing injector";
getActiveLocalAudioInjectors().removeOne(injector); getActiveLocalAudioInjectors().removeOne(injector);
} }
} }
void AudioClient::processReceivedSamples(const QByteArray& decodedBuffer, QByteArray& outputBuffer) { void AudioClient::processReceivedSamples(const QByteArray& decodedBuffer, QByteArray& outputBuffer) {
const int numDecodecSamples = decodedBuffer.size() / sizeof(int16_t);
const int numDeviceOutputSamples = _outputFrameSize;
Q_ASSERT(_outputFrameSize == numDecodecSamples * (_outputFormat.sampleRate() * _outputFormat.channelCount()) const int16_t* decodedSamples = reinterpret_cast<const int16_t*>(decodedBuffer.data());
/ (_desiredOutputFormat.sampleRate() * _desiredOutputFormat.channelCount()));
outputBuffer.resize(numDeviceOutputSamples * sizeof(int16_t));
const int16_t* decodedSamples;
int16_t* outputSamples = reinterpret_cast<int16_t*>(outputBuffer.data());
QByteArray decodedBufferCopy = decodedBuffer;
assert(decodedBuffer.size() == AudioConstants::NETWORK_FRAME_BYTES_STEREO); assert(decodedBuffer.size() == AudioConstants::NETWORK_FRAME_BYTES_STEREO);
if(getActiveLocalAudioInjectors().size() > 0) { outputBuffer.resize(_outputFrameSize * sizeof(int16_t));
mixLocalAudioInjectors((int16_t*)decodedBufferCopy.data()); int16_t* outputSamples = reinterpret_cast<int16_t*>(outputBuffer.data());
decodedSamples = reinterpret_cast<const int16_t*>(decodedBufferCopy.data());
} else { // convert network audio to float
decodedSamples = reinterpret_cast<const int16_t*>(decodedBuffer.data()); for (int i = 0; i < AudioConstants::NETWORK_FRAME_SAMPLES_STEREO; i++) {
_mixBuffer[i] = (float)decodedSamples[i] * (1/32768.0f);
}
// mix in active injectors
if (getActiveLocalAudioInjectors().size() > 0) {
mixLocalAudioInjectors(_mixBuffer);
} }
// copy the packet from the RB to the output // apply stereo reverb
possibleResampling(_networkToOutputResampler, decodedSamples, outputSamples,
numDecodecSamples, numDeviceOutputSamples,
_desiredOutputFormat, _outputFormat);
// apply stereo reverb at the listener, to the received audio
bool hasReverb = _reverb || _receivedAudioStream.hasReverb(); bool hasReverb = _reverb || _receivedAudioStream.hasReverb();
if (hasReverb) { if (hasReverb) {
assert(_outputFormat.channelCount() == 2);
updateReverbOptions(); updateReverbOptions();
_listenerReverb.render(outputSamples, outputSamples, numDeviceOutputSamples/2); _listenerReverb.render(_mixBuffer, _mixBuffer, AudioConstants::NETWORK_FRAME_SAMPLES_PER_CHANNEL);
} }
// apply peak limiter
_audioLimiter.render(_mixBuffer, _scratchBuffer, AudioConstants::NETWORK_FRAME_SAMPLES_PER_CHANNEL);
// resample to output sample rate
_networkToOutputResampler->render(_scratchBuffer, outputSamples, AudioConstants::NETWORK_FRAME_SAMPLES_PER_CHANNEL);
} }
void AudioClient::sendMuteEnvironmentPacket() { void AudioClient::sendMuteEnvironmentPacket() {

4
libraries/audio-client/src/AudioClient.h
View file

@ -227,7 +227,7 @@ protected:
private: private:
void outputFormatChanged(); void outputFormatChanged();
void mixLocalAudioInjectors(int16_t* inputBuffer); void mixLocalAudioInjectors(float* mixBuffer);
float azimuthForSource(const glm::vec3& relativePosition); float azimuthForSource(const glm::vec3& relativePosition);
float gainForSource(float distance, float volume); float gainForSource(float distance, float volume);
@ -309,7 +309,7 @@ private:
AudioSRC* _networkToOutputResampler; AudioSRC* _networkToOutputResampler;
// for local hrtf-ing // for local hrtf-ing
float _hrtfBuffer[AudioConstants::NETWORK_FRAME_SAMPLES_STEREO]; float _mixBuffer[AudioConstants::NETWORK_FRAME_SAMPLES_STEREO];
int16_t _scratchBuffer[AudioConstants::NETWORK_FRAME_SAMPLES_STEREO]; int16_t _scratchBuffer[AudioConstants::NETWORK_FRAME_SAMPLES_STEREO];
AudioLimiter _audioLimiter; AudioLimiter _audioLimiter;