From 80d33ee2516616beb3b08d9279e812686e37a866 Mon Sep 17 00:00:00 2001 From: David Kelly Date: Fri, 8 Jul 2016 13:59:49 -0700 Subject: [PATCH] Working now as frame-at-a-time So localOnly sounds get HRTF'd, one network frame at a time. Less processing (upsampling, limiting, etc...) than doing this at the end of the pipeline (in the AudioOutputIODevice::readData call). --- libraries/audio-client/src/AudioClient.cpp | 199 ++++++++------------- libraries/audio-client/src/AudioClient.h | 20 ++- 2 files changed, 87 insertions(+), 132 deletions(-) diff --git a/libraries/audio-client/src/AudioClient.cpp b/libraries/audio-client/src/AudioClient.cpp index 67fb1ff561..190dd1583b 100644 --- a/libraries/audio-client/src/AudioClient.cpp +++ b/libraries/audio-client/src/AudioClient.cpp @@ -43,10 +43,6 @@ #include #include -#include "AudioInjector.h" -#include "AudioLimiter.h" -#include "AudioHRTF.h" -#include "AudioConstants.h" #include "PositionalAudioStream.h" #include "AudioClientLogging.h" @@ -111,7 +107,8 @@ AudioClient::AudioClient() : _stats(&_receivedAudioStream), _inputGate(), _positionGetter(DEFAULT_POSITION_GETTER), - _orientationGetter(DEFAULT_ORIENTATION_GETTER) + _orientationGetter(DEFAULT_ORIENTATION_GETTER), + _audioLimiter(AudioConstants::SAMPLE_RATE, AudioConstants::STEREO) { // clear the array of locally injected samples memset(_localProceduralSamples, 0, AudioConstants::NETWORK_FRAME_BYTES_PER_CHANNEL); @@ -787,15 +784,84 @@ void AudioClient::handleRecordedAudioInput(const QByteArray& audio) { emitAudioPacket(audio.data(), audio.size(), _outgoingAvatarAudioSequenceNumber, audioTransform, PacketType::MicrophoneAudioWithEcho); } +void AudioClient::mixLocalAudioInjectors(int16_t* inputBuffer) { + + memset(_hrtfBuffer, 0, sizeof(_hrtfBuffer)); + QVector injectorsToRemove; + + bool injectorsHaveData = false; + + for (AudioInjector* injector : getActiveLocalAudioInjectors()) { + if (injector->getLocalBuffer()) { + + // get one (mono) frame from the injector + if (0 < injector->getLocalBuffer()->readData((char*)_scratchBuffer, AudioConstants::NETWORK_FRAME_SAMPLES_PER_CHANNEL*sizeof(int16_t)) ) { + + injectorsHaveData = true; + + // calculate gain and azimuth for hrtf + glm::vec3 relativePosition = injector->getPosition() - _positionGetter(); + float gain = gainForSource(relativePosition, injector->getVolume()); + float azimuth = azimuthForSource(relativePosition); + + + injector->getLocalHRTF().render(_scratchBuffer, _hrtfBuffer, 1, azimuth, gain, AudioConstants::NETWORK_FRAME_SAMPLES_PER_CHANNEL); + + } else { + + qDebug() << "injector has no more data, marking finished for removal"; + + injector->finish(); + injectorsToRemove.append(injector); + } + + } else { + + qDebug() << "injector has no local buffer, marking as finished for removal"; + + injector->finish(); + injectorsToRemove.append(injector); + } + } + + if(injectorsHaveData) { + + // mix network into the hrtfBuffer + for(int i=0; i(inputBuffer.data()); int16_t* outputSamples = reinterpret_cast(outputBuffer.data()); + const int16_t* receivedSamples; + char inputBufferCopy[AudioConstants::NETWORK_FRAME_BYTES_STEREO]; + assert(inputBuffer.size() == AudioConstants::NETWORK_FRAME_BYTES_STEREO); + + if(getActiveLocalAudioInjectors().size() > 0) { + // gotta copy the since it is const + for(int i = 0; i < sizeof(inputBufferCopy); i++) { + inputBufferCopy[i] = inputBuffer.data()[i]; + } + mixLocalAudioInjectors((int16_t*)inputBufferCopy); + receivedSamples = reinterpret_cast(inputBufferCopy); + } else { + receivedSamples = reinterpret_cast(inputBuffer.data()); + } // copy the packet from the RB to the output possibleResampling(_networkToOutputResampler, receivedSamples, outputSamples, @@ -807,6 +873,7 @@ void AudioClient::processReceivedSamples(const QByteArray& inputBuffer, QByteArr if (hasReverb) { assert(_outputFormat.channelCount() == 2); updateReverbOptions(); + qDebug() << "handling reverb"; _listenerReverb.render(outputSamples, outputSamples, numDeviceOutputSamples/2); } } @@ -1148,22 +1215,6 @@ float AudioClient::getAudioOutputMsecsUnplayed() const { return msecsAudioOutputUnplayed; } -void AudioClient::AudioOutputIODevice::renderHRTF(AudioHRTF& hrtf, int16_t* data, float* hrtfBuffer, float azimuth, float gain, qint64 numSamples) { - qint64 numFrames = numSamples/AudioConstants::NETWORK_FRAME_SAMPLES_PER_CHANNEL; - int16_t* dataPtr = data; - float* hrtfPtr = hrtfBuffer; - for(qint64 i=0; i < numFrames; i++) { - hrtf.render(dataPtr, hrtfPtr, 1, azimuth, gain, AudioConstants::NETWORK_FRAME_SAMPLES_PER_CHANNEL); - - //qDebug() << "processed frame " << i; - - dataPtr += AudioConstants::NETWORK_FRAME_SAMPLES_PER_CHANNEL; - hrtfPtr += 2*AudioConstants::NETWORK_FRAME_SAMPLES_PER_CHANNEL; - - } - assert(dataPtr - data <= numSamples); - assert(hrtfPtr - hrtfBuffer <= 2*numSamples); -} float AudioClient::azimuthForSource(const glm::vec3& relativePosition) { // copied from AudioMixer, more or less @@ -1226,105 +1277,11 @@ qint64 AudioClient::AudioOutputIODevice::readData(char * data, qint64 maxSize) { auto samplesRequested = maxSize / sizeof(int16_t); int samplesPopped; int bytesWritten; - - // limit the number of NETWORK_FRAME_SAMPLES_PER_CHANNEL to return regardless - // of what maxSize requests. - static const qint64 MAX_FRAMES = 256; - static float hrtfBuffer[AudioConstants::NETWORK_FRAME_SAMPLES_PER_CHANNEL*MAX_FRAMES]; - static float mixed48Khz[AudioConstants::NETWORK_FRAME_SAMPLES_PER_CHANNEL*MAX_FRAMES*2]; - static int16_t scratchBuffer[AudioConstants::NETWORK_FRAME_SAMPLES_PER_CHANNEL*MAX_FRAMES]; - // final output is 48Khz, so use this to limit network audio plus upsampled injectors - static AudioLimiter finalAudioLimiter(2*AudioConstants::SAMPLE_RATE, AudioConstants::STEREO); - - // Injectors are 24khz, but we are running at 48Khz here, so need an AudioSRC to upsample - // TODO: probably an existing src the AudioClient is appropriate, check! - static AudioSRC audioSRC(AudioConstants::SAMPLE_RATE, 2*AudioConstants::SAMPLE_RATE, AudioConstants::STEREO); - - // limit maxSize - if (maxSize > AudioConstants::NETWORK_FRAME_SAMPLES_PER_CHANNEL * MAX_FRAMES) { - maxSize = AudioConstants::NETWORK_FRAME_SAMPLES_PER_CHANNEL * MAX_FRAMES; - } - - // initialize stuff - memset(hrtfBuffer, 0, sizeof(hrtfBuffer)); - memset(mixed48Khz, 0, sizeof(mixed48Khz)); - QVector injectorsToRemove; - qint64 framesReadFromInjectors = 0; - - //qDebug() << "maxSize=" << maxSize; - - // first, grab the stuff from the network. Read into a if ((samplesPopped = _receivedAudioStream.popSamples((int)samplesRequested, false)) > 0) { AudioRingBuffer::ConstIterator lastPopOutput = _receivedAudioStream.getLastPopOutput(); lastPopOutput.readSamples((int16_t*)data, samplesPopped); - - //qDebug() << "AudioClient::AudioOutputIODevice::readData popped " << samplesPopped << "samples"; - - // no matter what, this is how many bytes will be written bytesWritten = samplesPopped * sizeof(int16_t); - - - // now, grab same number of frames (samplesPopped/2 since network is stereo) - // and divide by 2 again because these are 24Khz not 48Khz - // from each of the localInjectors, if any - for (AudioInjector* injector : _audio->getActiveLocalAudioInjectors()) { - if (injector->getLocalBuffer() ) { - - // we want to get the same number of frames that we got from - // the network. Since network is stereo, that is samplesPopped/2. - // Since each frame is 2 bytes, we read samplesPopped bytes - qint64 bytesReadFromInjector = injector->getLocalBuffer()->readData((char*)scratchBuffer, samplesPopped/2); - qint64 framesReadFromInjector = bytesReadFromInjector/sizeof(int16_t); - - // keep track of max frames read for all injectors - if (framesReadFromInjector > framesReadFromInjectors) { - framesReadFromInjectors = framesReadFromInjector; - } - - if (framesReadFromInjector > 0) { - - //qDebug() << "AudioClient::AudioOutputIODevice::readData found " << framesReadFromInjector << " frames"; - - // calculate gain and azimuth for hrtf - glm::vec3 relativePosition = injector->getPosition() - _audio->_positionGetter(); - float gain = _audio->gainForSource(relativePosition, injector->getVolume()); - float azimuth = _audio->azimuthForSource(relativePosition); - - // now hrtf this guy, one NETWORK_FRAME_SAMPLES_PER_CHANNEL at a time - this->renderHRTF(injector->getLocalHRTF(), scratchBuffer, hrtfBuffer, azimuth, gain, framesReadFromInjector); - - } else { - - qDebug() << "AudioClient::AudioOutputIODevice::readData no more data, adding to list of injectors to remove"; - injectorsToRemove.append(injector); - injector->finish(); - } - } else { - - qDebug() << "AudioClient::AudioOutputIODevice::readData injector " << injector << " has no local buffer, adding to list of injectors to remove"; - injectorsToRemove.append(injector); - injector->finish(); - } - } - - if (framesReadFromInjectors > 0) { - // resample the 24Khz injector audio in hrtfBuffer to 48Khz - audioSRC.render(hrtfBuffer, mixed48Khz, framesReadFromInjectors); - - //qDebug() << "upsampled to" << framesReadFromInjectors*2 << " frames, stereo"; - - int16_t* dataPtr = (int16_t*)data; - - // now mix in the network data - for (int i=0; igetActiveLocalAudioInjectors().removeOne(injector); - //qDebug() << "removed injector " << injector << " from active injector list!"; - } - - //qDebug() << "bytesWritten=" << bytesWritten; - int bytesAudioOutputUnplayed = _audio->_audioOutput->bufferSize() - _audio->_audioOutput->bytesFree(); if (!bytesAudioOutputUnplayed) { qCDebug(audioclient) << "empty audio buffer"; diff --git a/libraries/audio-client/src/AudioClient.h b/libraries/audio-client/src/AudioClient.h index 91987178b0..3630358771 100644 --- a/libraries/audio-client/src/AudioClient.h +++ b/libraries/audio-client/src/AudioClient.h @@ -38,11 +38,14 @@ #include #include #include +#include +#include +#include +#include +#include #include "AudioIOStats.h" #include "AudioNoiseGate.h" -#include "AudioSRC.h" -#include "AudioReverb.h" #ifdef _WIN32 #pragma warning( push ) @@ -92,7 +95,6 @@ public: MixedProcessedAudioStream& _receivedAudioStream; AudioClient* _audio; int _unfulfilledReads; - void renderHRTF(AudioHRTF& hrtf, int16_t* data, float* hrtfBuffer, float azimuth, float gain, qint64 numSamples); }; const MixedProcessedAudioStream& getReceivedAudioStream() const { return _receivedAudioStream; } @@ -208,6 +210,9 @@ protected: private: void outputFormatChanged(); + void AudioClient::mixLocalAudioInjectors(int16_t* inputBuffer); + float azimuthForSource(const glm::vec3& relativePosition); + float gainForSource(const glm::vec3& relativePosition, float volume); QByteArray firstInputFrame; QAudioInput* _audioInput; @@ -264,6 +269,11 @@ private: AudioSRC* _inputToNetworkResampler; AudioSRC* _networkToOutputResampler; + // for local hrtf-ing + float _hrtfBuffer[AudioConstants::NETWORK_FRAME_SAMPLES_STEREO]; + int16_t _scratchBuffer[AudioConstants::NETWORK_FRAME_SAMPLES_STEREO]; + AudioLimiter _audioLimiter; + // Adds Reverb void configureReverb(); void updateReverbOptions(); @@ -296,10 +306,6 @@ private: bool _hasReceivedFirstPacket = false; QVector _activeLocalAudioInjectors; - - float azimuthForSource(const glm::vec3& relativePosition); - float gainForSource(const glm::vec3& relativePosition, float volume); - };