From 600348bf102fdef7bbb28e16bffd104b208d6838 Mon Sep 17 00:00:00 2001
From: David Kelly <david@highfidelity.io>
Date: Fri, 1 Jul 2016 19:27:00 -0700
Subject: [PATCH 1/9] Initial cut of htrf for mono localOnly injectors

Probably need to clean up a bit, but wanted to get this out there
for comment on more general issues, etc...

To test, I added a localOnly: true to the cow in the tutorial.
---
 libraries/audio-client/src/AudioClient.cpp | 263 +++++++++++++++++++--
 libraries/audio-client/src/AudioClient.h   |  11 +-
 libraries/audio/src/AudioInjector.h        |  14 +-
 3 files changed, 262 insertions(+), 26 deletions(-)

diff --git a/libraries/audio-client/src/AudioClient.cpp b/libraries/audio-client/src/AudioClient.cpp
index 7628c09748..01bd0d3c22 100644
--- a/libraries/audio-client/src/AudioClient.cpp
+++ b/libraries/audio-client/src/AudioClient.cpp
@@ -14,6 +14,8 @@
 #include <sys/stat.h>
 
 #include <glm/glm.hpp>
+#include <glm/gtx/norm.hpp>
+#include <glm/gtx/vector_angle.hpp>
 
 #ifdef __APPLE__
 #include <CoreAudio/AudioHardware.h>
@@ -42,6 +44,8 @@
 #include <Transform.h>
 
 #include "AudioInjector.h"
+#include "AudioLimiter.h"
+#include "AudioHRTF.h"
 #include "AudioConstants.h"
 #include "PositionalAudioStream.h"
 #include "AudioClientLogging.h"
@@ -852,33 +856,43 @@ void AudioClient::setIsStereoInput(bool isStereoInput) {
 
 
 bool AudioClient::outputLocalInjector(bool isStereo, AudioInjector* injector) {
-    if (injector->getLocalBuffer()) {
-        QAudioFormat localFormat = _desiredOutputFormat;
-        localFormat.setChannelCount(isStereo ? 2 : 1);
+    if (injector->getLocalBuffer() && _audioInput ) {
+        if(isStereo) {
+            QAudioFormat localFormat = _desiredOutputFormat;
+            localFormat.setChannelCount(isStereo ? 2 : 1);
 
-        QAudioOutput* localOutput = new QAudioOutput(getNamedAudioDeviceForMode(QAudio::AudioOutput, _outputAudioDeviceName),
-                                                     localFormat,
-                                                     injector->getLocalBuffer());
+            QAudioOutput* localOutput = new QAudioOutput(getNamedAudioDeviceForMode(QAudio::AudioOutput, _outputAudioDeviceName),
+                    localFormat,
+                    injector->getLocalBuffer());
 
-        // move the localOutput to the same thread as the local injector buffer
-        localOutput->moveToThread(injector->getLocalBuffer()->thread());
+            // move the localOutput to the same thread as the local injector buffer
+            localOutput->moveToThread(injector->getLocalBuffer()->thread());
 
-        // have it be stopped when that local buffer is about to close
-        // We don't want to stop this localOutput and injector whenever this AudioClient singleton goes idle,
-        // only when the localOutput does. But the connection is to localOutput, so that it happens on the right thread.
-        connect(localOutput, &QAudioOutput::stateChanged, localOutput, [=](QAudio::State state) {
-            if (state == QAudio::IdleState) {
-                localOutput->stop();
-                injector->stop();
-            }
-        });
+            // have it be stopped when that local buffer is about to close
+            // We don't want to stop this localOutput and injector whenever this AudioClient singleton goes idle,
+            // only when the localOutput does. But the connection is to localOutput, so that it happens on the right thread.
+            connect(localOutput, &QAudioOutput::stateChanged, localOutput, [=](QAudio::State state) {
+                    if (state == QAudio::IdleState) {
+                    localOutput->stop();
+                    injector->stop();
+                    }
+                    });
 
-        connect(injector->getLocalBuffer(), &QIODevice::aboutToClose, localOutput, &QAudioOutput::stop);
+            connect(injector->getLocalBuffer(), &QIODevice::aboutToClose, localOutput, &QAudioOutput::stop);
 
-        qCDebug(audioclient) << "Starting QAudioOutput for local injector" << localOutput;
+            qCDebug(audioclient) << "Starting QAudioOutput for local injector" << localOutput;
+
+            localOutput->start(injector->getLocalBuffer());
+            return localOutput->state() == QAudio::ActiveState;
+        } else {
+        // just add it to the vector of active local injectors
+        // TODO: deal with concurrency perhaps?  Maybe not
+            qDebug() << "adding new injector!!!!!!!";
+
+            _activeLocalAudioInjectors.append(injector);
+            return true;
+        }
 
-        localOutput->start(injector->getLocalBuffer());
-        return localOutput->state() == QAudio::ActiveState;
     }
 
     return false;
@@ -1134,20 +1148,225 @@ float AudioClient::getAudioOutputMsecsUnplayed() const {
     return msecsAudioOutputUnplayed;
 }
 
+void AudioClient::AudioOutputIODevice::renderHRTF(AudioHRTF& hrtf, int16_t* data, float* hrtfBuffer, float azimuth, float gain, qint64 numSamples) {
+    qint64 numFrames = numSamples/AudioConstants::NETWORK_FRAME_SAMPLES_PER_CHANNEL;
+    int16_t* dataPtr = data;
+    float* hrtfPtr = hrtfBuffer;
+    for(qint64 i=0; i < numFrames; i++) {
+        hrtf.render(dataPtr, hrtfPtr, 1, azimuth, gain, AudioConstants::NETWORK_FRAME_SAMPLES_PER_CHANNEL);
+        
+        //qDebug() << "processed frame " << i;
+        
+        dataPtr += AudioConstants::NETWORK_FRAME_SAMPLES_PER_CHANNEL;
+        hrtfPtr += 2*AudioConstants::NETWORK_FRAME_SAMPLES_PER_CHANNEL;
+        
+    }
+    assert(dataPtr - data <= numSamples);
+    assert(hrtfPtr - hrtfBuffer <= 2*numSamples);
+}
+
+float AudioClient::azimuthForSource(const glm::vec3& relativePosition) {
+    // copied from AudioMixer, more or less
+    glm::quat inverseOrientation = glm::inverse(_orientationGetter());
+    
+    // compute sample delay for the 2 ears to create phase panning
+    glm::vec3 rotatedSourcePosition = inverseOrientation * relativePosition;
+    
+    // project the rotated source position vector onto x-y plane
+    rotatedSourcePosition.y = 0.0f;
+
+    static const float SOURCE_DISTANCE_THRESHOLD = 1e-30f;
+
+    if (glm::length2(rotatedSourcePosition) > SOURCE_DISTANCE_THRESHOLD) {
+        
+        // produce an oriented angle about the y-axis
+        return glm::orientedAngle(glm::vec3(0.0f, 0.0f, -1.0f), glm::normalize(rotatedSourcePosition), glm::vec3(0.0f, -1.0f, 0.0f));
+    } else {
+        
+        // no azimuth if they are in same spot
+        return 0.0f; 
+    }
+}
+
+float AudioClient::gainForSource(const glm::vec3& relativePosition, float volume) {
+    // TODO: put these in a place where we can share with AudioMixer!
+    const float LOUDNESS_TO_DISTANCE_RATIO = 0.00001f;
+    const float DEFAULT_ATTENUATION_PER_DOUBLING_IN_DISTANCE = 0.18f;
+    const float DEFAULT_NOISE_MUTING_THRESHOLD = 0.003f;
+    const float ATTENUATION_BEGINS_AT_DISTANCE = 1.0f;
+
+
+    //qDebug() << "initial gain is " << volume;
+    
+    // I'm assuming that the AudioMixer's getting of the stream's attenuation
+    // factor is basically same as getting volume
+    float gain = volume;
+    float distanceBetween = glm::length(relativePosition);
+    if (distanceBetween < EPSILON ) {
+        distanceBetween = EPSILON;
+    }
+
+    // audio mixer has notion of zones.  Unsure how to map that across here...
+
+    // attenuate based on distance now
+    if (distanceBetween >= ATTENUATION_BEGINS_AT_DISTANCE) {
+        float distanceCoefficient = 1.0f - (logf(distanceBetween/ATTENUATION_BEGINS_AT_DISTANCE) / logf(2.0f)
+                * DEFAULT_ATTENUATION_PER_DOUBLING_IN_DISTANCE);
+        if (distanceCoefficient < 0.0f) {
+            distanceCoefficient = 0.0f;
+        }
+
+        gain *= distanceCoefficient;
+    }
+    
+    //qDebug() << "calculated gain as " << gain;
+
+    return gain;
+}
+
 qint64 AudioClient::AudioOutputIODevice::readData(char * data, qint64 maxSize) {
     auto samplesRequested = maxSize / sizeof(int16_t);
     int samplesPopped;
     int bytesWritten;
 
+    // limit the number of NETWORK_FRAME_SAMPLES_PER_CHANNEL to return regardless
+    // of what maxSize requests.
+    static const qint64 MAX_FRAMES = 256;
+    static float hrtfBuffer[AudioConstants::NETWORK_FRAME_SAMPLES_PER_CHANNEL*MAX_FRAMES];
+    static int16_t scratchBuffer[AudioConstants::NETWORK_FRAME_SAMPLES_PER_CHANNEL*MAX_FRAMES];
+    
+    // injectors are 24Khz, use this to limit the injector audio only
+    static AudioLimiter audioLimiter(AudioConstants::SAMPLE_RATE, AudioConstants::STEREO);
+    
+    // final output is 48Khz, so use this to limit network audio plus upsampled injectors
+    static AudioLimiter finalAudioLimiter(2*AudioConstants::SAMPLE_RATE, AudioConstants::STEREO);
+    
+    // Injectors are 24khz, but we are running at 48Khz here, so need an AudioSRC to upsample
+    static AudioSRC audioSRC(AudioConstants::SAMPLE_RATE, 2*AudioConstants::SAMPLE_RATE, AudioConstants::STEREO);
+    
+    // limit maxSize
+    if (maxSize > AudioConstants::NETWORK_FRAME_SAMPLES_PER_CHANNEL * MAX_FRAMES) {
+        maxSize = AudioConstants::NETWORK_FRAME_SAMPLES_PER_CHANNEL * MAX_FRAMES; 
+    }
+
+    // initialize stuff
+    memset(hrtfBuffer, 0, sizeof(hrtfBuffer));
+    QVector<AudioInjector*> injectorsToRemove;
+    qint64 framesReadFromInjectors = 0;
+    
+    //qDebug() << "maxSize=" << maxSize << "data ptr: "<< (qint64)data;
+    
+    for (AudioInjector* injector : _audio->getActiveLocalAudioInjectors()) {
+        // pop off maxSize/4 (since it is mono, and 24Khz instead of 48Khz)
+        // bytes from the injector's localBuffer, using the data ptr as a temporary buffer
+        // note we are only hrtf-ing mono buffers, so we request half of what maxSize is, since it
+        // wants that many stereo bytes
+        if (injector->getLocalBuffer() ) {
+            
+            glm::vec3 relativePosition = injector->getPosition() - _audio->_positionGetter();
+
+            qint64 bytesReadFromInjector = injector->getLocalBuffer()->readData(data, maxSize/4);
+            qint64 framesReadFromInjector = bytesReadFromInjector/sizeof(int16_t);
+            if (framesReadFromInjector > framesReadFromInjectors) {
+                framesReadFromInjectors = framesReadFromInjector;
+            }
+            if (framesReadFromInjector > 0) {
+                
+                //qDebug() << "AudioClient::AudioOutputIODevice::readData found " << framesReadFromInjector << " frames";
+
+                // calculate these shortly
+                float gain = _audio->gainForSource(relativePosition, injector->getVolume()); 
+                float azimuth = _audio->azimuthForSource(relativePosition); 
+
+                // now hrtf this guy, one NETWORK_FRAME_SAMPLES_PER_CHANNEL at a time
+                this->renderHRTF(injector->getLocalHRTF(), (int16_t*)data, hrtfBuffer, azimuth, gain, framesReadFromInjector);
+
+            } else {
+
+                // probably need to be more clever here than just getting rid of the injector?
+                //qDebug() << "AudioClient::AudioOutputIODevice::readData no more data, adding to list of injectors to remove";
+                injectorsToRemove.append(injector);
+                injector->finish();
+            }
+        } else {
+
+            // probably need to be more clever here then just getting rid of injector?
+            //qDebug() << "AudioClient::AudioOutputIODevice::readData injector " << injector << " has no local buffer, adding to list of injectors to remove";
+            injectorsToRemove.append(injector);
+            injector->finish();
+        }
+    }
+    
+    if (framesReadFromInjectors > 0) {
+        // resample the 24Khz injector audio to 48Khz
+        // but first, hit with limiter :(
+        audioLimiter.render(hrtfBuffer, (int16_t*)data, framesReadFromInjectors);
+        audioSRC.render((int16_t*)data, scratchBuffer, framesReadFromInjectors);
+        
+        // now, lets move this back into the hrtfBuffer
+        for(int i=0; i<framesReadFromInjectors*4; i++) {
+            hrtfBuffer[i] = (float)(scratchBuffer[i]) * 1/32676.0f;
+        }
+        
+        //qDebug() << "upsampled " << framesReadFromInjectors*2 << " frames, stereo";
+
+    }
+
+    // k get rid of finished injectors
+    for (AudioInjector* injector : injectorsToRemove) {
+        // TODO: I'd expect concurrency issues here -- fix?
+        _audio->getActiveLocalAudioInjectors().removeOne(injector);
+        
+        //qDebug() << "removed injector " << injector << " from active injector list!";
+    }
+
+    // now grab the stuff from the network, and mix it in...
     if ((samplesPopped = _receivedAudioStream.popSamples((int)samplesRequested, false)) > 0) {
         AudioRingBuffer::ConstIterator lastPopOutput = _receivedAudioStream.getLastPopOutput();
         lastPopOutput.readSamples((int16_t*)data, samplesPopped);
+        
+        //qDebug() << "AudioClient::AudioOutputIODevice::readData popped " << samplesPopped << "samples" << "to " << (qint64)data;
+        
         bytesWritten = samplesPopped * sizeof(int16_t);
-    } else {
+        if (framesReadFromInjectors > 0) {
+
+            int16_t* dataPtr = (int16_t*)data;
+            // convert audio to floats for mixing with limiter...
+            //qDebug() << "AudioClient::AudioOUtputIODevice::readData converting to floats...";
+            for (int i=0; i<samplesPopped; i++) {
+                hrtfBuffer[i] += (float)(*dataPtr++) * 1/32676.0f;
+            }
+            
+            // now use limiter, outputting to the data buffer.  
+            // TODO:Since there
+            // are possibilities for pulling less data from the network than
+            // we got from injectors (or more), we need to look at being
+            // more clever here.  If we get more bytes here than in the injectors, I'd assume
+            // that is ok -- we are at the 'end' of the injector sound(s). But
+            // if we get less here than at the injectors, we maybe want to ponder
+            // what to do.
+
+            //qDebug() << "calling limiter...";
+            finalAudioLimiter.render(hrtfBuffer, (int16_t*)data, samplesPopped/2);
+            // the 2 is because the htrf turned the mono into stereo, to match
+            // what the network data has (2 channels)
+            if (8*framesReadFromInjectors > bytesWritten) {
+                bytesWritten = 8*framesReadFromInjectors;
+            }
+            
+            //qDebug() << "done, bytes written = " << bytesWritten;
+        }
+    } else if (framesReadFromInjectors == 0) {
+        // if nobody has data, 0 out buffer...
         memset(data, 0, maxSize);
         bytesWritten = maxSize;
+    } else {
+        // only sound from injectors, multiply by 2 as the htrf is stereo
+        finalAudioLimiter.render(hrtfBuffer, (int16_t*)data, framesReadFromInjectors*2);
+        bytesWritten = 8*framesReadFromInjectors;
     }
 
+    
     int bytesAudioOutputUnplayed = _audio->_audioOutput->bufferSize() - _audio->_audioOutput->bytesFree();
     if (!bytesAudioOutputUnplayed) {
         qCDebug(audioclient) << "empty audio buffer";
diff --git a/libraries/audio-client/src/AudioClient.h b/libraries/audio-client/src/AudioClient.h
index dc46db5657..91987178b0 100644
--- a/libraries/audio-client/src/AudioClient.h
+++ b/libraries/audio-client/src/AudioClient.h
@@ -37,6 +37,7 @@
 #include <SettingHandle.h>
 #include <Sound.h>
 #include <StDev.h>
+#include <AudioHRTF.h>
 
 #include "AudioIOStats.h"
 #include "AudioNoiseGate.h"
@@ -86,12 +87,12 @@ public:
         void stop() { close(); }
         qint64    readData(char * data, qint64 maxSize);
         qint64    writeData(const char * data, qint64 maxSize) { return 0; }
-
         int getRecentUnfulfilledReads() { int unfulfilledReads = _unfulfilledReads; _unfulfilledReads = 0; return unfulfilledReads; }
     private:
         MixedProcessedAudioStream& _receivedAudioStream;
         AudioClient* _audio;
         int _unfulfilledReads;
+        void renderHRTF(AudioHRTF& hrtf, int16_t* data, float* hrtfBuffer, float azimuth, float gain, qint64 numSamples);
     };
 
     const MixedProcessedAudioStream& getReceivedAudioStream() const { return _receivedAudioStream; }
@@ -124,6 +125,8 @@ public:
 
     void setPositionGetter(AudioPositionGetter positionGetter) { _positionGetter = positionGetter; }
     void setOrientationGetter(AudioOrientationGetter orientationGetter) { _orientationGetter = orientationGetter; }
+    
+    QVector<AudioInjector*>& getActiveLocalAudioInjectors() { return _activeLocalAudioInjectors; }
 
     static const float CALLBACK_ACCELERATOR_RATIO;
 
@@ -291,6 +294,12 @@ private:
     void checkDevices();
 
     bool _hasReceivedFirstPacket = false;
+    
+    QVector<AudioInjector*> _activeLocalAudioInjectors;
+
+    float azimuthForSource(const glm::vec3& relativePosition);
+    float gainForSource(const glm::vec3& relativePosition, float volume);
+
 };
 
 
diff --git a/libraries/audio/src/AudioInjector.h b/libraries/audio/src/AudioInjector.h
index c90256429d..a4cde77377 100644
--- a/libraries/audio/src/AudioInjector.h
+++ b/libraries/audio/src/AudioInjector.h
@@ -26,6 +26,7 @@
 
 #include "AudioInjectorLocalBuffer.h"
 #include "AudioInjectorOptions.h"
+#include "AudioHRTF.h"
 #include "Sound.h"
 
 class AbstractAudioInterface;
@@ -54,8 +55,11 @@ public:
     void setCurrentSendOffset(int currentSendOffset) { _currentSendOffset = currentSendOffset; }
     
     AudioInjectorLocalBuffer* getLocalBuffer() const { return _localBuffer; }
+    AudioHRTF& getLocalHRTF() { return _localHRTF; }
+
     bool isLocalOnly() const { return _options.localOnly; }
-    
+    float getVolume() const { return _options.volume; }
+    glm::vec3 getPosition() const { return _options.position; } 
     void setLocalAudioInterface(AbstractAudioInterface* localAudioInterface) { _localAudioInterface = localAudioInterface; }
 
     static AudioInjector* playSoundAndDelete(const QByteArray& buffer, const AudioInjectorOptions options, AbstractAudioInterface* localInterface);
@@ -74,14 +78,15 @@ public slots:
     
     float getLoudness() const { return _loudness; }
     bool isPlaying() const { return _state == State::NotFinished || _state == State::NotFinishedWithPendingDelete; }
+    void finish();
     
 signals:
     void finished();
     void restarting();
     
-private slots:
+/*private slots:
     void finish();
-    
+*/  
 private:
     void setupInjection();
     int64_t injectNextFrame();
@@ -103,6 +108,9 @@ private:
     std::unique_ptr<QElapsedTimer> _frameTimer { nullptr };
     quint16 _outgoingSequenceNumber { 0 };
     
+    // when the injector is local, we need this
+    AudioHRTF _localHRTF;
+    // for local injection, 
     friend class AudioInjectorManager;
 };
 

From 7a9b3978e96908034caee4c92c73082cd38bfafa Mon Sep 17 00:00:00 2001
From: David Kelly <david@highfidelity.io>
Date: Fri, 8 Jul 2016 13:01:06 -0700
Subject: [PATCH 2/9] Better processing of audio

A couple things Ken suggested.  First off, there are now AudioSRC
calls for floats, which simplfied stuff a bit.  Then, I switched to
grabbing network packets first, and only pulling at most that amount
of audio from the local injectors.  That improves things - the
occasional artifacts (when the injectors got more data than the
network for instance) are gone.

Also, fixed build issue (unused variable warning) for mac and android.
---
 libraries/audio-client/src/AudioClient.cpp | 168 +++++++++------------
 1 file changed, 71 insertions(+), 97 deletions(-)

diff --git a/libraries/audio-client/src/AudioClient.cpp b/libraries/audio-client/src/AudioClient.cpp
index 01bd0d3c22..0943585ab7 100644
--- a/libraries/audio-client/src/AudioClient.cpp
+++ b/libraries/audio-client/src/AudioClient.cpp
@@ -1190,9 +1190,7 @@ float AudioClient::azimuthForSource(const glm::vec3& relativePosition) {
 
 float AudioClient::gainForSource(const glm::vec3& relativePosition, float volume) {
     // TODO: put these in a place where we can share with AudioMixer!
-    const float LOUDNESS_TO_DISTANCE_RATIO = 0.00001f;
     const float DEFAULT_ATTENUATION_PER_DOUBLING_IN_DISTANCE = 0.18f;
-    const float DEFAULT_NOISE_MUTING_THRESHOLD = 0.003f;
     const float ATTENUATION_BEGINS_AT_DISTANCE = 1.0f;
 
 
@@ -1233,15 +1231,14 @@ qint64 AudioClient::AudioOutputIODevice::readData(char * data, qint64 maxSize) {
     // of what maxSize requests.
     static const qint64 MAX_FRAMES = 256;
     static float hrtfBuffer[AudioConstants::NETWORK_FRAME_SAMPLES_PER_CHANNEL*MAX_FRAMES];
+    static float mixed48Khz[AudioConstants::NETWORK_FRAME_SAMPLES_PER_CHANNEL*MAX_FRAMES*2];
     static int16_t scratchBuffer[AudioConstants::NETWORK_FRAME_SAMPLES_PER_CHANNEL*MAX_FRAMES];
     
-    // injectors are 24Khz, use this to limit the injector audio only
-    static AudioLimiter audioLimiter(AudioConstants::SAMPLE_RATE, AudioConstants::STEREO);
-    
     // final output is 48Khz, so use this to limit network audio plus upsampled injectors
     static AudioLimiter finalAudioLimiter(2*AudioConstants::SAMPLE_RATE, AudioConstants::STEREO);
     
     // Injectors are 24khz, but we are running at 48Khz here, so need an AudioSRC to upsample
+    // TODO: probably an existing src the AudioClient is appropriate, check!
     static AudioSRC audioSRC(AudioConstants::SAMPLE_RATE, 2*AudioConstants::SAMPLE_RATE, AudioConstants::STEREO);
     
     // limit maxSize
@@ -1251,121 +1248,98 @@ qint64 AudioClient::AudioOutputIODevice::readData(char * data, qint64 maxSize) {
 
     // initialize stuff
     memset(hrtfBuffer, 0, sizeof(hrtfBuffer));
+    memset(mixed48Khz, 0, sizeof(mixed48Khz));
     QVector<AudioInjector*> injectorsToRemove;
     qint64 framesReadFromInjectors = 0;
     
-    //qDebug() << "maxSize=" << maxSize << "data ptr: "<< (qint64)data;
+    qDebug() << "maxSize=" << maxSize;
     
-    for (AudioInjector* injector : _audio->getActiveLocalAudioInjectors()) {
-        // pop off maxSize/4 (since it is mono, and 24Khz instead of 48Khz)
-        // bytes from the injector's localBuffer, using the data ptr as a temporary buffer
-        // note we are only hrtf-ing mono buffers, so we request half of what maxSize is, since it
-        // wants that many stereo bytes
-        if (injector->getLocalBuffer() ) {
-            
-            glm::vec3 relativePosition = injector->getPosition() - _audio->_positionGetter();
+    // first, grab the stuff from the network.  Read into a 
+    if ((samplesPopped = _receivedAudioStream.popSamples((int)samplesRequested, false)) > 0) {
+        AudioRingBuffer::ConstIterator lastPopOutput = _receivedAudioStream.getLastPopOutput();
+        lastPopOutput.readSamples((int16_t*)data, samplesPopped);
 
-            qint64 bytesReadFromInjector = injector->getLocalBuffer()->readData(data, maxSize/4);
-            qint64 framesReadFromInjector = bytesReadFromInjector/sizeof(int16_t);
-            if (framesReadFromInjector > framesReadFromInjectors) {
-                framesReadFromInjectors = framesReadFromInjector;
-            }
-            if (framesReadFromInjector > 0) {
+        qDebug() << "AudioClient::AudioOutputIODevice::readData popped " << samplesPopped << "samples"; 
+
+        // no matter what, this is how many bytes will be written
+        bytesWritten = samplesPopped * sizeof(int16_t);
+
+
+        // now, grab same number of frames (samplesPopped/2 since network is stereo) 
+        // and divide by 2 again because these are 24Khz not 48Khz
+        // from each of the localInjectors, if any
+        for (AudioInjector* injector : _audio->getActiveLocalAudioInjectors()) {
+            if (injector->getLocalBuffer() ) {
+
+                // we want to get the same number of frames that we got from 
+                // the network.  Since network is stereo, that is samplesPopped/2.  
+                // Since each frame is 2 bytes, we read samplesPopped bytes
+                qint64 bytesReadFromInjector = injector->getLocalBuffer()->readData((char*)scratchBuffer, samplesPopped/2);
+                qint64 framesReadFromInjector = bytesReadFromInjector/sizeof(int16_t);
                 
-                //qDebug() << "AudioClient::AudioOutputIODevice::readData found " << framesReadFromInjector << " frames";
+                // keep track of max frames read for all injectors
+                if (framesReadFromInjector > framesReadFromInjectors) {
+                    framesReadFromInjectors = framesReadFromInjector;
+                }
 
-                // calculate these shortly
-                float gain = _audio->gainForSource(relativePosition, injector->getVolume()); 
-                float azimuth = _audio->azimuthForSource(relativePosition); 
+                if (framesReadFromInjector > 0) {
 
-                // now hrtf this guy, one NETWORK_FRAME_SAMPLES_PER_CHANNEL at a time
-                this->renderHRTF(injector->getLocalHRTF(), (int16_t*)data, hrtfBuffer, azimuth, gain, framesReadFromInjector);
+                    qDebug() << "AudioClient::AudioOutputIODevice::readData found " << framesReadFromInjector << " frames";
 
+                    // calculate gain and azimuth for hrtf
+                    glm::vec3 relativePosition = injector->getPosition() - _audio->_positionGetter();
+                    float gain = _audio->gainForSource(relativePosition, injector->getVolume()); 
+                    float azimuth = _audio->azimuthForSource(relativePosition); 
+
+                    // now hrtf this guy, one NETWORK_FRAME_SAMPLES_PER_CHANNEL at a time
+                    this->renderHRTF(injector->getLocalHRTF(), scratchBuffer, hrtfBuffer, azimuth, gain, framesReadFromInjector);
+
+                } else {
+
+                    qDebug() << "AudioClient::AudioOutputIODevice::readData no more data, adding to list of injectors to remove";
+                    injectorsToRemove.append(injector);
+                    injector->finish();
+                }
             } else {
 
-                // probably need to be more clever here than just getting rid of the injector?
-                //qDebug() << "AudioClient::AudioOutputIODevice::readData no more data, adding to list of injectors to remove";
+                qDebug() << "AudioClient::AudioOutputIODevice::readData injector " << injector << " has no local buffer, adding to list of injectors to remove";
                 injectorsToRemove.append(injector);
                 injector->finish();
             }
-        } else {
-
-            // probably need to be more clever here then just getting rid of injector?
-            //qDebug() << "AudioClient::AudioOutputIODevice::readData injector " << injector << " has no local buffer, adding to list of injectors to remove";
-            injectorsToRemove.append(injector);
-            injector->finish();
         }
-    }
-    
-    if (framesReadFromInjectors > 0) {
-        // resample the 24Khz injector audio to 48Khz
-        // but first, hit with limiter :(
-        audioLimiter.render(hrtfBuffer, (int16_t*)data, framesReadFromInjectors);
-        audioSRC.render((int16_t*)data, scratchBuffer, framesReadFromInjectors);
-        
-        // now, lets move this back into the hrtfBuffer
-        for(int i=0; i<framesReadFromInjectors*4; i++) {
-            hrtfBuffer[i] = (float)(scratchBuffer[i]) * 1/32676.0f;
-        }
-        
-        //qDebug() << "upsampled " << framesReadFromInjectors*2 << " frames, stereo";
 
+        if (framesReadFromInjectors > 0) {
+            // resample the 24Khz injector audio in hrtfBuffer to 48Khz
+            audioSRC.render(hrtfBuffer, mixed48Khz, framesReadFromInjectors);
+            
+            qDebug() << "upsampled to" << framesReadFromInjectors*2 << " frames, stereo";
+
+            int16_t* dataPtr = (int16_t*)data;
+            
+            // now mix in the network data
+            for (int i=0; i<samplesPopped; i++) {
+                mixed48Khz[i] += (float)(*dataPtr++) * 1/32676.0f;
+            }
+
+            qDebug() << "mixed network data into upsampled hrtf'd mixed injector data" ;
+            
+            finalAudioLimiter.render(mixed48Khz, (int16_t*)data, samplesPopped/2);
+        }
+    } else {
+        // nothing on network, don't grab anything from injectors, and just 
+        // return 0s
+        memset(data, 0, maxSize);
+        bytesWritten = maxSize;
+        
     }
 
     // k get rid of finished injectors
     for (AudioInjector* injector : injectorsToRemove) {
-        // TODO: I'd expect concurrency issues here -- fix?
         _audio->getActiveLocalAudioInjectors().removeOne(injector);
-        
-        //qDebug() << "removed injector " << injector << " from active injector list!";
-    }
-
-    // now grab the stuff from the network, and mix it in...
-    if ((samplesPopped = _receivedAudioStream.popSamples((int)samplesRequested, false)) > 0) {
-        AudioRingBuffer::ConstIterator lastPopOutput = _receivedAudioStream.getLastPopOutput();
-        lastPopOutput.readSamples((int16_t*)data, samplesPopped);
-        
-        //qDebug() << "AudioClient::AudioOutputIODevice::readData popped " << samplesPopped << "samples" << "to " << (qint64)data;
-        
-        bytesWritten = samplesPopped * sizeof(int16_t);
-        if (framesReadFromInjectors > 0) {
-
-            int16_t* dataPtr = (int16_t*)data;
-            // convert audio to floats for mixing with limiter...
-            //qDebug() << "AudioClient::AudioOUtputIODevice::readData converting to floats...";
-            for (int i=0; i<samplesPopped; i++) {
-                hrtfBuffer[i] += (float)(*dataPtr++) * 1/32676.0f;
-            }
-            
-            // now use limiter, outputting to the data buffer.  
-            // TODO:Since there
-            // are possibilities for pulling less data from the network than
-            // we got from injectors (or more), we need to look at being
-            // more clever here.  If we get more bytes here than in the injectors, I'd assume
-            // that is ok -- we are at the 'end' of the injector sound(s). But
-            // if we get less here than at the injectors, we maybe want to ponder
-            // what to do.
-
-            //qDebug() << "calling limiter...";
-            finalAudioLimiter.render(hrtfBuffer, (int16_t*)data, samplesPopped/2);
-            // the 2 is because the htrf turned the mono into stereo, to match
-            // what the network data has (2 channels)
-            if (8*framesReadFromInjectors > bytesWritten) {
-                bytesWritten = 8*framesReadFromInjectors;
-            }
-            
-            //qDebug() << "done, bytes written = " << bytesWritten;
-        }
-    } else if (framesReadFromInjectors == 0) {
-        // if nobody has data, 0 out buffer...
-        memset(data, 0, maxSize);
-        bytesWritten = maxSize;
-    } else {
-        // only sound from injectors, multiply by 2 as the htrf is stereo
-        finalAudioLimiter.render(hrtfBuffer, (int16_t*)data, framesReadFromInjectors*2);
-        bytesWritten = 8*framesReadFromInjectors;
+        qDebug() << "removed injector " << injector << " from active injector list!";
     }
 
+    qDebug() << "bytesWritten=" << bytesWritten;
     
     int bytesAudioOutputUnplayed = _audio->_audioOutput->bufferSize() - _audio->_audioOutput->bytesFree();
     if (!bytesAudioOutputUnplayed) {

From bc7123d701ba33135c3c0feb9e2e09482153c1fe Mon Sep 17 00:00:00 2001
From: David Kelly <david@highfidelity.io>
Date: Fri, 8 Jul 2016 13:59:49 -0700
Subject: [PATCH 3/9] comment out logging for now

---
 libraries/audio-client/src/AudioClient.cpp | 14 +++++++-------
 1 file changed, 7 insertions(+), 7 deletions(-)

diff --git a/libraries/audio-client/src/AudioClient.cpp b/libraries/audio-client/src/AudioClient.cpp
index 0943585ab7..67fb1ff561 100644
--- a/libraries/audio-client/src/AudioClient.cpp
+++ b/libraries/audio-client/src/AudioClient.cpp
@@ -1252,14 +1252,14 @@ qint64 AudioClient::AudioOutputIODevice::readData(char * data, qint64 maxSize) {
     QVector<AudioInjector*> injectorsToRemove;
     qint64 framesReadFromInjectors = 0;
     
-    qDebug() << "maxSize=" << maxSize;
+    //qDebug() << "maxSize=" << maxSize;
     
     // first, grab the stuff from the network.  Read into a 
     if ((samplesPopped = _receivedAudioStream.popSamples((int)samplesRequested, false)) > 0) {
         AudioRingBuffer::ConstIterator lastPopOutput = _receivedAudioStream.getLastPopOutput();
         lastPopOutput.readSamples((int16_t*)data, samplesPopped);
 
-        qDebug() << "AudioClient::AudioOutputIODevice::readData popped " << samplesPopped << "samples"; 
+        //qDebug() << "AudioClient::AudioOutputIODevice::readData popped " << samplesPopped << "samples"; 
 
         // no matter what, this is how many bytes will be written
         bytesWritten = samplesPopped * sizeof(int16_t);
@@ -1284,7 +1284,7 @@ qint64 AudioClient::AudioOutputIODevice::readData(char * data, qint64 maxSize) {
 
                 if (framesReadFromInjector > 0) {
 
-                    qDebug() << "AudioClient::AudioOutputIODevice::readData found " << framesReadFromInjector << " frames";
+                    //qDebug() << "AudioClient::AudioOutputIODevice::readData found " << framesReadFromInjector << " frames";
 
                     // calculate gain and azimuth for hrtf
                     glm::vec3 relativePosition = injector->getPosition() - _audio->_positionGetter();
@@ -1312,7 +1312,7 @@ qint64 AudioClient::AudioOutputIODevice::readData(char * data, qint64 maxSize) {
             // resample the 24Khz injector audio in hrtfBuffer to 48Khz
             audioSRC.render(hrtfBuffer, mixed48Khz, framesReadFromInjectors);
             
-            qDebug() << "upsampled to" << framesReadFromInjectors*2 << " frames, stereo";
+            //qDebug() << "upsampled to" << framesReadFromInjectors*2 << " frames, stereo";
 
             int16_t* dataPtr = (int16_t*)data;
             
@@ -1321,7 +1321,7 @@ qint64 AudioClient::AudioOutputIODevice::readData(char * data, qint64 maxSize) {
                 mixed48Khz[i] += (float)(*dataPtr++) * 1/32676.0f;
             }
 
-            qDebug() << "mixed network data into upsampled hrtf'd mixed injector data" ;
+            //qDebug() << "mixed network data into upsampled hrtf'd mixed injector data" ;
             
             finalAudioLimiter.render(mixed48Khz, (int16_t*)data, samplesPopped/2);
         }
@@ -1336,10 +1336,10 @@ qint64 AudioClient::AudioOutputIODevice::readData(char * data, qint64 maxSize) {
     // k get rid of finished injectors
     for (AudioInjector* injector : injectorsToRemove) {
         _audio->getActiveLocalAudioInjectors().removeOne(injector);
-        qDebug() << "removed injector " << injector << " from active injector list!";
+        //qDebug() << "removed injector " << injector << " from active injector list!";
     }
 
-    qDebug() << "bytesWritten=" << bytesWritten;
+    //qDebug() << "bytesWritten=" << bytesWritten;
     
     int bytesAudioOutputUnplayed = _audio->_audioOutput->bufferSize() - _audio->_audioOutput->bytesFree();
     if (!bytesAudioOutputUnplayed) {

From 80d33ee2516616beb3b08d9279e812686e37a866 Mon Sep 17 00:00:00 2001
From: David Kelly <david@highfidelity.io>
Date: Fri, 8 Jul 2016 13:59:49 -0700
Subject: [PATCH 4/9] Working now as frame-at-a-time

So localOnly sounds get HRTF'd, one network frame at a time.  Less
processing (upsampling, limiting, etc...) than doing this at the
end of the pipeline (in the AudioOutputIODevice::readData call).
---
 libraries/audio-client/src/AudioClient.cpp | 199 ++++++++-------------
 libraries/audio-client/src/AudioClient.h   |  20 ++-
 2 files changed, 87 insertions(+), 132 deletions(-)

diff --git a/libraries/audio-client/src/AudioClient.cpp b/libraries/audio-client/src/AudioClient.cpp
index 67fb1ff561..190dd1583b 100644
--- a/libraries/audio-client/src/AudioClient.cpp
+++ b/libraries/audio-client/src/AudioClient.cpp
@@ -43,10 +43,6 @@
 #include <UUID.h>
 #include <Transform.h>
 
-#include "AudioInjector.h"
-#include "AudioLimiter.h"
-#include "AudioHRTF.h"
-#include "AudioConstants.h"
 #include "PositionalAudioStream.h"
 #include "AudioClientLogging.h"
 
@@ -111,7 +107,8 @@ AudioClient::AudioClient() :
     _stats(&_receivedAudioStream),
     _inputGate(),
     _positionGetter(DEFAULT_POSITION_GETTER),
-    _orientationGetter(DEFAULT_ORIENTATION_GETTER)
+    _orientationGetter(DEFAULT_ORIENTATION_GETTER),
+    _audioLimiter(AudioConstants::SAMPLE_RATE, AudioConstants::STEREO)
 {
     // clear the array of locally injected samples
     memset(_localProceduralSamples, 0, AudioConstants::NETWORK_FRAME_BYTES_PER_CHANNEL);
@@ -787,15 +784,84 @@ void AudioClient::handleRecordedAudioInput(const QByteArray& audio) {
     emitAudioPacket(audio.data(), audio.size(), _outgoingAvatarAudioSequenceNumber, audioTransform, PacketType::MicrophoneAudioWithEcho);
 }
 
+void AudioClient::mixLocalAudioInjectors(int16_t* inputBuffer) {
+
+    memset(_hrtfBuffer, 0, sizeof(_hrtfBuffer));
+    QVector<AudioInjector*> injectorsToRemove;
+
+    bool injectorsHaveData = false;
+
+    for (AudioInjector* injector : getActiveLocalAudioInjectors()) {
+        if (injector->getLocalBuffer()) {
+
+            // get one (mono) frame from the injector
+            if (0 < injector->getLocalBuffer()->readData((char*)_scratchBuffer, AudioConstants::NETWORK_FRAME_SAMPLES_PER_CHANNEL*sizeof(int16_t)) ) {
+                
+                injectorsHaveData = true;
+
+                // calculate gain and azimuth for hrtf
+                glm::vec3 relativePosition = injector->getPosition() - _positionGetter();
+                float gain = gainForSource(relativePosition, injector->getVolume()); 
+                float azimuth = azimuthForSource(relativePosition); 
+                
+                
+                injector->getLocalHRTF().render(_scratchBuffer, _hrtfBuffer, 1, azimuth, gain, AudioConstants::NETWORK_FRAME_SAMPLES_PER_CHANNEL);
+            
+            } else {
+                
+                qDebug() << "injector has no more data, marking finished for removal";
+                
+                injector->finish();
+                injectorsToRemove.append(injector);
+            }
+
+        } else {
+            
+            qDebug() << "injector has no local buffer, marking as finished for removal";
+            
+            injector->finish();
+            injectorsToRemove.append(injector);
+        }
+    }
+    
+    if(injectorsHaveData) {
+        
+        // mix network into the hrtfBuffer
+        for(int i=0; i<AudioConstants::NETWORK_FRAME_SAMPLES_STEREO; i++) {
+            _hrtfBuffer[i] += (float)(inputBuffer[i]) * 1/32678.0f;
+        }
+        
+        // now, use limiter to write back to the inputBuffer
+        _audioLimiter.render(_hrtfBuffer, inputBuffer, AudioConstants::NETWORK_FRAME_SAMPLES_PER_CHANNEL);
+    }
+
+    for(AudioInjector* injector : injectorsToRemove) {
+        qDebug() << "removing injector";
+        getActiveLocalAudioInjectors().removeOne(injector);
+    }
+}
+
 void AudioClient::processReceivedSamples(const QByteArray& inputBuffer, QByteArray& outputBuffer) {
     const int numNetworkOutputSamples = inputBuffer.size() / sizeof(int16_t);
     const int numDeviceOutputSamples = numNetworkOutputSamples * (_outputFormat.sampleRate() * _outputFormat.channelCount())
         / (_desiredOutputFormat.sampleRate() * _desiredOutputFormat.channelCount());
 
     outputBuffer.resize(numDeviceOutputSamples * sizeof(int16_t));
-
-    const int16_t* receivedSamples = reinterpret_cast<const int16_t*>(inputBuffer.data());
     int16_t* outputSamples = reinterpret_cast<int16_t*>(outputBuffer.data());
+    const int16_t* receivedSamples;
+    char inputBufferCopy[AudioConstants::NETWORK_FRAME_BYTES_STEREO];
+    assert(inputBuffer.size() == AudioConstants::NETWORK_FRAME_BYTES_STEREO);
+    
+    if(getActiveLocalAudioInjectors().size() > 0) {
+        // gotta copy the since it is const
+        for(int i = 0; i < sizeof(inputBufferCopy); i++) {
+            inputBufferCopy[i] = inputBuffer.data()[i];
+        }
+        mixLocalAudioInjectors((int16_t*)inputBufferCopy);
+        receivedSamples = reinterpret_cast<const int16_t*>(inputBufferCopy);
+    } else {
+        receivedSamples = reinterpret_cast<const int16_t*>(inputBuffer.data());
+    }
 
     // copy the packet from the RB to the output
     possibleResampling(_networkToOutputResampler, receivedSamples, outputSamples,
@@ -807,6 +873,7 @@ void AudioClient::processReceivedSamples(const QByteArray& inputBuffer, QByteArr
     if (hasReverb) {
         assert(_outputFormat.channelCount() == 2);
         updateReverbOptions();
+        qDebug() << "handling reverb";
         _listenerReverb.render(outputSamples, outputSamples, numDeviceOutputSamples/2);
     }
 }
@@ -1148,22 +1215,6 @@ float AudioClient::getAudioOutputMsecsUnplayed() const {
     return msecsAudioOutputUnplayed;
 }
 
-void AudioClient::AudioOutputIODevice::renderHRTF(AudioHRTF& hrtf, int16_t* data, float* hrtfBuffer, float azimuth, float gain, qint64 numSamples) {
-    qint64 numFrames = numSamples/AudioConstants::NETWORK_FRAME_SAMPLES_PER_CHANNEL;
-    int16_t* dataPtr = data;
-    float* hrtfPtr = hrtfBuffer;
-    for(qint64 i=0; i < numFrames; i++) {
-        hrtf.render(dataPtr, hrtfPtr, 1, azimuth, gain, AudioConstants::NETWORK_FRAME_SAMPLES_PER_CHANNEL);
-        
-        //qDebug() << "processed frame " << i;
-        
-        dataPtr += AudioConstants::NETWORK_FRAME_SAMPLES_PER_CHANNEL;
-        hrtfPtr += 2*AudioConstants::NETWORK_FRAME_SAMPLES_PER_CHANNEL;
-        
-    }
-    assert(dataPtr - data <= numSamples);
-    assert(hrtfPtr - hrtfBuffer <= 2*numSamples);
-}
 
 float AudioClient::azimuthForSource(const glm::vec3& relativePosition) {
     // copied from AudioMixer, more or less
@@ -1226,105 +1277,11 @@ qint64 AudioClient::AudioOutputIODevice::readData(char * data, qint64 maxSize) {
     auto samplesRequested = maxSize / sizeof(int16_t);
     int samplesPopped;
     int bytesWritten;
-
-    // limit the number of NETWORK_FRAME_SAMPLES_PER_CHANNEL to return regardless
-    // of what maxSize requests.
-    static const qint64 MAX_FRAMES = 256;
-    static float hrtfBuffer[AudioConstants::NETWORK_FRAME_SAMPLES_PER_CHANNEL*MAX_FRAMES];
-    static float mixed48Khz[AudioConstants::NETWORK_FRAME_SAMPLES_PER_CHANNEL*MAX_FRAMES*2];
-    static int16_t scratchBuffer[AudioConstants::NETWORK_FRAME_SAMPLES_PER_CHANNEL*MAX_FRAMES];
     
-    // final output is 48Khz, so use this to limit network audio plus upsampled injectors
-    static AudioLimiter finalAudioLimiter(2*AudioConstants::SAMPLE_RATE, AudioConstants::STEREO);
-    
-    // Injectors are 24khz, but we are running at 48Khz here, so need an AudioSRC to upsample
-    // TODO: probably an existing src the AudioClient is appropriate, check!
-    static AudioSRC audioSRC(AudioConstants::SAMPLE_RATE, 2*AudioConstants::SAMPLE_RATE, AudioConstants::STEREO);
-    
-    // limit maxSize
-    if (maxSize > AudioConstants::NETWORK_FRAME_SAMPLES_PER_CHANNEL * MAX_FRAMES) {
-        maxSize = AudioConstants::NETWORK_FRAME_SAMPLES_PER_CHANNEL * MAX_FRAMES; 
-    }
-
-    // initialize stuff
-    memset(hrtfBuffer, 0, sizeof(hrtfBuffer));
-    memset(mixed48Khz, 0, sizeof(mixed48Khz));
-    QVector<AudioInjector*> injectorsToRemove;
-    qint64 framesReadFromInjectors = 0;
-    
-    //qDebug() << "maxSize=" << maxSize;
-    
-    // first, grab the stuff from the network.  Read into a 
     if ((samplesPopped = _receivedAudioStream.popSamples((int)samplesRequested, false)) > 0) {
         AudioRingBuffer::ConstIterator lastPopOutput = _receivedAudioStream.getLastPopOutput();
         lastPopOutput.readSamples((int16_t*)data, samplesPopped);
-
-        //qDebug() << "AudioClient::AudioOutputIODevice::readData popped " << samplesPopped << "samples"; 
-
-        // no matter what, this is how many bytes will be written
         bytesWritten = samplesPopped * sizeof(int16_t);
-
-
-        // now, grab same number of frames (samplesPopped/2 since network is stereo) 
-        // and divide by 2 again because these are 24Khz not 48Khz
-        // from each of the localInjectors, if any
-        for (AudioInjector* injector : _audio->getActiveLocalAudioInjectors()) {
-            if (injector->getLocalBuffer() ) {
-
-                // we want to get the same number of frames that we got from 
-                // the network.  Since network is stereo, that is samplesPopped/2.  
-                // Since each frame is 2 bytes, we read samplesPopped bytes
-                qint64 bytesReadFromInjector = injector->getLocalBuffer()->readData((char*)scratchBuffer, samplesPopped/2);
-                qint64 framesReadFromInjector = bytesReadFromInjector/sizeof(int16_t);
-                
-                // keep track of max frames read for all injectors
-                if (framesReadFromInjector > framesReadFromInjectors) {
-                    framesReadFromInjectors = framesReadFromInjector;
-                }
-
-                if (framesReadFromInjector > 0) {
-
-                    //qDebug() << "AudioClient::AudioOutputIODevice::readData found " << framesReadFromInjector << " frames";
-
-                    // calculate gain and azimuth for hrtf
-                    glm::vec3 relativePosition = injector->getPosition() - _audio->_positionGetter();
-                    float gain = _audio->gainForSource(relativePosition, injector->getVolume()); 
-                    float azimuth = _audio->azimuthForSource(relativePosition); 
-
-                    // now hrtf this guy, one NETWORK_FRAME_SAMPLES_PER_CHANNEL at a time
-                    this->renderHRTF(injector->getLocalHRTF(), scratchBuffer, hrtfBuffer, azimuth, gain, framesReadFromInjector);
-
-                } else {
-
-                    qDebug() << "AudioClient::AudioOutputIODevice::readData no more data, adding to list of injectors to remove";
-                    injectorsToRemove.append(injector);
-                    injector->finish();
-                }
-            } else {
-
-                qDebug() << "AudioClient::AudioOutputIODevice::readData injector " << injector << " has no local buffer, adding to list of injectors to remove";
-                injectorsToRemove.append(injector);
-                injector->finish();
-            }
-        }
-
-        if (framesReadFromInjectors > 0) {
-            // resample the 24Khz injector audio in hrtfBuffer to 48Khz
-            audioSRC.render(hrtfBuffer, mixed48Khz, framesReadFromInjectors);
-            
-            //qDebug() << "upsampled to" << framesReadFromInjectors*2 << " frames, stereo";
-
-            int16_t* dataPtr = (int16_t*)data;
-            
-            // now mix in the network data
-            for (int i=0; i<samplesPopped; i++) {
-                mixed48Khz[i] += (float)(*dataPtr++) * 1/32676.0f;
-            }
-
-            //qDebug() << "mixed network data into upsampled hrtf'd mixed injector data" ;
-            
-            finalAudioLimiter.render(mixed48Khz, (int16_t*)data, samplesPopped/2);
-        }
     } else {
         // nothing on network, don't grab anything from injectors, and just 
         // return 0s
@@ -1333,14 +1290,6 @@ qint64 AudioClient::AudioOutputIODevice::readData(char * data, qint64 maxSize) {
         
     }
 
-    // k get rid of finished injectors
-    for (AudioInjector* injector : injectorsToRemove) {
-        _audio->getActiveLocalAudioInjectors().removeOne(injector);
-        //qDebug() << "removed injector " << injector << " from active injector list!";
-    }
-
-    //qDebug() << "bytesWritten=" << bytesWritten;
-    
     int bytesAudioOutputUnplayed = _audio->_audioOutput->bufferSize() - _audio->_audioOutput->bytesFree();
     if (!bytesAudioOutputUnplayed) {
         qCDebug(audioclient) << "empty audio buffer";
diff --git a/libraries/audio-client/src/AudioClient.h b/libraries/audio-client/src/AudioClient.h
index 91987178b0..3630358771 100644
--- a/libraries/audio-client/src/AudioClient.h
+++ b/libraries/audio-client/src/AudioClient.h
@@ -38,11 +38,14 @@
 #include <Sound.h>
 #include <StDev.h>
 #include <AudioHRTF.h>
+#include <AudioSRC.h>
+#include <AudioInjector.h>
+#include <AudioReverb.h>
+#include <AudioLimiter.h>
+#include <AudioConstants.h>
 
 #include "AudioIOStats.h"
 #include "AudioNoiseGate.h"
-#include "AudioSRC.h"
-#include "AudioReverb.h"
 
 #ifdef _WIN32
 #pragma warning( push )
@@ -92,7 +95,6 @@ public:
         MixedProcessedAudioStream& _receivedAudioStream;
         AudioClient* _audio;
         int _unfulfilledReads;
-        void renderHRTF(AudioHRTF& hrtf, int16_t* data, float* hrtfBuffer, float azimuth, float gain, qint64 numSamples);
     };
 
     const MixedProcessedAudioStream& getReceivedAudioStream() const { return _receivedAudioStream; }
@@ -208,6 +210,9 @@ protected:
 
 private:
     void outputFormatChanged();
+    void AudioClient::mixLocalAudioInjectors(int16_t* inputBuffer);
+    float azimuthForSource(const glm::vec3& relativePosition);
+    float gainForSource(const glm::vec3& relativePosition, float volume);
 
     QByteArray firstInputFrame;
     QAudioInput* _audioInput;
@@ -264,6 +269,11 @@ private:
     AudioSRC* _inputToNetworkResampler;
     AudioSRC* _networkToOutputResampler;
 
+    // for local hrtf-ing
+    float _hrtfBuffer[AudioConstants::NETWORK_FRAME_SAMPLES_STEREO];
+    int16_t _scratchBuffer[AudioConstants::NETWORK_FRAME_SAMPLES_STEREO];
+    AudioLimiter _audioLimiter;
+
     // Adds Reverb
     void configureReverb();
     void updateReverbOptions();
@@ -296,10 +306,6 @@ private:
     bool _hasReceivedFirstPacket = false;
     
     QVector<AudioInjector*> _activeLocalAudioInjectors;
-
-    float azimuthForSource(const glm::vec3& relativePosition);
-    float gainForSource(const glm::vec3& relativePosition, float volume);
-
 };
 
 

From 46dc5bea0404526d1e487c43d19ee7a325c8ea89 Mon Sep 17 00:00:00 2001
From: David Kelly <david@highfidelity.io>
Date: Mon, 11 Jul 2016 09:31:05 -0700
Subject: [PATCH 5/9] cleanup from PR comments

---
 libraries/audio-client/src/AudioClient.cpp | 6 +++---
 libraries/audio/src/AudioInjector.h        | 4 ----
 2 files changed, 3 insertions(+), 7 deletions(-)

diff --git a/libraries/audio-client/src/AudioClient.cpp b/libraries/audio-client/src/AudioClient.cpp
index 190dd1583b..b83165eac6 100644
--- a/libraries/audio-client/src/AudioClient.cpp
+++ b/libraries/audio-client/src/AudioClient.cpp
@@ -939,11 +939,11 @@ bool AudioClient::outputLocalInjector(bool isStereo, AudioInjector* injector) {
             // We don't want to stop this localOutput and injector whenever this AudioClient singleton goes idle,
             // only when the localOutput does. But the connection is to localOutput, so that it happens on the right thread.
             connect(localOutput, &QAudioOutput::stateChanged, localOutput, [=](QAudio::State state) {
-                    if (state == QAudio::IdleState) {
+                if (state == QAudio::IdleState) {
                     localOutput->stop();
                     injector->stop();
-                    }
-                    });
+                }
+            });
 
             connect(injector->getLocalBuffer(), &QIODevice::aboutToClose, localOutput, &QAudioOutput::stop);
 
diff --git a/libraries/audio/src/AudioInjector.h b/libraries/audio/src/AudioInjector.h
index a4cde77377..7560863b03 100644
--- a/libraries/audio/src/AudioInjector.h
+++ b/libraries/audio/src/AudioInjector.h
@@ -84,9 +84,6 @@ signals:
     void finished();
     void restarting();
     
-/*private slots:
-    void finish();
-*/  
 private:
     void setupInjection();
     int64_t injectNextFrame();
@@ -110,7 +107,6 @@ private:
     
     // when the injector is local, we need this
     AudioHRTF _localHRTF;
-    // for local injection, 
     friend class AudioInjectorManager;
 };
 

From 4f80c77b77b6ae808e4b8a33eb6405a5092e6f19 Mon Sep 17 00:00:00 2001
From: David Kelly <david@highfidelity.io>
Date: Mon, 11 Jul 2016 10:36:55 -0700
Subject: [PATCH 6/9] PR feedback

simple enough - not sure why I had an issue with this in the first
place.
---
 libraries/audio-client/src/AudioClient.cpp | 10 +++-------
 1 file changed, 3 insertions(+), 7 deletions(-)

diff --git a/libraries/audio-client/src/AudioClient.cpp b/libraries/audio-client/src/AudioClient.cpp
index b83165eac6..fad34f01de 100644
--- a/libraries/audio-client/src/AudioClient.cpp
+++ b/libraries/audio-client/src/AudioClient.cpp
@@ -849,16 +849,12 @@ void AudioClient::processReceivedSamples(const QByteArray& inputBuffer, QByteArr
     outputBuffer.resize(numDeviceOutputSamples * sizeof(int16_t));
     int16_t* outputSamples = reinterpret_cast<int16_t*>(outputBuffer.data());
     const int16_t* receivedSamples;
-    char inputBufferCopy[AudioConstants::NETWORK_FRAME_BYTES_STEREO];
+    QByteArray inputBufferCopy = inputBuffer;
     assert(inputBuffer.size() == AudioConstants::NETWORK_FRAME_BYTES_STEREO);
     
     if(getActiveLocalAudioInjectors().size() > 0) {
-        // gotta copy the since it is const
-        for(int i = 0; i < sizeof(inputBufferCopy); i++) {
-            inputBufferCopy[i] = inputBuffer.data()[i];
-        }
-        mixLocalAudioInjectors((int16_t*)inputBufferCopy);
-        receivedSamples = reinterpret_cast<const int16_t*>(inputBufferCopy);
+        mixLocalAudioInjectors((int16_t*)inputBufferCopy.data());
+        receivedSamples = reinterpret_cast<const int16_t*>(inputBufferCopy.data());
     } else {
         receivedSamples = reinterpret_cast<const int16_t*>(inputBuffer.data());
     }

From c84ef5f62688f181b6e99ee9ceb19d5205c003da Mon Sep 17 00:00:00 2001
From: David Kelly <david@highfidelity.io>
Date: Mon, 11 Jul 2016 11:31:47 -0700
Subject: [PATCH 7/9] Stereo injectors now handled same as mono

Except of course no HRTF.
---
 libraries/audio-client/src/AudioClient.cpp | 65 ++++++++--------------
 libraries/audio/src/AudioInjector.h        |  3 +-
 2 files changed, 26 insertions(+), 42 deletions(-)

diff --git a/libraries/audio-client/src/AudioClient.cpp b/libraries/audio-client/src/AudioClient.cpp
index fad34f01de..3178a53e94 100644
--- a/libraries/audio-client/src/AudioClient.cpp
+++ b/libraries/audio-client/src/AudioClient.cpp
@@ -788,24 +788,37 @@ void AudioClient::mixLocalAudioInjectors(int16_t* inputBuffer) {
 
     memset(_hrtfBuffer, 0, sizeof(_hrtfBuffer));
     QVector<AudioInjector*> injectorsToRemove;
+    static const float INT16_TO_FLOAT_SCALE_FACTOR = 1/32768.0f;
 
     bool injectorsHaveData = false;
 
     for (AudioInjector* injector : getActiveLocalAudioInjectors()) {
         if (injector->getLocalBuffer()) {
 
-            // get one (mono) frame from the injector
-            if (0 < injector->getLocalBuffer()->readData((char*)_scratchBuffer, AudioConstants::NETWORK_FRAME_SAMPLES_PER_CHANNEL*sizeof(int16_t)) ) {
+            qint64 samplesToRead = injector->isStereo() ? 
+                AudioConstants::NETWORK_FRAME_BYTES_STEREO :
+                AudioConstants::NETWORK_FRAME_BYTES_PER_CHANNEL;
+
+            // get one frame from the injector (mono or stereo)
+            if (0 < injector->getLocalBuffer()->readData((char*)_scratchBuffer, samplesToRead)) {
                 
                 injectorsHaveData = true;
 
-                // calculate gain and azimuth for hrtf
-                glm::vec3 relativePosition = injector->getPosition() - _positionGetter();
-                float gain = gainForSource(relativePosition, injector->getVolume()); 
-                float azimuth = azimuthForSource(relativePosition); 
+                if (injector->isStereo() ) {
+                    for(int i=0; i<AudioConstants::NETWORK_FRAME_SAMPLES_STEREO; i++) {
+                        _hrtfBuffer[i] += (float)(_scratchBuffer[i]) * INT16_TO_FLOAT_SCALE_FACTOR;
+                    }
+                    
+                } else {
+
+                    // calculate gain and azimuth for hrtf
+                    glm::vec3 relativePosition = injector->getPosition() - _positionGetter();
+                    float gain = gainForSource(relativePosition, injector->getVolume()); 
+                    float azimuth = azimuthForSource(relativePosition); 
                 
                 
-                injector->getLocalHRTF().render(_scratchBuffer, _hrtfBuffer, 1, azimuth, gain, AudioConstants::NETWORK_FRAME_SAMPLES_PER_CHANNEL);
+                    injector->getLocalHRTF().render(_scratchBuffer, _hrtfBuffer, 1, azimuth, gain, AudioConstants::NETWORK_FRAME_SAMPLES_PER_CHANNEL);
+                }
             
             } else {
                 
@@ -828,7 +841,7 @@ void AudioClient::mixLocalAudioInjectors(int16_t* inputBuffer) {
         
         // mix network into the hrtfBuffer
         for(int i=0; i<AudioConstants::NETWORK_FRAME_SAMPLES_STEREO; i++) {
-            _hrtfBuffer[i] += (float)(inputBuffer[i]) * 1/32678.0f;
+            _hrtfBuffer[i] += (float)(inputBuffer[i]) * INT16_TO_FLOAT_SCALE_FACTOR;
         }
         
         // now, use limiter to write back to the inputBuffer
@@ -920,42 +933,12 @@ void AudioClient::setIsStereoInput(bool isStereoInput) {
 
 bool AudioClient::outputLocalInjector(bool isStereo, AudioInjector* injector) {
     if (injector->getLocalBuffer() && _audioInput ) {
-        if(isStereo) {
-            QAudioFormat localFormat = _desiredOutputFormat;
-            localFormat.setChannelCount(isStereo ? 2 : 1);
-
-            QAudioOutput* localOutput = new QAudioOutput(getNamedAudioDeviceForMode(QAudio::AudioOutput, _outputAudioDeviceName),
-                    localFormat,
-                    injector->getLocalBuffer());
-
-            // move the localOutput to the same thread as the local injector buffer
-            localOutput->moveToThread(injector->getLocalBuffer()->thread());
-
-            // have it be stopped when that local buffer is about to close
-            // We don't want to stop this localOutput and injector whenever this AudioClient singleton goes idle,
-            // only when the localOutput does. But the connection is to localOutput, so that it happens on the right thread.
-            connect(localOutput, &QAudioOutput::stateChanged, localOutput, [=](QAudio::State state) {
-                if (state == QAudio::IdleState) {
-                    localOutput->stop();
-                    injector->stop();
-                }
-            });
-
-            connect(injector->getLocalBuffer(), &QIODevice::aboutToClose, localOutput, &QAudioOutput::stop);
-
-            qCDebug(audioclient) << "Starting QAudioOutput for local injector" << localOutput;
-
-            localOutput->start(injector->getLocalBuffer());
-            return localOutput->state() == QAudio::ActiveState;
-        } else {
         // just add it to the vector of active local injectors
         // TODO: deal with concurrency perhaps?  Maybe not
-            qDebug() << "adding new injector!!!!!!!";
-
-            _activeLocalAudioInjectors.append(injector);
-            return true;
-        }
+        qDebug() << "adding new injector!!!!!!!";
 
+        _activeLocalAudioInjectors.append(injector);
+        return true;
     }
 
     return false;
diff --git a/libraries/audio/src/AudioInjector.h b/libraries/audio/src/AudioInjector.h
index 7560863b03..c74497ce4d 100644
--- a/libraries/audio/src/AudioInjector.h
+++ b/libraries/audio/src/AudioInjector.h
@@ -59,7 +59,8 @@ public:
 
     bool isLocalOnly() const { return _options.localOnly; }
     float getVolume() const { return _options.volume; }
-    glm::vec3 getPosition() const { return _options.position; } 
+    glm::vec3 getPosition() const { return _options.position; }
+    bool isStereo() const { return _options.stereo; }
     void setLocalAudioInterface(AbstractAudioInterface* localAudioInterface) { _localAudioInterface = localAudioInterface; }
 
     static AudioInjector* playSoundAndDelete(const QByteArray& buffer, const AudioInjectorOptions options, AbstractAudioInterface* localInterface);

From 3955e03aa2fa5a770e034c3729477222cc77b2e8 Mon Sep 17 00:00:00 2001
From: David Kelly <david@highfidelity.io>
Date: Tue, 12 Jul 2016 11:38:30 -0700
Subject: [PATCH 8/9] Fix for build error

doh!
---
 libraries/audio-client/src/AudioClient.cpp | 4 ++--
 libraries/audio-client/src/AudioClient.h   | 2 +-
 2 files changed, 3 insertions(+), 3 deletions(-)

diff --git a/libraries/audio-client/src/AudioClient.cpp b/libraries/audio-client/src/AudioClient.cpp
index 3178a53e94..cb53e8f5de 100644
--- a/libraries/audio-client/src/AudioClient.cpp
+++ b/libraries/audio-client/src/AudioClient.cpp
@@ -102,13 +102,13 @@ AudioClient::AudioClient() :
     _reverbOptions(&_scriptReverbOptions),
     _inputToNetworkResampler(NULL),
     _networkToOutputResampler(NULL),
+    _audioLimiter(AudioConstants::SAMPLE_RATE, AudioConstants::STEREO),
     _outgoingAvatarAudioSequenceNumber(0),
     _audioOutputIODevice(_receivedAudioStream, this),
     _stats(&_receivedAudioStream),
     _inputGate(),
     _positionGetter(DEFAULT_POSITION_GETTER),
-    _orientationGetter(DEFAULT_ORIENTATION_GETTER),
-    _audioLimiter(AudioConstants::SAMPLE_RATE, AudioConstants::STEREO)
+    _orientationGetter(DEFAULT_ORIENTATION_GETTER)
 {
     // clear the array of locally injected samples
     memset(_localProceduralSamples, 0, AudioConstants::NETWORK_FRAME_BYTES_PER_CHANNEL);
diff --git a/libraries/audio-client/src/AudioClient.h b/libraries/audio-client/src/AudioClient.h
index 3630358771..49c3a40627 100644
--- a/libraries/audio-client/src/AudioClient.h
+++ b/libraries/audio-client/src/AudioClient.h
@@ -210,7 +210,7 @@ protected:
 
 private:
     void outputFormatChanged();
-    void AudioClient::mixLocalAudioInjectors(int16_t* inputBuffer);
+    void mixLocalAudioInjectors(int16_t* inputBuffer);
     float azimuthForSource(const glm::vec3& relativePosition);
     float gainForSource(const glm::vec3& relativePosition, float volume);
 

From c951f507e3be2383a88b43c48e69201ce653ecdc Mon Sep 17 00:00:00 2001
From: David Kelly <david@highfidelity.io>
Date: Wed, 13 Jul 2016 11:28:36 -0700
Subject: [PATCH 9/9] Fixed issue with calling setOptions

Since the stereo option is computed from the .wav file, if you call
setOptions later (like the cow.js does), it resets stereo to false.
So, I now just copy the stereo flag into the new options, since the
sound file is the same.

Also, calling AudioClient::outputLocalInjector on the AudioClient
thread now to avoid potential concurrency issues accessing the
vector of injectors.
---
 libraries/audio-client/src/AudioClient.cpp | 25 ++++++++++++++--------
 libraries/audio/src/AudioInjector.cpp      | 20 +++++++++++++----
 libraries/audio/src/AudioInjector.h        |  3 +--
 3 files changed, 33 insertions(+), 15 deletions(-)

diff --git a/libraries/audio-client/src/AudioClient.cpp b/libraries/audio-client/src/AudioClient.cpp
index cb53e8f5de..799c357944 100644
--- a/libraries/audio-client/src/AudioClient.cpp
+++ b/libraries/audio-client/src/AudioClient.cpp
@@ -823,7 +823,6 @@ void AudioClient::mixLocalAudioInjectors(int16_t* inputBuffer) {
             } else {
                 
                 qDebug() << "injector has no more data, marking finished for removal";
-                
                 injector->finish();
                 injectorsToRemove.append(injector);
             }
@@ -831,7 +830,6 @@ void AudioClient::mixLocalAudioInjectors(int16_t* inputBuffer) {
         } else {
             
             qDebug() << "injector has no local buffer, marking as finished for removal";
-            
             injector->finish();
             injectorsToRemove.append(injector);
         }
@@ -933,15 +931,24 @@ void AudioClient::setIsStereoInput(bool isStereoInput) {
 
 bool AudioClient::outputLocalInjector(bool isStereo, AudioInjector* injector) {
     if (injector->getLocalBuffer() && _audioInput ) {
-        // just add it to the vector of active local injectors
-        // TODO: deal with concurrency perhaps?  Maybe not
-        qDebug() << "adding new injector!!!!!!!";
-
-        _activeLocalAudioInjectors.append(injector);
+        // just add it to the vector of active local injectors, if 
+        // not already there.
+        // Since this is invoked with invokeMethod, there _should_ be
+        // no reason to lock access to the vector of injectors.
+        if (!_activeLocalAudioInjectors.contains(injector)) {
+            qDebug() << "adding new injector";
+            
+            _activeLocalAudioInjectors.append(injector);
+        } else {
+            qDebug() << "injector exists in active list already";
+        }
+        
         return true;
-    }
 
-    return false;
+    } else {
+        // no local buffer or audio
+        return false;
+    }
 }
 
 void AudioClient::outputFormatChanged() {
diff --git a/libraries/audio/src/AudioInjector.cpp b/libraries/audio/src/AudioInjector.cpp
index 878a4c627c..d4382340bf 100644
--- a/libraries/audio/src/AudioInjector.cpp
+++ b/libraries/audio/src/AudioInjector.cpp
@@ -26,6 +26,8 @@
 
 #include "AudioInjector.h"
 
+int audioInjectorPtrMetaTypeId = qRegisterMetaType<AudioInjector*>();
+
 AudioInjector::AudioInjector(QObject* parent) :
     QObject(parent)
 {
@@ -41,11 +43,20 @@ AudioInjector::AudioInjector(const Sound& sound, const AudioInjectorOptions& inj
 
 AudioInjector::AudioInjector(const QByteArray& audioData, const AudioInjectorOptions& injectorOptions) :
     _audioData(audioData),
-    _options(injectorOptions)
+    _options(injectorOptions) 
 {
 
 }
 
+void AudioInjector::setOptions(const AudioInjectorOptions& options) {
+    // since options.stereo is computed from the audio stream, 
+    // we need to copy it from existing options just in case.
+    bool currentlyStereo = _options.stereo;
+    _options = options;
+    _options.stereo = currentlyStereo;
+}
+
+
 void AudioInjector::finish() {
     bool shouldDelete = (_state == State::NotFinishedWithPendingDelete);
     _state = State::Finished;
@@ -115,11 +126,11 @@ void AudioInjector::restart() {
         _hasSetup = false;
         _shouldStop = false;
         _state = State::NotFinished;
-
+        
         // call inject audio to start injection over again
         setupInjection();
 
-        // if we're a local injector call inject locally to start injecting again
+        // if we're a local injector, just inject again 
         if (_options.localOnly) {
             injectLocally();
         } else {
@@ -145,7 +156,8 @@ bool AudioInjector::injectLocally() {
             // give our current send position to the local buffer
             _localBuffer->setCurrentOffset(_currentSendOffset);
 
-            success = _localAudioInterface->outputLocalInjector(_options.stereo, this);
+            // call this function on the AudioClient's thread
+            success = QMetaObject::invokeMethod(_localAudioInterface, "outputLocalInjector", Q_ARG(bool, _options.stereo), Q_ARG(AudioInjector*, this));
 
             if (!success) {
                 qCDebug(audio) << "AudioInjector::injectLocally could not output locally via _localAudioInterface";
diff --git a/libraries/audio/src/AudioInjector.h b/libraries/audio/src/AudioInjector.h
index c74497ce4d..1af733ace6 100644
--- a/libraries/audio/src/AudioInjector.h
+++ b/libraries/audio/src/AudioInjector.h
@@ -75,7 +75,7 @@ public slots:
     void stopAndDeleteLater();
     
     const AudioInjectorOptions& getOptions() const { return _options; }
-    void setOptions(const AudioInjectorOptions& options) { _options = options;  }
+    void setOptions(const AudioInjectorOptions& options);
     
     float getLoudness() const { return _loudness; }
     bool isPlaying() const { return _state == State::NotFinished || _state == State::NotFinishedWithPendingDelete; }
@@ -111,5 +111,4 @@ private:
     friend class AudioInjectorManager;
 };
 
-
 #endif // hifi_AudioInjector_h