Merge pull request #8209 from davidkelly/dk/firstLameAttempt

Initial cut of htrf for mono localOnly injectors

libraries/audio-client/src/AudioClient.cpp

@@ -14,6 +14,8 @@
 #include <sys/stat.h>
 
 #include <glm/glm.hpp>
+#include <glm/gtx/norm.hpp>
+#include <glm/gtx/vector_angle.hpp>
 
 #ifdef __APPLE__
 #include <CoreAudio/AudioHardware.h>
@@ -43,8 +45,6 @@
 #include <UUID.h>
 #include <Transform.h>
 
-#include "AudioInjector.h"
-#include "AudioConstants.h"
 #include "PositionalAudioStream.h"
 #include "AudioClientLogging.h"
 
@@ -104,6 +104,7 @@ AudioClient::AudioClient() :
     _reverbOptions(&_scriptReverbOptions),
     _inputToNetworkResampler(NULL),
     _networkToOutputResampler(NULL),
+    _audioLimiter(AudioConstants::SAMPLE_RATE, AudioConstants::STEREO),
     _outgoingAvatarAudioSequenceNumber(0),
     _audioOutputIODevice(_receivedAudioStream, this),
     _stats(&_receivedAudioStream),
@@ -852,6 +853,74 @@ void AudioClient::handleRecordedAudioInput(const QByteArray& audio) {
     emitAudioPacket(encodedBuffer.data(), encodedBuffer.size(), _outgoingAvatarAudioSequenceNumber, audioTransform, PacketType::MicrophoneAudioWithEcho);
 }
 
+void AudioClient::mixLocalAudioInjectors(int16_t* inputBuffer) {
+
+    memset(_hrtfBuffer, 0, sizeof(_hrtfBuffer));
+    QVector<AudioInjector*> injectorsToRemove;
+    static const float INT16_TO_FLOAT_SCALE_FACTOR = 1/32768.0f;
+
+    bool injectorsHaveData = false;
+
+    for (AudioInjector* injector : getActiveLocalAudioInjectors()) {
+        if (injector->getLocalBuffer()) {
+
+            qint64 samplesToRead = injector->isStereo() ? 
+                AudioConstants::NETWORK_FRAME_BYTES_STEREO :
+                AudioConstants::NETWORK_FRAME_BYTES_PER_CHANNEL;
+
+            // get one frame from the injector (mono or stereo)
+            if (0 < injector->getLocalBuffer()->readData((char*)_scratchBuffer, samplesToRead)) {
+                
+                injectorsHaveData = true;
+
+                if (injector->isStereo() ) {
+                    for(int i=0; i<AudioConstants::NETWORK_FRAME_SAMPLES_STEREO; i++) {
+                        _hrtfBuffer[i] += (float)(_scratchBuffer[i]) * INT16_TO_FLOAT_SCALE_FACTOR;
+                    }
+                    
+                } else {
+
+                    // calculate gain and azimuth for hrtf
+                    glm::vec3 relativePosition = injector->getPosition() - _positionGetter();
+                    float gain = gainForSource(relativePosition, injector->getVolume()); 
+                    float azimuth = azimuthForSource(relativePosition); 
+                
+                
+                    injector->getLocalHRTF().render(_scratchBuffer, _hrtfBuffer, 1, azimuth, gain, AudioConstants::NETWORK_FRAME_SAMPLES_PER_CHANNEL);
+                }
+            
+            } else {
+                
+                qDebug() << "injector has no more data, marking finished for removal";
+                injector->finish();
+                injectorsToRemove.append(injector);
+            }
+
+        } else {
+            
+            qDebug() << "injector has no local buffer, marking as finished for removal";
+            injector->finish();
+            injectorsToRemove.append(injector);
+        }
+    }
+    
+    if(injectorsHaveData) {
+        
+        // mix network into the hrtfBuffer
+        for(int i=0; i<AudioConstants::NETWORK_FRAME_SAMPLES_STEREO; i++) {
+            _hrtfBuffer[i] += (float)(inputBuffer[i]) * INT16_TO_FLOAT_SCALE_FACTOR;
+        }
+        
+        // now, use limiter to write back to the inputBuffer
+        _audioLimiter.render(_hrtfBuffer, inputBuffer, AudioConstants::NETWORK_FRAME_SAMPLES_PER_CHANNEL);
+    }
+
+    for(AudioInjector* injector : injectorsToRemove) {
+        qDebug() << "removing injector";
+        getActiveLocalAudioInjectors().removeOne(injector);
+    }
+}
+
 void AudioClient::processReceivedSamples(const QByteArray& decodedBuffer, QByteArray& outputBuffer) {
     const int numDecodecSamples = decodedBuffer.size() / sizeof(int16_t);
     const int numDeviceOutputSamples = _outputFrameSize;
@@ -861,8 +930,17 @@ void AudioClient::processReceivedSamples(const QByteArray& decodedBuffer, QByteA
 
     outputBuffer.resize(numDeviceOutputSamples * sizeof(int16_t));
 
-    const int16_t* decodedSamples = reinterpret_cast<const int16_t*>(decodedBuffer.data());
+    const int16_t* decodedSamples;
     int16_t* outputSamples = reinterpret_cast<int16_t*>(outputBuffer.data());
+    QByteArray decodedBufferCopy = decodedBuffer;
+    assert(decodedBuffer.size() == AudioConstants::NETWORK_FRAME_BYTES_STEREO);
+    
+    if(getActiveLocalAudioInjectors().size() > 0) {
+        mixLocalAudioInjectors((int16_t*)decodedBufferCopy.data());
+        decodedSamples = reinterpret_cast<const int16_t*>(decodedBufferCopy.data());
+    } else {
+        decodedSamples = reinterpret_cast<const int16_t*>(decodedBuffer.data());
+    }
 
     // copy the packet from the RB to the output
     possibleResampling(_networkToOutputResampler, decodedSamples, outputSamples,
@@ -874,6 +952,7 @@ void AudioClient::processReceivedSamples(const QByteArray& decodedBuffer, QByteA
     if (hasReverb) {
         assert(_outputFormat.channelCount() == 2);
         updateReverbOptions();
+        qDebug() << "handling reverb";
         _listenerReverb.render(outputSamples, outputSamples, numDeviceOutputSamples/2);
     }
 }
@@ -923,36 +1002,25 @@ void AudioClient::setIsStereoInput(bool isStereoInput) {
 
 
 bool AudioClient::outputLocalInjector(bool isStereo, AudioInjector* injector) {
-    if (injector->getLocalBuffer()) {
+    if (injector->getLocalBuffer() && _audioInput ) {
-        QAudioFormat localFormat = _desiredOutputFormat;
+        // just add it to the vector of active local injectors, if 
-        localFormat.setChannelCount(isStereo ? 2 : 1);
+        // not already there.
+        // Since this is invoked with invokeMethod, there _should_ be
+        // no reason to lock access to the vector of injectors.
+        if (!_activeLocalAudioInjectors.contains(injector)) {
+            qDebug() << "adding new injector";
+            
+            _activeLocalAudioInjectors.append(injector);
+        } else {
+            qDebug() << "injector exists in active list already";
+        }
+        
+        return true;
 
-        QAudioOutput* localOutput = new QAudioOutput(getNamedAudioDeviceForMode(QAudio::AudioOutput, _outputAudioDeviceName),
+    } else {
-                                                     localFormat,
+        // no local buffer or audio
-                                                     injector->getLocalBuffer());
+        return false;
-
-        // move the localOutput to the same thread as the local injector buffer
-        localOutput->moveToThread(injector->getLocalBuffer()->thread());
-
-        // have it be stopped when that local buffer is about to close
-        // We don't want to stop this localOutput and injector whenever this AudioClient singleton goes idle,
-        // only when the localOutput does. But the connection is to localOutput, so that it happens on the right thread.
-        connect(localOutput, &QAudioOutput::stateChanged, localOutput, [=](QAudio::State state) {
-            if (state == QAudio::IdleState) {
-                localOutput->stop();
-                injector->stop();
-            }
-        });
-
-        connect(injector->getLocalBuffer(), &QIODevice::aboutToClose, localOutput, &QAudioOutput::stop);
-
-        qCDebug(audioclient) << "Starting QAudioOutput for local injector" << localOutput;
-
-        localOutput->start(injector->getLocalBuffer());
-        return localOutput->state() == QAudio::ActiveState;
     }
-
-    return false;
 }
 
 void AudioClient::outputFormatChanged() {
@@ -1205,18 +1273,79 @@ float AudioClient::getAudioOutputMsecsUnplayed() const {
     return msecsAudioOutputUnplayed;
 }
 
+
+float AudioClient::azimuthForSource(const glm::vec3& relativePosition) {
+    // copied from AudioMixer, more or less
+    glm::quat inverseOrientation = glm::inverse(_orientationGetter());
+    
+    // compute sample delay for the 2 ears to create phase panning
+    glm::vec3 rotatedSourcePosition = inverseOrientation * relativePosition;
+    
+    // project the rotated source position vector onto x-y plane
+    rotatedSourcePosition.y = 0.0f;
+
+    static const float SOURCE_DISTANCE_THRESHOLD = 1e-30f;
+
+    if (glm::length2(rotatedSourcePosition) > SOURCE_DISTANCE_THRESHOLD) {
+        
+        // produce an oriented angle about the y-axis
+        return glm::orientedAngle(glm::vec3(0.0f, 0.0f, -1.0f), glm::normalize(rotatedSourcePosition), glm::vec3(0.0f, -1.0f, 0.0f));
+    } else {
+        
+        // no azimuth if they are in same spot
+        return 0.0f; 
+    }
+}
+
+float AudioClient::gainForSource(const glm::vec3& relativePosition, float volume) {
+    // TODO: put these in a place where we can share with AudioMixer!
+    const float DEFAULT_ATTENUATION_PER_DOUBLING_IN_DISTANCE = 0.18f;
+    const float ATTENUATION_BEGINS_AT_DISTANCE = 1.0f;
+
+
+    //qDebug() << "initial gain is " << volume;
+    
+    // I'm assuming that the AudioMixer's getting of the stream's attenuation
+    // factor is basically same as getting volume
+    float gain = volume;
+    float distanceBetween = glm::length(relativePosition);
+    if (distanceBetween < EPSILON ) {
+        distanceBetween = EPSILON;
+    }
+
+    // audio mixer has notion of zones.  Unsure how to map that across here...
+
+    // attenuate based on distance now
+    if (distanceBetween >= ATTENUATION_BEGINS_AT_DISTANCE) {
+        float distanceCoefficient = 1.0f - (logf(distanceBetween/ATTENUATION_BEGINS_AT_DISTANCE) / logf(2.0f)
+                * DEFAULT_ATTENUATION_PER_DOUBLING_IN_DISTANCE);
+        if (distanceCoefficient < 0.0f) {
+            distanceCoefficient = 0.0f;
+        }
+
+        gain *= distanceCoefficient;
+    }
+    
+    //qDebug() << "calculated gain as " << gain;
+
+    return gain;
+}
+
 qint64 AudioClient::AudioOutputIODevice::readData(char * data, qint64 maxSize) {
     auto samplesRequested = maxSize / sizeof(int16_t);
     int samplesPopped;
     int bytesWritten;
-
+    
     if ((samplesPopped = _receivedAudioStream.popSamples((int)samplesRequested, false)) > 0) {
         AudioRingBuffer::ConstIterator lastPopOutput = _receivedAudioStream.getLastPopOutput();
         lastPopOutput.readSamples((int16_t*)data, samplesPopped);
         bytesWritten = samplesPopped * sizeof(int16_t);
     } else {
+        // nothing on network, don't grab anything from injectors, and just 
+        // return 0s
         memset(data, 0, maxSize);
         bytesWritten = maxSize;
+        
     }
 
     int bytesAudioOutputUnplayed = _audio->_audioOutput->bufferSize() - _audio->_audioOutput->bytesFree();

libraries/audio-client/src/AudioClient.h

@@ -37,13 +37,17 @@
 #include <SettingHandle.h>
 #include <Sound.h>
 #include <StDev.h>
+#include <AudioHRTF.h>
+#include <AudioSRC.h>
+#include <AudioInjector.h>
+#include <AudioReverb.h>
+#include <AudioLimiter.h>
+#include <AudioConstants.h>
 
 #include <plugins/CodecPlugin.h>
 
 #include "AudioIOStats.h"
 #include "AudioNoiseGate.h"
-#include "AudioSRC.h"
-#include "AudioReverb.h"
 
 #ifdef _WIN32
 #pragma warning( push )
@@ -88,7 +92,6 @@ public:
         void stop() { close(); }
         qint64    readData(char * data, qint64 maxSize);
         qint64    writeData(const char * data, qint64 maxSize) { return 0; }
-
         int getRecentUnfulfilledReads() { int unfulfilledReads = _unfulfilledReads; _unfulfilledReads = 0; return unfulfilledReads; }
     private:
         MixedProcessedAudioStream& _receivedAudioStream;
@@ -128,6 +131,8 @@ public:
 
     void setPositionGetter(AudioPositionGetter positionGetter) { _positionGetter = positionGetter; }
     void setOrientationGetter(AudioOrientationGetter orientationGetter) { _orientationGetter = orientationGetter; }
+    
+    QVector<AudioInjector*>& getActiveLocalAudioInjectors() { return _activeLocalAudioInjectors; }
 
     static const float CALLBACK_ACCELERATOR_RATIO;
 
@@ -210,6 +215,9 @@ protected:
 
 private:
     void outputFormatChanged();
+    void mixLocalAudioInjectors(int16_t* inputBuffer);
+    float azimuthForSource(const glm::vec3& relativePosition);
+    float gainForSource(const glm::vec3& relativePosition, float volume);
 
     QByteArray firstInputFrame;
     QAudioInput* _audioInput;
@@ -266,6 +274,11 @@ private:
     AudioSRC* _inputToNetworkResampler;
     AudioSRC* _networkToOutputResampler;
 
+    // for local hrtf-ing
+    float _hrtfBuffer[AudioConstants::NETWORK_FRAME_SAMPLES_STEREO];
+    int16_t _scratchBuffer[AudioConstants::NETWORK_FRAME_SAMPLES_STEREO];
+    AudioLimiter _audioLimiter;
+
     // Adds Reverb
     void configureReverb();
     void updateReverbOptions();
@@ -296,6 +309,8 @@ private:
     void checkDevices();
 
     bool _hasReceivedFirstPacket = false;
+    
+    QVector<AudioInjector*> _activeLocalAudioInjectors;
 
     CodecPluginPointer _codec;
     QString _selectedCodecName;

libraries/audio/src/AudioInjector.cpp

@@ -26,6 +26,8 @@
 
 #include "AudioInjector.h"
 
+int audioInjectorPtrMetaTypeId = qRegisterMetaType<AudioInjector*>();
+
 AudioInjector::AudioInjector(QObject* parent) :
     QObject(parent)
 {
@@ -41,11 +43,20 @@ AudioInjector::AudioInjector(const Sound& sound, const AudioInjectorOptions& inj
 
 AudioInjector::AudioInjector(const QByteArray& audioData, const AudioInjectorOptions& injectorOptions) :
     _audioData(audioData),
-    _options(injectorOptions)
+    _options(injectorOptions) 
 {
 
 }
 
+void AudioInjector::setOptions(const AudioInjectorOptions& options) {
+    // since options.stereo is computed from the audio stream, 
+    // we need to copy it from existing options just in case.
+    bool currentlyStereo = _options.stereo;
+    _options = options;
+    _options.stereo = currentlyStereo;
+}
+
+
 void AudioInjector::finish() {
     bool shouldDelete = (_state == State::NotFinishedWithPendingDelete);
     _state = State::Finished;
@@ -115,11 +126,11 @@ void AudioInjector::restart() {
         _hasSetup = false;
         _shouldStop = false;
         _state = State::NotFinished;
-
+        
         // call inject audio to start injection over again
         setupInjection();
 
-        // if we're a local injector call inject locally to start injecting again
+        // if we're a local injector, just inject again 
         if (_options.localOnly) {
             injectLocally();
         } else {
@@ -145,7 +156,8 @@ bool AudioInjector::injectLocally() {
             // give our current send position to the local buffer
             _localBuffer->setCurrentOffset(_currentSendOffset);
 
-            success = _localAudioInterface->outputLocalInjector(_options.stereo, this);
+            // call this function on the AudioClient's thread
+            success = QMetaObject::invokeMethod(_localAudioInterface, "outputLocalInjector", Q_ARG(bool, _options.stereo), Q_ARG(AudioInjector*, this));
 
             if (!success) {
                 qCDebug(audio) << "AudioInjector::injectLocally could not output locally via _localAudioInterface";

libraries/audio/src/AudioInjector.h

@@ -26,6 +26,7 @@
 
 #include "AudioInjectorLocalBuffer.h"
 #include "AudioInjectorOptions.h"
+#include "AudioHRTF.h"
 #include "Sound.h"
 
 class AbstractAudioInterface;
@@ -54,8 +55,12 @@ public:
     void setCurrentSendOffset(int currentSendOffset) { _currentSendOffset = currentSendOffset; }
     
     AudioInjectorLocalBuffer* getLocalBuffer() const { return _localBuffer; }
+    AudioHRTF& getLocalHRTF() { return _localHRTF; }
+
     bool isLocalOnly() const { return _options.localOnly; }
-    
+    float getVolume() const { return _options.volume; }
+    glm::vec3 getPosition() const { return _options.position; }
+    bool isStereo() const { return _options.stereo; }
     void setLocalAudioInterface(AbstractAudioInterface* localAudioInterface) { _localAudioInterface = localAudioInterface; }
 
     static AudioInjector* playSoundAndDelete(const QByteArray& buffer, const AudioInjectorOptions options, AbstractAudioInterface* localInterface);
@@ -70,18 +75,16 @@ public slots:
     void stopAndDeleteLater();
     
     const AudioInjectorOptions& getOptions() const { return _options; }
-    void setOptions(const AudioInjectorOptions& options) { _options = options;  }
+    void setOptions(const AudioInjectorOptions& options);
     
     float getLoudness() const { return _loudness; }
     bool isPlaying() const { return _state == State::NotFinished || _state == State::NotFinishedWithPendingDelete; }
+    void finish();
     
 signals:
     void finished();
     void restarting();
     
-private slots:
-    void finish();
-    
 private:
     void setupInjection();
     int64_t injectNextFrame();
@@ -103,8 +106,9 @@ private:
     std::unique_ptr<QElapsedTimer> _frameTimer { nullptr };
     quint16 _outgoingSequenceNumber { 0 };
     
+    // when the injector is local, we need this
+    AudioHRTF _localHRTF;
     friend class AudioInjectorManager;
 };
 
-
 #endif // hifi_AudioInjector_h