diff --git a/libraries/audio-client/src/AudioClient.cpp b/libraries/audio-client/src/AudioClient.cpp index 0cd01bb579..c9ddd13830 100644 --- a/libraries/audio-client/src/AudioClient.cpp +++ b/libraries/audio-client/src/AudioClient.cpp @@ -14,6 +14,8 @@ #include #include +#include +#include #ifdef __APPLE__ #include @@ -43,8 +45,6 @@ #include #include -#include "AudioInjector.h" -#include "AudioConstants.h" #include "PositionalAudioStream.h" #include "AudioClientLogging.h" @@ -104,6 +104,7 @@ AudioClient::AudioClient() : _reverbOptions(&_scriptReverbOptions), _inputToNetworkResampler(NULL), _networkToOutputResampler(NULL), + _audioLimiter(AudioConstants::SAMPLE_RATE, AudioConstants::STEREO), _outgoingAvatarAudioSequenceNumber(0), _audioOutputIODevice(_receivedAudioStream, this), _stats(&_receivedAudioStream), @@ -852,6 +853,74 @@ void AudioClient::handleRecordedAudioInput(const QByteArray& audio) { emitAudioPacket(encodedBuffer.data(), encodedBuffer.size(), _outgoingAvatarAudioSequenceNumber, audioTransform, PacketType::MicrophoneAudioWithEcho); } +void AudioClient::mixLocalAudioInjectors(int16_t* inputBuffer) { + + memset(_hrtfBuffer, 0, sizeof(_hrtfBuffer)); + QVector injectorsToRemove; + static const float INT16_TO_FLOAT_SCALE_FACTOR = 1/32768.0f; + + bool injectorsHaveData = false; + + for (AudioInjector* injector : getActiveLocalAudioInjectors()) { + if (injector->getLocalBuffer()) { + + qint64 samplesToRead = injector->isStereo() ? + AudioConstants::NETWORK_FRAME_BYTES_STEREO : + AudioConstants::NETWORK_FRAME_BYTES_PER_CHANNEL; + + // get one frame from the injector (mono or stereo) + if (0 < injector->getLocalBuffer()->readData((char*)_scratchBuffer, samplesToRead)) { + + injectorsHaveData = true; + + if (injector->isStereo() ) { + for(int i=0; igetPosition() - _positionGetter(); + float gain = gainForSource(relativePosition, injector->getVolume()); + float azimuth = azimuthForSource(relativePosition); + + + injector->getLocalHRTF().render(_scratchBuffer, _hrtfBuffer, 1, azimuth, gain, AudioConstants::NETWORK_FRAME_SAMPLES_PER_CHANNEL); + } + + } else { + + qDebug() << "injector has no more data, marking finished for removal"; + injector->finish(); + injectorsToRemove.append(injector); + } + + } else { + + qDebug() << "injector has no local buffer, marking as finished for removal"; + injector->finish(); + injectorsToRemove.append(injector); + } + } + + if(injectorsHaveData) { + + // mix network into the hrtfBuffer + for(int i=0; i(decodedBuffer.data()); + const int16_t* decodedSamples; int16_t* outputSamples = reinterpret_cast(outputBuffer.data()); + QByteArray decodedBufferCopy = decodedBuffer; + assert(decodedBuffer.size() == AudioConstants::NETWORK_FRAME_BYTES_STEREO); + + if(getActiveLocalAudioInjectors().size() > 0) { + mixLocalAudioInjectors((int16_t*)decodedBufferCopy.data()); + decodedSamples = reinterpret_cast(decodedBufferCopy.data()); + } else { + decodedSamples = reinterpret_cast(decodedBuffer.data()); + } // copy the packet from the RB to the output possibleResampling(_networkToOutputResampler, decodedSamples, outputSamples, @@ -874,6 +952,7 @@ void AudioClient::processReceivedSamples(const QByteArray& decodedBuffer, QByteA if (hasReverb) { assert(_outputFormat.channelCount() == 2); updateReverbOptions(); + qDebug() << "handling reverb"; _listenerReverb.render(outputSamples, outputSamples, numDeviceOutputSamples/2); } } @@ -923,36 +1002,25 @@ void AudioClient::setIsStereoInput(bool isStereoInput) { bool AudioClient::outputLocalInjector(bool isStereo, AudioInjector* injector) { - if (injector->getLocalBuffer()) { - QAudioFormat localFormat = _desiredOutputFormat; - localFormat.setChannelCount(isStereo ? 2 : 1); + if (injector->getLocalBuffer() && _audioInput ) { + // just add it to the vector of active local injectors, if + // not already there. + // Since this is invoked with invokeMethod, there _should_ be + // no reason to lock access to the vector of injectors. + if (!_activeLocalAudioInjectors.contains(injector)) { + qDebug() << "adding new injector"; + + _activeLocalAudioInjectors.append(injector); + } else { + qDebug() << "injector exists in active list already"; + } + + return true; - QAudioOutput* localOutput = new QAudioOutput(getNamedAudioDeviceForMode(QAudio::AudioOutput, _outputAudioDeviceName), - localFormat, - injector->getLocalBuffer()); - - // move the localOutput to the same thread as the local injector buffer - localOutput->moveToThread(injector->getLocalBuffer()->thread()); - - // have it be stopped when that local buffer is about to close - // We don't want to stop this localOutput and injector whenever this AudioClient singleton goes idle, - // only when the localOutput does. But the connection is to localOutput, so that it happens on the right thread. - connect(localOutput, &QAudioOutput::stateChanged, localOutput, [=](QAudio::State state) { - if (state == QAudio::IdleState) { - localOutput->stop(); - injector->stop(); - } - }); - - connect(injector->getLocalBuffer(), &QIODevice::aboutToClose, localOutput, &QAudioOutput::stop); - - qCDebug(audioclient) << "Starting QAudioOutput for local injector" << localOutput; - - localOutput->start(injector->getLocalBuffer()); - return localOutput->state() == QAudio::ActiveState; + } else { + // no local buffer or audio + return false; } - - return false; } void AudioClient::outputFormatChanged() { @@ -1205,18 +1273,79 @@ float AudioClient::getAudioOutputMsecsUnplayed() const { return msecsAudioOutputUnplayed; } + +float AudioClient::azimuthForSource(const glm::vec3& relativePosition) { + // copied from AudioMixer, more or less + glm::quat inverseOrientation = glm::inverse(_orientationGetter()); + + // compute sample delay for the 2 ears to create phase panning + glm::vec3 rotatedSourcePosition = inverseOrientation * relativePosition; + + // project the rotated source position vector onto x-y plane + rotatedSourcePosition.y = 0.0f; + + static const float SOURCE_DISTANCE_THRESHOLD = 1e-30f; + + if (glm::length2(rotatedSourcePosition) > SOURCE_DISTANCE_THRESHOLD) { + + // produce an oriented angle about the y-axis + return glm::orientedAngle(glm::vec3(0.0f, 0.0f, -1.0f), glm::normalize(rotatedSourcePosition), glm::vec3(0.0f, -1.0f, 0.0f)); + } else { + + // no azimuth if they are in same spot + return 0.0f; + } +} + +float AudioClient::gainForSource(const glm::vec3& relativePosition, float volume) { + // TODO: put these in a place where we can share with AudioMixer! + const float DEFAULT_ATTENUATION_PER_DOUBLING_IN_DISTANCE = 0.18f; + const float ATTENUATION_BEGINS_AT_DISTANCE = 1.0f; + + + //qDebug() << "initial gain is " << volume; + + // I'm assuming that the AudioMixer's getting of the stream's attenuation + // factor is basically same as getting volume + float gain = volume; + float distanceBetween = glm::length(relativePosition); + if (distanceBetween < EPSILON ) { + distanceBetween = EPSILON; + } + + // audio mixer has notion of zones. Unsure how to map that across here... + + // attenuate based on distance now + if (distanceBetween >= ATTENUATION_BEGINS_AT_DISTANCE) { + float distanceCoefficient = 1.0f - (logf(distanceBetween/ATTENUATION_BEGINS_AT_DISTANCE) / logf(2.0f) + * DEFAULT_ATTENUATION_PER_DOUBLING_IN_DISTANCE); + if (distanceCoefficient < 0.0f) { + distanceCoefficient = 0.0f; + } + + gain *= distanceCoefficient; + } + + //qDebug() << "calculated gain as " << gain; + + return gain; +} + qint64 AudioClient::AudioOutputIODevice::readData(char * data, qint64 maxSize) { auto samplesRequested = maxSize / sizeof(int16_t); int samplesPopped; int bytesWritten; - + if ((samplesPopped = _receivedAudioStream.popSamples((int)samplesRequested, false)) > 0) { AudioRingBuffer::ConstIterator lastPopOutput = _receivedAudioStream.getLastPopOutput(); lastPopOutput.readSamples((int16_t*)data, samplesPopped); bytesWritten = samplesPopped * sizeof(int16_t); } else { + // nothing on network, don't grab anything from injectors, and just + // return 0s memset(data, 0, maxSize); bytesWritten = maxSize; + } int bytesAudioOutputUnplayed = _audio->_audioOutput->bufferSize() - _audio->_audioOutput->bytesFree(); diff --git a/libraries/audio-client/src/AudioClient.h b/libraries/audio-client/src/AudioClient.h index bd0afe453d..df7b10ab04 100644 --- a/libraries/audio-client/src/AudioClient.h +++ b/libraries/audio-client/src/AudioClient.h @@ -37,13 +37,17 @@ #include #include #include +#include +#include +#include +#include +#include +#include #include #include "AudioIOStats.h" #include "AudioNoiseGate.h" -#include "AudioSRC.h" -#include "AudioReverb.h" #ifdef _WIN32 #pragma warning( push ) @@ -88,7 +92,6 @@ public: void stop() { close(); } qint64 readData(char * data, qint64 maxSize); qint64 writeData(const char * data, qint64 maxSize) { return 0; } - int getRecentUnfulfilledReads() { int unfulfilledReads = _unfulfilledReads; _unfulfilledReads = 0; return unfulfilledReads; } private: MixedProcessedAudioStream& _receivedAudioStream; @@ -128,6 +131,8 @@ public: void setPositionGetter(AudioPositionGetter positionGetter) { _positionGetter = positionGetter; } void setOrientationGetter(AudioOrientationGetter orientationGetter) { _orientationGetter = orientationGetter; } + + QVector& getActiveLocalAudioInjectors() { return _activeLocalAudioInjectors; } static const float CALLBACK_ACCELERATOR_RATIO; @@ -210,6 +215,9 @@ protected: private: void outputFormatChanged(); + void mixLocalAudioInjectors(int16_t* inputBuffer); + float azimuthForSource(const glm::vec3& relativePosition); + float gainForSource(const glm::vec3& relativePosition, float volume); QByteArray firstInputFrame; QAudioInput* _audioInput; @@ -266,6 +274,11 @@ private: AudioSRC* _inputToNetworkResampler; AudioSRC* _networkToOutputResampler; + // for local hrtf-ing + float _hrtfBuffer[AudioConstants::NETWORK_FRAME_SAMPLES_STEREO]; + int16_t _scratchBuffer[AudioConstants::NETWORK_FRAME_SAMPLES_STEREO]; + AudioLimiter _audioLimiter; + // Adds Reverb void configureReverb(); void updateReverbOptions(); @@ -296,6 +309,8 @@ private: void checkDevices(); bool _hasReceivedFirstPacket = false; + + QVector _activeLocalAudioInjectors; CodecPluginPointer _codec; QString _selectedCodecName; diff --git a/libraries/audio/src/AudioInjector.cpp b/libraries/audio/src/AudioInjector.cpp index fee33dcb92..08143b8491 100644 --- a/libraries/audio/src/AudioInjector.cpp +++ b/libraries/audio/src/AudioInjector.cpp @@ -26,6 +26,8 @@ #include "AudioInjector.h" +int audioInjectorPtrMetaTypeId = qRegisterMetaType(); + AudioInjector::AudioInjector(QObject* parent) : QObject(parent) { @@ -41,11 +43,20 @@ AudioInjector::AudioInjector(const Sound& sound, const AudioInjectorOptions& inj AudioInjector::AudioInjector(const QByteArray& audioData, const AudioInjectorOptions& injectorOptions) : _audioData(audioData), - _options(injectorOptions) + _options(injectorOptions) { } +void AudioInjector::setOptions(const AudioInjectorOptions& options) { + // since options.stereo is computed from the audio stream, + // we need to copy it from existing options just in case. + bool currentlyStereo = _options.stereo; + _options = options; + _options.stereo = currentlyStereo; +} + + void AudioInjector::finish() { bool shouldDelete = (_state == State::NotFinishedWithPendingDelete); _state = State::Finished; @@ -115,11 +126,11 @@ void AudioInjector::restart() { _hasSetup = false; _shouldStop = false; _state = State::NotFinished; - + // call inject audio to start injection over again setupInjection(); - // if we're a local injector call inject locally to start injecting again + // if we're a local injector, just inject again if (_options.localOnly) { injectLocally(); } else { @@ -145,7 +156,8 @@ bool AudioInjector::injectLocally() { // give our current send position to the local buffer _localBuffer->setCurrentOffset(_currentSendOffset); - success = _localAudioInterface->outputLocalInjector(_options.stereo, this); + // call this function on the AudioClient's thread + success = QMetaObject::invokeMethod(_localAudioInterface, "outputLocalInjector", Q_ARG(bool, _options.stereo), Q_ARG(AudioInjector*, this)); if (!success) { qCDebug(audio) << "AudioInjector::injectLocally could not output locally via _localAudioInterface"; diff --git a/libraries/audio/src/AudioInjector.h b/libraries/audio/src/AudioInjector.h index c90256429d..1af733ace6 100644 --- a/libraries/audio/src/AudioInjector.h +++ b/libraries/audio/src/AudioInjector.h @@ -26,6 +26,7 @@ #include "AudioInjectorLocalBuffer.h" #include "AudioInjectorOptions.h" +#include "AudioHRTF.h" #include "Sound.h" class AbstractAudioInterface; @@ -54,8 +55,12 @@ public: void setCurrentSendOffset(int currentSendOffset) { _currentSendOffset = currentSendOffset; } AudioInjectorLocalBuffer* getLocalBuffer() const { return _localBuffer; } + AudioHRTF& getLocalHRTF() { return _localHRTF; } + bool isLocalOnly() const { return _options.localOnly; } - + float getVolume() const { return _options.volume; } + glm::vec3 getPosition() const { return _options.position; } + bool isStereo() const { return _options.stereo; } void setLocalAudioInterface(AbstractAudioInterface* localAudioInterface) { _localAudioInterface = localAudioInterface; } static AudioInjector* playSoundAndDelete(const QByteArray& buffer, const AudioInjectorOptions options, AbstractAudioInterface* localInterface); @@ -70,18 +75,16 @@ public slots: void stopAndDeleteLater(); const AudioInjectorOptions& getOptions() const { return _options; } - void setOptions(const AudioInjectorOptions& options) { _options = options; } + void setOptions(const AudioInjectorOptions& options); float getLoudness() const { return _loudness; } bool isPlaying() const { return _state == State::NotFinished || _state == State::NotFinishedWithPendingDelete; } + void finish(); signals: void finished(); void restarting(); -private slots: - void finish(); - private: void setupInjection(); int64_t injectNextFrame(); @@ -103,8 +106,9 @@ private: std::unique_ptr _frameTimer { nullptr }; quint16 _outgoingSequenceNumber { 0 }; + // when the injector is local, we need this + AudioHRTF _localHRTF; friend class AudioInjectorManager; }; - #endif // hifi_AudioInjector_h