// // AudioClient.h // libraries/audio-client/src // // Created by Stephen Birarda on 1/22/13. // Copyright 2013 High Fidelity, Inc. // // Distributed under the Apache License, Version 2.0. // See the accompanying file LICENSE or http://www.apache.org/licenses/LICENSE-2.0.html // #ifndef hifi_AudioClient_h #define hifi_AudioClient_h #include #include #include #include #include #include #include #include #include #include #include #include #include #include #include #include #include #include #include #include #include #include #include #include #include #include #include #include #include #include #include #include #include #include #include #include "AudioIOStats.h" #include "AudioFileWav.h" #include "HifiAudioDeviceInfo.h" #ifdef _WIN32 #pragma warning( push ) #pragma warning( disable : 4273 ) #pragma warning( disable : 4305 ) #endif #ifdef _WIN32 #pragma warning( pop ) #endif #if defined (Q_OS_ANDROID) #define VOICE_RECOGNITION "voicerecognition" #define VOICE_COMMUNICATION "voicecommunication" #define SETTING_AEC_KEY "Android/aec" #define DEFAULT_AEC_ENABLED true #endif class QAudioInput; class QAudioOutput; class QIODevice; class Transform; class NLPacket; #define DEFAULT_STARVE_DETECTION_ENABLED true #define DEFAULT_BUFFER_FRAMES 1 class AudioClient : public AbstractAudioInterface, public Dependency { Q_OBJECT SINGLETON_DEPENDENCY using LocalInjectorsStream = AudioMixRingBuffer; public: static const int MIN_BUFFER_FRAMES; static const int MAX_BUFFER_FRAMES; using AudioPositionGetter = std::function; using AudioOrientationGetter = std::function; using Mutex = std::mutex; using Lock = std::unique_lock; class AudioOutputIODevice : public QIODevice { public: AudioOutputIODevice(LocalInjectorsStream& localInjectorsStream, MixedProcessedAudioStream& receivedAudioStream, AudioClient* audio) : _localInjectorsStream(localInjectorsStream), _receivedAudioStream(receivedAudioStream), _audio(audio), _unfulfilledReads(0) {} void start() { open(QIODevice::ReadOnly | QIODevice::Unbuffered); } qint64 readData(char* data, qint64 maxSize) override; qint64 writeData(const char* data, qint64 maxSize) override { return 0; } int getRecentUnfulfilledReads() { int unfulfilledReads = _unfulfilledReads; _unfulfilledReads = 0; return unfulfilledReads; } private: LocalInjectorsStream& _localInjectorsStream; MixedProcessedAudioStream& _receivedAudioStream; AudioClient* _audio; int _unfulfilledReads; }; void startThread(); void negotiateAudioFormat(); void selectAudioFormat(const QString& selectedCodecName); Q_INVOKABLE QString getSelectedAudioFormat() const { return _selectedCodecName; } Q_INVOKABLE bool getNoiseGateOpen() const { return _audioGateOpen; } Q_INVOKABLE float getSilentInboundPPS() const { return _silentInbound.rate(); } Q_INVOKABLE float getAudioInboundPPS() const { return _audioInbound.rate(); } Q_INVOKABLE float getSilentOutboundPPS() const { return _silentOutbound.rate(); } Q_INVOKABLE float getAudioOutboundPPS() const { return _audioOutbound.rate(); } const MixedProcessedAudioStream& getReceivedAudioStream() const { return _receivedAudioStream; } MixedProcessedAudioStream& getReceivedAudioStream() { return _receivedAudioStream; } const QAudioFormat& getOutputFormat() const { return _outputFormat; } float getLastInputLoudness() const { return _lastInputLoudness; } float getTimeSinceLastClip() const { return _timeSinceLastClip; } float getAudioAverageInputLoudness() const { return _lastInputLoudness; } const AudioIOStats& getStats() const { return _stats; } int getOutputBufferSize() { return _outputBufferSizeFrames.get(); } bool getOutputStarveDetectionEnabled() { return _outputStarveDetectionEnabled.get(); } void setOutputStarveDetectionEnabled(bool enabled) { _outputStarveDetectionEnabled.set(enabled); } bool isSimulatingJitter() { return _gate.isSimulatingJitter(); } void setIsSimulatingJitter(bool enable) { _gate.setIsSimulatingJitter(enable); } int getGateThreshold() { return _gate.getThreshold(); } void setGateThreshold(int threshold) { _gate.setThreshold(threshold); } void setPositionGetter(AudioPositionGetter positionGetter) { _positionGetter = positionGetter; } void setOrientationGetter(AudioOrientationGetter orientationGetter) { _orientationGetter = orientationGetter; } void setIsPlayingBackRecording(bool isPlayingBackRecording) { _isPlayingBackRecording = isPlayingBackRecording; } Q_INVOKABLE void setAvatarBoundingBoxParameters(glm::vec3 corner, glm::vec3 scale); bool outputLocalInjector(const AudioInjectorPointer& injector) override; HifiAudioDeviceInfo getActiveAudioDevice(QAudio::Mode mode) const; QList getAudioDevices(QAudio::Mode mode) const; void enablePeakValues(bool enable) { _enablePeakValues = enable; } bool peakValuesAvailable() const; static const float CALLBACK_ACCELERATOR_RATIO; bool getNamedAudioDeviceForModeExists(QAudio::Mode mode, const QString& deviceName); void setRecording(bool isRecording) { _isRecording = isRecording; }; bool getRecording() { return _isRecording; }; bool startRecording(const QString& filename); void stopRecording(); void setAudioPaused(bool pause); AudioSolo& getAudioSolo() override { return _solo; } #ifdef Q_OS_WIN static QString getWinDeviceName(wchar_t* guid); #endif #if defined(Q_OS_ANDROID) bool isHeadsetPluggedIn() { return _isHeadsetPluggedIn; } #endif int getNumLocalInjectors(); public slots: void start(); void stop(); void handleAudioEnvironmentDataPacket(QSharedPointer message); void handleAudioDataPacket(QSharedPointer message); void handleNoisyMutePacket(QSharedPointer message); void handleMuteEnvironmentPacket(QSharedPointer message); void handleSelectedAudioFormat(QSharedPointer message); void handleMismatchAudioFormat(SharedNodePointer node, const QString& currentCodec, const QString& recievedCodec); void sendDownstreamAudioStatsPacket() { _stats.publish(); } void handleMicAudioInput(); void audioInputStateChanged(QAudio::State state); void checkInputTimeout(); void handleDummyAudioInput(); void handleRecordedAudioInput(const QByteArray& audio); void reset(); void audioMixerKilled(); void setMuted(bool muted, bool emitSignal = true); bool isMuted() { return _muted; } virtual bool setIsStereoInput(bool stereo) override; virtual bool isStereoInput() override { return _isStereoInput; } void setNoiseReduction(bool isNoiseGateEnabled, bool emitSignal = true); bool isNoiseReductionEnabled() const { return _isNoiseGateEnabled; } void setWarnWhenMuted(bool isNoiseGateEnabled, bool emitSignal = true); bool isWarnWhenMutedEnabled() const { return _warnWhenMuted; } void setAcousticEchoCancellation(bool isAECEnabled, bool emitSignal = true); bool isAcousticEchoCancellationEnabled() const { return _isAECEnabled; } virtual bool getLocalEcho() override { return _shouldEchoLocally; } virtual void setLocalEcho(bool localEcho) override { _shouldEchoLocally = localEcho; } virtual void toggleLocalEcho() override { _shouldEchoLocally = !_shouldEchoLocally; } virtual bool getServerEcho() override { return _shouldEchoToServer; } virtual void setServerEcho(bool serverEcho) override { _shouldEchoToServer = serverEcho; } virtual void toggleServerEcho() override { _shouldEchoToServer = !_shouldEchoToServer; } void processReceivedSamples(const QByteArray& inputBuffer, QByteArray& outputBuffer); void sendMuteEnvironmentPacket(); int setOutputBufferSize(int numFrames, bool persist = true); bool shouldLoopbackInjectors() override { return _shouldEchoToServer; } // calling with a null QAudioDevice will use the system default bool switchAudioDevice(QAudio::Mode mode, const HifiAudioDeviceInfo& deviceInfo = HifiAudioDeviceInfo()); bool switchAudioDevice(QAudio::Mode mode, const QString& deviceName, bool isHmd); void setHmdAudioName(QAudio::Mode mode, const QString& name); // Qt opensles plugin is not able to detect when the headset is plugged in void setHeadsetPluggedIn(bool pluggedIn); float getInputVolume() const { return (_audioInput) ? (float)_audioInput->volume() : 0.0f; } void setInputVolume(float volume, bool emitSignal = true); void setReverb(bool reverb); void setReverbOptions(const AudioEffectOptions* options); void setLocalInjectorGain(float gain) { _localInjectorGain = gain; }; void setSystemInjectorGain(float gain) { _systemInjectorGain = gain; }; void setOutputGain(float gain) { _outputGain = gain; }; void outputNotify(); void noteAwakening(); void loadSettings(); void saveSettings(); signals: void inputVolumeChanged(float volume); void muteToggled(bool muted); void noiseReductionChanged(bool noiseReductionEnabled); void warnWhenMutedChanged(bool warnWhenMutedEnabled); void acousticEchoCancellationChanged(bool acousticEchoCancellationEnabled); void mutedByMixer(); void inputReceived(const QByteArray& inputSamples); void inputLoudnessChanged(float loudness, bool isClipping); void outputBytesToNetwork(int numBytes); void inputBytesFromNetwork(int numBytes); void noiseGateOpened(); void noiseGateClosed(); void changeDevice(const HifiAudioDeviceInfo& outputDeviceInfo); void deviceChanged(QAudio::Mode mode, const HifiAudioDeviceInfo& device); void devicesChanged(QAudio::Mode mode, const QList& devices); void peakValueListChanged(const QList peakValueList); void receivedFirstPacket(); void disconnected(); void audioFinished(); void muteEnvironmentRequested(glm::vec3 position, float radius); void outputBufferReceived(const QByteArray _outputBuffer); protected: AudioClient(); ~AudioClient(); virtual void customDeleter() override; private: static const int RECEIVED_AUDIO_STREAM_CAPACITY_FRAMES{ 100 }; // OUTPUT_CHANNEL_COUNT is audio pipeline output format, which is always 2 channel. // _outputFormat.channelCount() is device output format, which may be 1 or multichannel. static const int OUTPUT_CHANNEL_COUNT{ 2 }; static const int STARVE_DETECTION_THRESHOLD{ 3 }; static const int STARVE_DETECTION_PERIOD{ 10 * 1000 }; // 10 Seconds static const AudioPositionGetter DEFAULT_POSITION_GETTER; static const AudioOrientationGetter DEFAULT_ORIENTATION_GETTER; friend class CheckDevicesThread; friend class LocalInjectorsThread; // background tasks void checkDevices(); void checkPeakValues(); void outputFormatChanged(); void handleAudioInput(QByteArray& audioBuffer); void prepareLocalAudioInjectors(std::unique_ptr localAudioLock = nullptr); bool mixLocalAudioInjectors(float* mixBuffer); float azimuthForSource(const glm::vec3& relativePosition); float gainForSource(float distance, float volume); #ifdef Q_OS_ANDROID QTimer _checkInputTimer{ this }; long _inputReadsSinceLastCheck = 0l; bool _isHeadsetPluggedIn { false }; #endif class Gate { public: Gate(AudioClient* audioClient); bool isSimulatingJitter() { return _isSimulatingJitter; } void setIsSimulatingJitter(bool enable); int getThreshold() { return _threshold; } void setThreshold(int threshold); void insert(QSharedPointer message); private: void flush(); AudioClient* _audioClient; std::queue> _queue; std::mutex _mutex; int _index{ 0 }; int _threshold{ 1 }; bool _isSimulatingJitter{ false }; }; Gate _gate{ this }; Mutex _injectorsMutex; QAudioInput* _audioInput{ nullptr }; QTimer* _dummyAudioInput{ nullptr }; QAudioFormat _desiredInputFormat; QAudioFormat _inputFormat; QIODevice* _inputDevice{ nullptr }; int _numInputCallbackBytes{ 0 }; QAudioOutput* _audioOutput{ nullptr }; std::atomic _audioOutputInitialized { false }; QAudioFormat _desiredOutputFormat; QAudioFormat _outputFormat; int _outputFrameSize{ 0 }; int _numOutputCallbackBytes{ 0 }; QAudioOutput* _loopbackAudioOutput{ nullptr }; QIODevice* _loopbackOutputDevice{ nullptr }; AudioRingBuffer _inputRingBuffer{ 0 }; LocalInjectorsStream _localInjectorsStream{ 0 , 1 }; // In order to use _localInjectorsStream as a lock-free pipe, // use it with a single producer/consumer, and track available samples and injectors std::atomic _localSamplesAvailable { 0 }; std::atomic _localInjectorsAvailable { false }; MixedProcessedAudioStream _receivedAudioStream{ RECEIVED_AUDIO_STREAM_CAPACITY_FRAMES }; bool _isStereoInput{ false }; std::atomic _enablePeakValues { false }; quint64 _outputStarveDetectionStartTimeMsec{ 0 }; int _outputStarveDetectionCount { 0 }; Setting::Handle _outputBufferSizeFrames{"audioOutputBufferFrames", DEFAULT_BUFFER_FRAMES}; int _sessionOutputBufferSizeFrames{ _outputBufferSizeFrames.get() }; Setting::Handle _outputStarveDetectionEnabled{ "audioOutputStarveDetectionEnabled", DEFAULT_STARVE_DETECTION_ENABLED}; StDev _stdev; QElapsedTimer _timeSinceLastReceived; float _lastRawInputLoudness{ 0.0f }; // before mute/gate float _lastSmoothedRawInputLoudness{ 0.0f }; float _lastInputLoudness{ 0.0f }; // after mute/gate float _timeSinceLastClip{ -1.0f }; int _totalInputAudioSamples; bool _muted{ false }; bool _shouldEchoLocally{ false }; bool _shouldEchoToServer{ false }; bool _isNoiseGateEnabled{ true }; bool _warnWhenMuted; bool _isAECEnabled{ true }; bool _reverb{ false }; AudioEffectOptions _scriptReverbOptions; AudioEffectOptions _zoneReverbOptions; AudioEffectOptions* _reverbOptions{ &_scriptReverbOptions }; AudioReverb _sourceReverb { AudioConstants::SAMPLE_RATE }; AudioReverb _listenerReverb { AudioConstants::SAMPLE_RATE }; AudioReverb _localReverb { AudioConstants::SAMPLE_RATE }; // possible streams needed for resample AudioSRC* _inputToNetworkResampler{ nullptr }; AudioSRC* _networkToOutputResampler{ nullptr }; AudioSRC* _localToOutputResampler{ nullptr }; AudioSRC* _loopbackResampler{ nullptr }; // for network audio (used by network audio thread) int16_t _networkScratchBuffer[AudioConstants::NETWORK_FRAME_SAMPLES_AMBISONIC]; // for output audio (used by this thread) int _outputPeriod { 0 }; float* _outputMixBuffer { NULL }; int16_t* _outputScratchBuffer { NULL }; std::atomic _outputGain { 1.0f }; float _lastOutputGain { 1.0f }; // for local audio (used by audio injectors thread) std::atomic _localInjectorGain { 1.0f }; std::atomic _systemInjectorGain { 1.0f }; float _localMixBuffer[AudioConstants::NETWORK_FRAME_SAMPLES_STEREO]; int16_t _localScratchBuffer[AudioConstants::NETWORK_FRAME_SAMPLES_AMBISONIC]; float* _localOutputMixBuffer { NULL }; Mutex _localAudioMutex; AudioLimiter _audioLimiter{ AudioConstants::SAMPLE_RATE, OUTPUT_CHANNEL_COUNT }; // Adds Reverb void configureReverb(); void updateReverbOptions(); void handleLocalEchoAndReverb(QByteArray& inputByteArray); #if defined(WEBRTC_ENABLED) static const int WEBRTC_SAMPLE_RATE_MAX = 96000; static const int WEBRTC_CHANNELS_MAX = 2; static const int WEBRTC_FRAMES_MAX = webrtc::AudioProcessing::kChunkSizeMs * WEBRTC_SAMPLE_RATE_MAX / 1000; webrtc::AudioProcessing* _apm { nullptr }; int16_t _fifoFarEnd[WEBRTC_CHANNELS_MAX * WEBRTC_FRAMES_MAX] {}; int _numFifoFarEnd = 0; // numFrames saved in fifo void configureWebrtc(); void processWebrtcFarEnd(const int16_t* samples, int numFrames, int numChannels, int sampleRate); void processWebrtcNearEnd(int16_t* samples, int numFrames, int numChannels, int sampleRate); #endif bool switchInputToAudioDevice(const HifiAudioDeviceInfo inputDeviceInfo, bool isShutdownRequest = false); bool switchOutputToAudioDevice(const HifiAudioDeviceInfo outputDeviceInfo, bool isShutdownRequest = false); // Callback acceleration dependent calculations int calculateNumberOfInputCallbackBytes(const QAudioFormat& format) const; int calculateNumberOfFrameSamples(int numBytes) const; quint16 _outgoingAvatarAudioSequenceNumber{ 0 }; AudioOutputIODevice _audioOutputIODevice{ _localInjectorsStream, _receivedAudioStream, this }; AudioIOStats _stats{ &_receivedAudioStream }; AudioGate* _audioGate { nullptr }; bool _audioGateOpen { true }; AudioPositionGetter _positionGetter{ DEFAULT_POSITION_GETTER }; AudioOrientationGetter _orientationGetter{ DEFAULT_ORIENTATION_GETTER }; glm::vec3 avatarBoundingBoxCorner; glm::vec3 avatarBoundingBoxScale; HifiAudioDeviceInfo _inputDeviceInfo; HifiAudioDeviceInfo _outputDeviceInfo; QList _inputDevices; QList _outputDevices; QString _hmdInputName { QString() }; QString _hmdOutputName{ QString() }; AudioFileWav _audioFileWav; bool _hasReceivedFirstPacket { false }; QVector _activeLocalAudioInjectors; bool _isPlayingBackRecording { false }; bool _audioPaused { false }; CodecPluginPointer _codec; QString _selectedCodecName; Encoder* _encoder { nullptr }; // for outbound mic stream RateCounter<> _silentOutbound; RateCounter<> _audioOutbound; RateCounter<> _silentInbound; RateCounter<> _audioInbound; #if defined(Q_OS_ANDROID) bool _shouldRestartInputSetup { true }; // Should we restart the input device because of an unintended stop? #endif AudioSolo _solo; QFuture _localPrepInjectorFuture; QReadWriteLock _hmdNameLock; Mutex _checkDevicesMutex; QTimer* _checkDevicesTimer { nullptr }; Mutex _checkPeakValuesMutex; QTimer* _checkPeakValuesTimer { nullptr }; bool _isRecording { false }; }; #endif // hifi_AudioClient_h