overte-HifiExperiments/libraries/audio-client/src/AudioClient.h
2019-11-26 11:41:41 -08:00

522 lines
No EOL
19 KiB
C++

//
// AudioClient.h
// libraries/audio-client/src
//
// Created by Stephen Birarda on 1/22/13.
// Copyright 2013 High Fidelity, Inc.
//
// Distributed under the Apache License, Version 2.0.
// See the accompanying file LICENSE or http://www.apache.org/licenses/LICENSE-2.0.html
//
#ifndef hifi_AudioClient_h
#define hifi_AudioClient_h
#include <fstream>
#include <memory>
#include <vector>
#include <mutex>
#include <queue>
#include <QFuture>
#include <QtCore/QtGlobal>
#include <QtCore/QByteArray>
#include <QtCore/QElapsedTimer>
#include <QtCore/QObject>
#include <QtCore/QVector>
#include <QtMultimedia/QAudio>
#include <QtMultimedia/QAudioFormat>
#include <QtMultimedia/QAudioInput>
#include <AbstractAudioInterface.h>
#include <AudioEffectOptions.h>
#include <AudioStreamStats.h>
#include <shared/WebRTC.h>
#include <DependencyManager.h>
#include <HifiSockAddr.h>
#include <NLPacket.h>
#include <MixedProcessedAudioStream.h>
#include <RingBufferHistory.h>
#include <SettingHandle.h>
#include <Sound.h>
#include <StDev.h>
#include <AudioHRTF.h>
#include <AudioSRC.h>
#include <AudioInjector.h>
#include <AudioReverb.h>
#include <AudioLimiter.h>
#include <AudioConstants.h>
#include <AudioGate.h>
#include <shared/RateCounter.h>
#include <plugins/CodecPlugin.h>
#include "AudioIOStats.h"
#include "AudioFileWav.h"
#include "HifiAudioDeviceInfo.h"
#ifdef _WIN32
#pragma warning( push )
#pragma warning( disable : 4273 )
#pragma warning( disable : 4305 )
#endif
#ifdef _WIN32
#pragma warning( pop )
#endif
#if defined (Q_OS_ANDROID)
#define VOICE_RECOGNITION "voicerecognition"
#define VOICE_COMMUNICATION "voicecommunication"
#define SETTING_AEC_KEY "Android/aec"
#define DEFAULT_AEC_ENABLED true
#endif
class QAudioInput;
class QAudioOutput;
class QIODevice;
class Transform;
class NLPacket;
#define DEFAULT_STARVE_DETECTION_ENABLED true
#define DEFAULT_BUFFER_FRAMES 1
class AudioClient : public AbstractAudioInterface, public Dependency {
Q_OBJECT
SINGLETON_DEPENDENCY
using LocalInjectorsStream = AudioMixRingBuffer;
public:
static const int MIN_BUFFER_FRAMES;
static const int MAX_BUFFER_FRAMES;
using AudioPositionGetter = std::function<glm::vec3()>;
using AudioOrientationGetter = std::function<glm::quat()>;
using Mutex = std::mutex;
using Lock = std::unique_lock<Mutex>;
class AudioOutputIODevice : public QIODevice {
public:
AudioOutputIODevice(LocalInjectorsStream& localInjectorsStream, MixedProcessedAudioStream& receivedAudioStream,
AudioClient* audio) :
_localInjectorsStream(localInjectorsStream), _receivedAudioStream(receivedAudioStream),
_audio(audio), _unfulfilledReads(0) {}
void start() { open(QIODevice::ReadOnly | QIODevice::Unbuffered); }
qint64 readData(char* data, qint64 maxSize) override;
qint64 writeData(const char* data, qint64 maxSize) override { return 0; }
int getRecentUnfulfilledReads() { int unfulfilledReads = _unfulfilledReads; _unfulfilledReads = 0; return unfulfilledReads; }
private:
LocalInjectorsStream& _localInjectorsStream;
MixedProcessedAudioStream& _receivedAudioStream;
AudioClient* _audio;
int _unfulfilledReads;
};
void startThread();
void negotiateAudioFormat();
void selectAudioFormat(const QString& selectedCodecName);
Q_INVOKABLE QString getSelectedAudioFormat() const { return _selectedCodecName; }
Q_INVOKABLE bool getNoiseGateOpen() const { return _audioGateOpen; }
Q_INVOKABLE float getSilentInboundPPS() const { return _silentInbound.rate(); }
Q_INVOKABLE float getAudioInboundPPS() const { return _audioInbound.rate(); }
Q_INVOKABLE float getSilentOutboundPPS() const { return _silentOutbound.rate(); }
Q_INVOKABLE float getAudioOutboundPPS() const { return _audioOutbound.rate(); }
const MixedProcessedAudioStream& getReceivedAudioStream() const { return _receivedAudioStream; }
MixedProcessedAudioStream& getReceivedAudioStream() { return _receivedAudioStream; }
const QAudioFormat& getOutputFormat() const { return _outputFormat; }
float getLastInputLoudness() const { return _lastInputLoudness; }
float getTimeSinceLastClip() const { return _timeSinceLastClip; }
float getAudioAverageInputLoudness() const { return _lastInputLoudness; }
const AudioIOStats& getStats() const { return _stats; }
int getOutputBufferSize() { return _outputBufferSizeFrames.get(); }
bool getOutputStarveDetectionEnabled() { return _outputStarveDetectionEnabled.get(); }
void setOutputStarveDetectionEnabled(bool enabled) { _outputStarveDetectionEnabled.set(enabled); }
bool isSimulatingJitter() { return _gate.isSimulatingJitter(); }
void setIsSimulatingJitter(bool enable) { _gate.setIsSimulatingJitter(enable); }
int getGateThreshold() { return _gate.getThreshold(); }
void setGateThreshold(int threshold) { _gate.setThreshold(threshold); }
void setPositionGetter(AudioPositionGetter positionGetter) { _positionGetter = positionGetter; }
void setOrientationGetter(AudioOrientationGetter orientationGetter) { _orientationGetter = orientationGetter; }
void setIsPlayingBackRecording(bool isPlayingBackRecording) { _isPlayingBackRecording = isPlayingBackRecording; }
Q_INVOKABLE void setAvatarBoundingBoxParameters(glm::vec3 corner, glm::vec3 scale);
bool outputLocalInjector(const AudioInjectorPointer& injector) override;
HifiAudioDeviceInfo getActiveAudioDevice(QAudio::Mode mode) const;
QList<HifiAudioDeviceInfo> getAudioDevices(QAudio::Mode mode) const;
void enablePeakValues(bool enable) { _enablePeakValues = enable; }
bool peakValuesAvailable() const;
static const float CALLBACK_ACCELERATOR_RATIO;
bool getNamedAudioDeviceForModeExists(QAudio::Mode mode, const QString& deviceName);
void setRecording(bool isRecording) { _isRecording = isRecording; };
bool getRecording() { return _isRecording; };
bool startRecording(const QString& filename);
void stopRecording();
void setAudioPaused(bool pause);
AudioSolo& getAudioSolo() override { return _solo; }
#ifdef Q_OS_WIN
static QString getWinDeviceName(wchar_t* guid);
#endif
#if defined(Q_OS_ANDROID)
bool isHeadsetPluggedIn() { return _isHeadsetPluggedIn; }
#endif
int getNumLocalInjectors();
public slots:
void start();
void stop();
void handleAudioEnvironmentDataPacket(QSharedPointer<ReceivedMessage> message);
void handleAudioDataPacket(QSharedPointer<ReceivedMessage> message);
void handleNoisyMutePacket(QSharedPointer<ReceivedMessage> message);
void handleMuteEnvironmentPacket(QSharedPointer<ReceivedMessage> message);
void handleSelectedAudioFormat(QSharedPointer<ReceivedMessage> message);
void handleMismatchAudioFormat(SharedNodePointer node, const QString& currentCodec, const QString& recievedCodec);
void sendDownstreamAudioStatsPacket() { _stats.publish(); }
void handleMicAudioInput();
void audioInputStateChanged(QAudio::State state);
void checkInputTimeout();
void handleDummyAudioInput();
void handleRecordedAudioInput(const QByteArray& audio);
void reset();
void audioMixerKilled();
void setMuted(bool muted, bool emitSignal = true);
bool isMuted() { return _muted; }
virtual bool setIsStereoInput(bool stereo) override;
virtual bool isStereoInput() override { return _isStereoInput; }
void setNoiseReduction(bool isNoiseGateEnabled, bool emitSignal = true);
bool isNoiseReductionEnabled() const { return _isNoiseGateEnabled; }
void setWarnWhenMuted(bool isNoiseGateEnabled, bool emitSignal = true);
bool isWarnWhenMutedEnabled() const { return _warnWhenMuted; }
void setAcousticEchoCancellation(bool isAECEnabled, bool emitSignal = true);
bool isAcousticEchoCancellationEnabled() const { return _isAECEnabled; }
virtual bool getLocalEcho() override { return _shouldEchoLocally; }
virtual void setLocalEcho(bool localEcho) override { _shouldEchoLocally = localEcho; }
virtual void toggleLocalEcho() override { _shouldEchoLocally = !_shouldEchoLocally; }
virtual bool getServerEcho() override { return _shouldEchoToServer; }
virtual void setServerEcho(bool serverEcho) override { _shouldEchoToServer = serverEcho; }
virtual void toggleServerEcho() override { _shouldEchoToServer = !_shouldEchoToServer; }
void processReceivedSamples(const QByteArray& inputBuffer, QByteArray& outputBuffer);
void sendMuteEnvironmentPacket();
int setOutputBufferSize(int numFrames, bool persist = true);
bool shouldLoopbackInjectors() override { return _shouldEchoToServer; }
// calling with a null QAudioDevice will use the system default
bool switchAudioDevice(QAudio::Mode mode, const HifiAudioDeviceInfo& deviceInfo = HifiAudioDeviceInfo());
bool switchAudioDevice(QAudio::Mode mode, const QString& deviceName, bool isHmd);
void setHmdAudioName(QAudio::Mode mode, const QString& name);
// Qt opensles plugin is not able to detect when the headset is plugged in
void setHeadsetPluggedIn(bool pluggedIn);
float getInputVolume() const { return (_audioInput) ? (float)_audioInput->volume() : 0.0f; }
void setInputVolume(float volume, bool emitSignal = true);
void setReverb(bool reverb);
void setReverbOptions(const AudioEffectOptions* options);
void setLocalInjectorGain(float gain) { _localInjectorGain = gain; };
void setSystemInjectorGain(float gain) { _systemInjectorGain = gain; };
void setOutputGain(float gain) { _outputGain = gain; };
void outputNotify();
void noteAwakening();
void loadSettings();
void saveSettings();
signals:
void inputVolumeChanged(float volume);
void muteToggled(bool muted);
void noiseReductionChanged(bool noiseReductionEnabled);
void warnWhenMutedChanged(bool warnWhenMutedEnabled);
void acousticEchoCancellationChanged(bool acousticEchoCancellationEnabled);
void mutedByMixer();
void inputReceived(const QByteArray& inputSamples);
void inputLoudnessChanged(float loudness, bool isClipping);
void outputBytesToNetwork(int numBytes);
void inputBytesFromNetwork(int numBytes);
void noiseGateOpened();
void noiseGateClosed();
void changeDevice(const HifiAudioDeviceInfo& outputDeviceInfo);
void deviceChanged(QAudio::Mode mode, const HifiAudioDeviceInfo& device);
void devicesChanged(QAudio::Mode mode, const QList<HifiAudioDeviceInfo>& devices);
void peakValueListChanged(const QList<float> peakValueList);
void receivedFirstPacket();
void disconnected();
void audioFinished();
void muteEnvironmentRequested(glm::vec3 position, float radius);
void outputBufferReceived(const QByteArray _outputBuffer);
protected:
AudioClient();
~AudioClient();
virtual void customDeleter() override;
private:
static const int RECEIVED_AUDIO_STREAM_CAPACITY_FRAMES{ 100 };
// OUTPUT_CHANNEL_COUNT is audio pipeline output format, which is always 2 channel.
// _outputFormat.channelCount() is device output format, which may be 1 or multichannel.
static const int OUTPUT_CHANNEL_COUNT{ 2 };
static const int STARVE_DETECTION_THRESHOLD{ 3 };
static const int STARVE_DETECTION_PERIOD{ 10 * 1000 }; // 10 Seconds
static const AudioPositionGetter DEFAULT_POSITION_GETTER;
static const AudioOrientationGetter DEFAULT_ORIENTATION_GETTER;
friend class CheckDevicesThread;
friend class LocalInjectorsThread;
// background tasks
void checkDevices();
void checkPeakValues();
void outputFormatChanged();
void handleAudioInput(QByteArray& audioBuffer);
void prepareLocalAudioInjectors(std::unique_ptr<Lock> localAudioLock = nullptr);
bool mixLocalAudioInjectors(float* mixBuffer);
float azimuthForSource(const glm::vec3& relativePosition);
float gainForSource(float distance, float volume);
#ifdef Q_OS_ANDROID
QTimer _checkInputTimer{ this };
long _inputReadsSinceLastCheck = 0l;
bool _isHeadsetPluggedIn { false };
#endif
class Gate {
public:
Gate(AudioClient* audioClient);
bool isSimulatingJitter() { return _isSimulatingJitter; }
void setIsSimulatingJitter(bool enable);
int getThreshold() { return _threshold; }
void setThreshold(int threshold);
void insert(QSharedPointer<ReceivedMessage> message);
private:
void flush();
AudioClient* _audioClient;
std::queue<QSharedPointer<ReceivedMessage>> _queue;
std::mutex _mutex;
int _index{ 0 };
int _threshold{ 1 };
bool _isSimulatingJitter{ false };
};
Gate _gate{ this };
Mutex _injectorsMutex;
QAudioInput* _audioInput{ nullptr };
QTimer* _dummyAudioInput{ nullptr };
QAudioFormat _desiredInputFormat;
QAudioFormat _inputFormat;
QIODevice* _inputDevice{ nullptr };
int _numInputCallbackBytes{ 0 };
QAudioOutput* _audioOutput{ nullptr };
std::atomic<bool> _audioOutputInitialized { false };
QAudioFormat _desiredOutputFormat;
QAudioFormat _outputFormat;
int _outputFrameSize{ 0 };
int _numOutputCallbackBytes{ 0 };
QAudioOutput* _loopbackAudioOutput{ nullptr };
QIODevice* _loopbackOutputDevice{ nullptr };
AudioRingBuffer _inputRingBuffer{ 0 };
LocalInjectorsStream _localInjectorsStream{ 0 , 1 };
// In order to use _localInjectorsStream as a lock-free pipe,
// use it with a single producer/consumer, and track available samples and injectors
std::atomic<int> _localSamplesAvailable { 0 };
std::atomic<bool> _localInjectorsAvailable { false };
MixedProcessedAudioStream _receivedAudioStream{ RECEIVED_AUDIO_STREAM_CAPACITY_FRAMES };
bool _isStereoInput{ false };
std::atomic<bool> _enablePeakValues { false };
quint64 _outputStarveDetectionStartTimeMsec{ 0 };
int _outputStarveDetectionCount { 0 };
Setting::Handle<int> _outputBufferSizeFrames{"audioOutputBufferFrames", DEFAULT_BUFFER_FRAMES};
int _sessionOutputBufferSizeFrames{ _outputBufferSizeFrames.get() };
Setting::Handle<bool> _outputStarveDetectionEnabled{ "audioOutputStarveDetectionEnabled", DEFAULT_STARVE_DETECTION_ENABLED};
StDev _stdev;
QElapsedTimer _timeSinceLastReceived;
float _lastRawInputLoudness{ 0.0f }; // before mute/gate
float _lastSmoothedRawInputLoudness{ 0.0f };
float _lastInputLoudness{ 0.0f }; // after mute/gate
float _timeSinceLastClip{ -1.0f };
int _totalInputAudioSamples;
bool _muted{ false };
bool _shouldEchoLocally{ false };
bool _shouldEchoToServer{ false };
bool _isNoiseGateEnabled{ true };
bool _warnWhenMuted;
bool _isAECEnabled{ true };
bool _reverb{ false };
AudioEffectOptions _scriptReverbOptions;
AudioEffectOptions _zoneReverbOptions;
AudioEffectOptions* _reverbOptions{ &_scriptReverbOptions };
AudioReverb _sourceReverb { AudioConstants::SAMPLE_RATE };
AudioReverb _listenerReverb { AudioConstants::SAMPLE_RATE };
AudioReverb _localReverb { AudioConstants::SAMPLE_RATE };
// possible streams needed for resample
AudioSRC* _inputToNetworkResampler{ nullptr };
AudioSRC* _networkToOutputResampler{ nullptr };
AudioSRC* _localToOutputResampler{ nullptr };
AudioSRC* _loopbackResampler{ nullptr };
// for network audio (used by network audio thread)
int16_t _networkScratchBuffer[AudioConstants::NETWORK_FRAME_SAMPLES_AMBISONIC];
// for output audio (used by this thread)
int _outputPeriod { 0 };
float* _outputMixBuffer { NULL };
int16_t* _outputScratchBuffer { NULL };
std::atomic<float> _outputGain { 1.0f };
float _lastOutputGain { 1.0f };
// for local audio (used by audio injectors thread)
std::atomic<float> _localInjectorGain { 1.0f };
std::atomic<float> _systemInjectorGain { 1.0f };
float _localMixBuffer[AudioConstants::NETWORK_FRAME_SAMPLES_STEREO];
int16_t _localScratchBuffer[AudioConstants::NETWORK_FRAME_SAMPLES_AMBISONIC];
float* _localOutputMixBuffer { NULL };
Mutex _localAudioMutex;
AudioLimiter _audioLimiter{ AudioConstants::SAMPLE_RATE, OUTPUT_CHANNEL_COUNT };
// Adds Reverb
void configureReverb();
void updateReverbOptions();
void handleLocalEchoAndReverb(QByteArray& inputByteArray);
#if defined(WEBRTC_ENABLED)
static const int WEBRTC_SAMPLE_RATE_MAX = 96000;
static const int WEBRTC_CHANNELS_MAX = 2;
static const int WEBRTC_FRAMES_MAX = webrtc::AudioProcessing::kChunkSizeMs * WEBRTC_SAMPLE_RATE_MAX / 1000;
webrtc::AudioProcessing* _apm { nullptr };
int16_t _fifoFarEnd[WEBRTC_CHANNELS_MAX * WEBRTC_FRAMES_MAX] {};
int _numFifoFarEnd = 0; // numFrames saved in fifo
void configureWebrtc();
void processWebrtcFarEnd(const int16_t* samples, int numFrames, int numChannels, int sampleRate);
void processWebrtcNearEnd(int16_t* samples, int numFrames, int numChannels, int sampleRate);
#endif
bool switchInputToAudioDevice(const HifiAudioDeviceInfo inputDeviceInfo, bool isShutdownRequest = false);
bool switchOutputToAudioDevice(const HifiAudioDeviceInfo outputDeviceInfo, bool isShutdownRequest = false);
// Callback acceleration dependent calculations
int calculateNumberOfInputCallbackBytes(const QAudioFormat& format) const;
int calculateNumberOfFrameSamples(int numBytes) const;
quint16 _outgoingAvatarAudioSequenceNumber{ 0 };
AudioOutputIODevice _audioOutputIODevice{ _localInjectorsStream, _receivedAudioStream, this };
AudioIOStats _stats{ &_receivedAudioStream };
AudioGate* _audioGate { nullptr };
bool _audioGateOpen { true };
AudioPositionGetter _positionGetter{ DEFAULT_POSITION_GETTER };
AudioOrientationGetter _orientationGetter{ DEFAULT_ORIENTATION_GETTER };
glm::vec3 avatarBoundingBoxCorner;
glm::vec3 avatarBoundingBoxScale;
HifiAudioDeviceInfo _inputDeviceInfo;
HifiAudioDeviceInfo _outputDeviceInfo;
QList<HifiAudioDeviceInfo> _inputDevices;
QList<HifiAudioDeviceInfo> _outputDevices;
QString _hmdInputName { QString() };
QString _hmdOutputName{ QString() };
AudioFileWav _audioFileWav;
bool _hasReceivedFirstPacket { false };
QVector<AudioInjectorPointer> _activeLocalAudioInjectors;
bool _isPlayingBackRecording { false };
bool _audioPaused { false };
CodecPluginPointer _codec;
QString _selectedCodecName;
Encoder* _encoder { nullptr }; // for outbound mic stream
RateCounter<> _silentOutbound;
RateCounter<> _audioOutbound;
RateCounter<> _silentInbound;
RateCounter<> _audioInbound;
#if defined(Q_OS_ANDROID)
bool _shouldRestartInputSetup { true }; // Should we restart the input device because of an unintended stop?
#endif
AudioSolo _solo;
QFuture<void> _localPrepInjectorFuture;
QReadWriteLock _hmdNameLock;
Mutex _checkDevicesMutex;
QTimer* _checkDevicesTimer { nullptr };
Mutex _checkPeakValuesMutex;
QTimer* _checkPeakValuesTimer { nullptr };
bool _isRecording { false };
};
#endif // hifi_AudioClient_h