diff --git a/interface/src/scripting/Audio.cpp b/interface/src/scripting/Audio.cpp index ba392f0cd1..43958188e3 100644 --- a/interface/src/scripting/Audio.cpp +++ b/interface/src/scripting/Audio.cpp @@ -440,13 +440,13 @@ void Audio::handlePushedToTalk(bool enabled) { } } -void Audio::setInputDevice(const QAudioDeviceInfo& device, bool isHMD) { +void Audio::setInputDevice(const HifiAudioDeviceInfo& device, bool isHMD) { withWriteLock([&] { _devices.chooseInputDevice(device, isHMD); }); } -void Audio::setOutputDevice(const QAudioDeviceInfo& device, bool isHMD) { +void Audio::setOutputDevice(const HifiAudioDeviceInfo& device, bool isHMD) { withWriteLock([&] { _devices.chooseOutputDevice(device, isHMD); }); diff --git a/interface/src/scripting/Audio.h b/interface/src/scripting/Audio.h index d3bc6b9449..ab4fe89ce9 100644 --- a/interface/src/scripting/Audio.h +++ b/interface/src/scripting/Audio.h @@ -19,6 +19,7 @@ #include "SettingHandle.h" #include "AudioFileWav.h" #include +#include "HifiAudioDeviceInfo.h" using MutedGetter = std::function; using MutedSetter = std::function; @@ -158,7 +159,7 @@ public: * @param {boolean} isHMD - Is HMD. * @deprecated This function is deprecated and will be removed. */ - Q_INVOKABLE void setInputDevice(const QAudioDeviceInfo& device, bool isHMD); + Q_INVOKABLE void setInputDevice(const HifiAudioDeviceInfo& device, bool isHMD); /**jsdoc * @function Audio.setOutputDevice @@ -166,7 +167,7 @@ public: * @param {boolean} isHMD - Is HMD. * @deprecated This function is deprecated and will be removed. */ - Q_INVOKABLE void setOutputDevice(const QAudioDeviceInfo& device, bool isHMD); + Q_INVOKABLE void setOutputDevice(const HifiAudioDeviceInfo& device, bool isHMD); /**jsdoc * Enables or disables reverberation. Reverberation is done by the client on the post-mix audio. The reverberation options diff --git a/interface/src/scripting/AudioDevices.cpp b/interface/src/scripting/AudioDevices.cpp index 02072d3ff0..dbdd0c4852 100644 --- a/interface/src/scripting/AudioDevices.cpp +++ b/interface/src/scripting/AudioDevices.cpp @@ -29,6 +29,8 @@ static Setting::Handle desktopOutputDeviceSetting { QStringList { Audio static Setting::Handle hmdInputDeviceSetting { QStringList { Audio::AUDIO, Audio::HMD, "INPUT" }}; static Setting::Handle hmdOutputDeviceSetting { QStringList { Audio::AUDIO, Audio::HMD, "OUTPUT" }}; +Q_DECLARE_METATYPE(HifiAudioDeviceInfo); + Setting::Handle& getSetting(bool contextIsHMD, QAudio::Mode mode) { if (mode == QAudio::AudioInput) { return contextIsHMD ? hmdInputDeviceSetting : desktopInputDeviceSetting; @@ -139,7 +141,7 @@ QVariant AudioDeviceList::data(const QModelIndex& index, int role) const { } else if (role == SelectedHMDRole) { return _devices.at(index.row())->selectedHMD; } else if (role == InfoRole) { - return QVariant::fromValue(_devices.at(index.row())->info); + return QVariant::fromValue(_devices.at(index.row())->info); } else { return QVariant(); } @@ -191,8 +193,8 @@ void AudioDeviceList::resetDevice(bool contextIsHMD) { #endif } -void AudioDeviceList::onDeviceChanged(const QAudioDeviceInfo& device, bool isHMD) { - QAudioDeviceInfo& selectedDevice = isHMD ? _selectedHMDDevice : _selectedDesktopDevice; +void AudioDeviceList::onDeviceChanged(const HifiAudioDeviceInfo& device, bool isHMD) { + HifiAudioDeviceInfo& selectedDevice = isHMD ? _selectedHMDDevice : _selectedDesktopDevice; selectedDevice = device; for (auto i = 0; i < _devices.size(); ++i) { @@ -259,25 +261,25 @@ std::shared_ptr getSimilarDevice(const QString& deviceNa return devices[minDistanceIndex]; } -void AudioDeviceList::onDevicesChanged(const QList& devices) { +void AudioDeviceList::onDevicesChanged(const QList& devices) { beginResetModel(); QList> newDevices; bool hmdIsSelected = false; bool desktopIsSelected = false; - foreach(const QAudioDeviceInfo& deviceInfo, devices) { + foreach(const HifiAudioDeviceInfo& deviceInfo, devices) { for (bool isHMD : {false, true}) { auto& backupSelectedDeviceName = isHMD ? _backupSelectedHMDDeviceName : _backupSelectedDesktopDeviceName; if (deviceInfo.deviceName() == backupSelectedDeviceName) { - QAudioDeviceInfo& selectedDevice = isHMD ? _selectedHMDDevice : _selectedDesktopDevice; + HifiAudioDeviceInfo& selectedDevice = isHMD ? _selectedHMDDevice : _selectedDesktopDevice; selectedDevice = deviceInfo; backupSelectedDeviceName.clear(); } } } - foreach(const QAudioDeviceInfo& deviceInfo, devices) { + foreach(const HifiAudioDeviceInfo& deviceInfo, devices) { AudioDevice device; device.info = deviceInfo; device.display = device.info.deviceName() @@ -286,10 +288,10 @@ void AudioDeviceList::onDevicesChanged(const QList& devices) { .replace(" )", ")"); for (bool isHMD : {false, true}) { - QAudioDeviceInfo& selectedDevice = isHMD ? _selectedHMDDevice : _selectedDesktopDevice; + HifiAudioDeviceInfo& selectedDevice = isHMD ? _selectedHMDDevice : _selectedDesktopDevice; bool& isSelected = isHMD ? device.selectedHMD : device.selectedDesktop; - if (!selectedDevice.isNull()) { + if (!selectedDevice.getDevice().isNull()) { isSelected = (device.info == selectedDevice); } else { @@ -325,13 +327,13 @@ void AudioDeviceList::onDevicesChanged(const QList& devices) { if (!newDevices.isEmpty()) { if (!hmdIsSelected) { - _backupSelectedHMDDeviceName = !_selectedHMDDevice.isNull() ? _selectedHMDDevice.deviceName() : _hmdSavedDeviceName; + _backupSelectedHMDDeviceName = !_selectedHMDDevice.getDevice().isNull() ? _selectedHMDDevice.deviceName() : _hmdSavedDeviceName; auto device = getSimilarDevice(_backupSelectedHMDDeviceName, newDevices); device->selectedHMD = true; emit selectedDevicePlugged(device->info, true); } if (!desktopIsSelected) { - _backupSelectedDesktopDeviceName = !_selectedDesktopDevice.isNull() ? _selectedDesktopDevice.deviceName() : _desktopSavedDeviceName; + _backupSelectedDesktopDeviceName = !_selectedDesktopDevice.getDevice().isNull() ? _selectedDesktopDevice.deviceName() : _desktopSavedDeviceName; auto device = getSimilarDevice(_backupSelectedDesktopDeviceName, newDevices); device->selectedDesktop = true; emit selectedDevicePlugged(device->info, false); @@ -382,8 +384,8 @@ AudioDevices::AudioDevices(bool& contextIsHMD) : _contextIsHMD(contextIsHMD) { _outputs.onDeviceChanged(client->getActiveAudioDevice(QAudio::AudioOutput), contextIsHMD); // connections are made after client is initialized, so we must also fetch the devices - const QList& devicesInput = client->getAudioDevices(QAudio::AudioInput); - const QList& devicesOutput = client->getAudioDevices(QAudio::AudioOutput); + const QList& devicesInput = client->getAudioDevices(QAudio::AudioInput); + const QList& devicesOutput = client->getAudioDevices(QAudio::AudioOutput); //setup devices _inputs.onDevicesChanged(devicesInput); @@ -397,9 +399,9 @@ void AudioDevices::onContextChanged(const QString& context) { _outputs.resetDevice(_contextIsHMD); } -void AudioDevices::onDeviceSelected(QAudio::Mode mode, const QAudioDeviceInfo& device, - const QAudioDeviceInfo& previousDevice, bool isHMD) { - QString deviceName = device.isNull() ? QString() : device.deviceName(); +void AudioDevices::onDeviceSelected(QAudio::Mode mode, const HifiAudioDeviceInfo& device, + const HifiAudioDeviceInfo& previousDevice, bool isHMD) { + QString deviceName = device.getDevice().isNull() ? QString() : device.deviceName(); auto& setting = getSetting(isHMD, mode); @@ -410,7 +412,7 @@ void AudioDevices::onDeviceSelected(QAudio::Mode mode, const QAudioDeviceInfo& d setting.set(deviceName); // log the selected device - if (!device.isNull()) { + if (!device.getDevice().isNull()) { QJsonObject data; const QString MODE = "audio_mode"; @@ -434,13 +436,13 @@ void AudioDevices::onDeviceSelected(QAudio::Mode mode, const QAudioDeviceInfo& d } } -void AudioDevices::onDeviceChanged(QAudio::Mode mode, const QAudioDeviceInfo& device) { +void AudioDevices::onDeviceChanged(QAudio::Mode mode, const HifiAudioDeviceInfo& device) { if (mode == QAudio::AudioInput) { if (_requestedInputDevice == device) { onDeviceSelected(QAudio::AudioInput, device, _contextIsHMD ? _inputs._selectedHMDDevice : _inputs._selectedDesktopDevice, _contextIsHMD); - _requestedInputDevice = QAudioDeviceInfo(); + _requestedInputDevice = HifiAudioDeviceInfo(); } _inputs.onDeviceChanged(device, _contextIsHMD); } else { // if (mode == QAudio::AudioOutput) @@ -448,13 +450,13 @@ void AudioDevices::onDeviceChanged(QAudio::Mode mode, const QAudioDeviceInfo& de onDeviceSelected(QAudio::AudioOutput, device, _contextIsHMD ? _outputs._selectedHMDDevice : _outputs._selectedDesktopDevice, _contextIsHMD); - _requestedOutputDevice = QAudioDeviceInfo(); + _requestedOutputDevice = HifiAudioDeviceInfo(); } _outputs.onDeviceChanged(device, _contextIsHMD); } } -void AudioDevices::onDevicesChanged(QAudio::Mode mode, const QList& devices) { +void AudioDevices::onDevicesChanged(QAudio::Mode mode, const QList& devices) { static std::once_flag once; std::call_once(once, [&] { //readout settings @@ -503,14 +505,14 @@ void AudioDevices::onDevicesChanged(QAudio::Mode mode, const QList().data(); _requestedInputDevice = device; QMetaObject::invokeMethod(client, "switchAudioDevice", Q_ARG(QAudio::Mode, QAudio::AudioInput), - Q_ARG(const QAudioDeviceInfo&, device)); + Q_ARG(const QAudioDeviceInfo&, device.getDevice())); } else { //context is different. just save device in settings onDeviceSelected(QAudio::AudioInput, device, @@ -520,14 +522,14 @@ void AudioDevices::chooseInputDevice(const QAudioDeviceInfo& device, bool isHMD) } } -void AudioDevices::chooseOutputDevice(const QAudioDeviceInfo& device, bool isHMD) { +void AudioDevices::chooseOutputDevice(const HifiAudioDeviceInfo& device, bool isHMD) { //check if current context equals device to change if (_contextIsHMD == isHMD) { auto client = DependencyManager::get().data(); _requestedOutputDevice = device; QMetaObject::invokeMethod(client, "switchAudioDevice", Q_ARG(QAudio::Mode, QAudio::AudioOutput), - Q_ARG(const QAudioDeviceInfo&, device)); + Q_ARG(const QAudioDeviceInfo&, device.getDevice())); } else { //context is different. just save device in settings onDeviceSelected(QAudio::AudioOutput, device, diff --git a/interface/src/scripting/AudioDevices.h b/interface/src/scripting/AudioDevices.h index 2ea034d9fe..21cd875f7b 100644 --- a/interface/src/scripting/AudioDevices.h +++ b/interface/src/scripting/AudioDevices.h @@ -19,11 +19,13 @@ #include #include +#include "HifiAudioDeviceInfo.h" + namespace scripting { class AudioDevice { public: - QAudioDeviceInfo info; + HifiAudioDeviceInfo info; QString display; bool selectedDesktop { false }; bool selectedHMD { false }; @@ -50,12 +52,12 @@ public: void resetDevice(bool contextIsHMD); signals: - void deviceChanged(const QAudioDeviceInfo& device); - void selectedDevicePlugged(const QAudioDeviceInfo& device, bool isHMD); + void deviceChanged(const HifiAudioDeviceInfo& device); + void selectedDevicePlugged(const HifiAudioDeviceInfo& device, bool isHMD); protected slots: - void onDeviceChanged(const QAudioDeviceInfo& device, bool isHMD); - void onDevicesChanged(const QList& devices); + void onDeviceChanged(const HifiAudioDeviceInfo& device, bool isHMD); + void onDevicesChanged(const QList& devices); protected: friend class AudioDevices; @@ -63,8 +65,8 @@ protected: static QHash _roles; static Qt::ItemFlags _flags; const QAudio::Mode _mode; - QAudioDeviceInfo _selectedDesktopDevice; - QAudioDeviceInfo _selectedHMDDevice; + HifiAudioDeviceInfo _selectedDesktopDevice; + HifiAudioDeviceInfo _selectedHMDDevice; QString _backupSelectedDesktopDeviceName; QString _backupSelectedHMDDeviceName; QList> _devices; @@ -124,14 +126,14 @@ signals: void nop(); private slots: - void chooseInputDevice(const QAudioDeviceInfo& device, bool isHMD); - void chooseOutputDevice(const QAudioDeviceInfo& device, bool isHMD); + void chooseInputDevice(const HifiAudioDeviceInfo& device, bool isHMD); + void chooseOutputDevice(const HifiAudioDeviceInfo& device, bool isHMD); void onContextChanged(const QString& context); - void onDeviceSelected(QAudio::Mode mode, const QAudioDeviceInfo& device, - const QAudioDeviceInfo& previousDevice, bool isHMD); - void onDeviceChanged(QAudio::Mode mode, const QAudioDeviceInfo& device); - void onDevicesChanged(QAudio::Mode mode, const QList& devices); + void onDeviceSelected(QAudio::Mode mode, const HifiAudioDeviceInfo& device, + const HifiAudioDeviceInfo& previousDevice, bool isHMD); + void onDeviceChanged(QAudio::Mode mode, const HifiAudioDeviceInfo& device); + void onDevicesChanged(QAudio::Mode mode, const QList& devices); private: friend class Audio; @@ -141,8 +143,8 @@ private: AudioInputDeviceList _inputs; AudioDeviceList _outputs { QAudio::AudioOutput }; - QAudioDeviceInfo _requestedOutputDevice; - QAudioDeviceInfo _requestedInputDevice; + HifiAudioDeviceInfo _requestedOutputDevice; + HifiAudioDeviceInfo _requestedInputDevice; const bool& _contextIsHMD; }; diff --git a/libraries/audio-client/src/AudioClient.cpp b/libraries/audio-client/src/AudioClient.cpp index 6add52141c..195afc619c 100644 --- a/libraries/audio-client/src/AudioClient.cpp +++ b/libraries/audio-client/src/AudioClient.cpp @@ -67,7 +67,7 @@ static const int CHECK_INPUT_READS_MSECS = 2000; static const int MIN_READS_TO_CONSIDER_INPUT_ALIVE = 10; #endif -static const auto DEFAULT_POSITION_GETTER = []{ return Vectors::ZERO; }; +static const auto DEFAULT_POSITION_GETTER = [] { return Vectors::ZERO; }; static const auto DEFAULT_ORIENTATION_GETTER = [] { return Quaternions::IDENTITY; }; static const int DEFAULT_BUFFER_FRAMES = 1; @@ -78,12 +78,11 @@ static const int OUTPUT_CHANNEL_COUNT = 2; static const bool DEFAULT_STARVE_DETECTION_ENABLED = true; static const int STARVE_DETECTION_THRESHOLD = 3; -static const int STARVE_DETECTION_PERIOD = 10 * 1000; // 10 Seconds +static const int STARVE_DETECTION_PERIOD = 10 * 1000; // 10 Seconds Setting::Handle dynamicJitterBufferEnabled("dynamicJitterBuffersEnabled", - InboundAudioStream::DEFAULT_DYNAMIC_JITTER_BUFFER_ENABLED); -Setting::Handle staticJitterBufferFrames("staticJitterBufferFrames", - InboundAudioStream::DEFAULT_STATIC_JITTER_FRAMES); + InboundAudioStream::DEFAULT_DYNAMIC_JITTER_BUFFER_ENABLED); +Setting::Handle staticJitterBufferFrames("staticJitterBufferFrames", InboundAudioStream::DEFAULT_STATIC_JITTER_FRAMES); // protect the Qt internal device list using Mutex = std::mutex; @@ -92,10 +91,25 @@ Mutex _deviceMutex; Mutex _recordMutex; // thread-safe -QList getAvailableDevices(QAudio::Mode mode) { +QList getAvailableDevices(QAudio::Mode mode) { // NOTE: availableDevices() clobbers the Qt internal device list Lock lock(_deviceMutex); - return QAudioDeviceInfo::availableDevices(mode); + auto devices = QAudioDeviceInfo::availableDevices(mode); + + QList newDevices; + QAudioDeviceInfo::defaultOutputDevice(); + + for (auto& device : devices) { + bool isDefault = false; + if (mode == QAudio::Mode::AudioInput) { + isDefault = device == QAudioDeviceInfo::defaultInputDevice(); + } else { + isDefault = device == QAudioDeviceInfo::defaultOutputDevice(); + } + newDevices.push_back(HifiAudioDeviceInfo(device, isDefault, mode)); + } + + return newDevices; } // now called from a background thread, to keep blocking operations off the audio thread @@ -122,29 +136,28 @@ void AudioClient::checkDevices() { } } -QAudioDeviceInfo AudioClient::getActiveAudioDevice(QAudio::Mode mode) const { +HifiAudioDeviceInfo AudioClient::getActiveAudioDevice(QAudio::Mode mode) const { Lock lock(_deviceMutex); if (mode == QAudio::AudioInput) { return _inputDeviceInfo; - } else { // if (mode == QAudio::AudioOutput) + } else { return _outputDeviceInfo; } } -QList AudioClient::getAudioDevices(QAudio::Mode mode) const { +QList AudioClient::getAudioDevices(QAudio::Mode mode) const { Lock lock(_deviceMutex); if (mode == QAudio::AudioInput) { return _inputDevices; - } else { // if (mode == QAudio::AudioOutput) + } else { // if (mode == QAudio::AudioOutput) return _outputDevices; } } static void channelUpmix(int16_t* source, int16_t* dest, int numSamples, int numExtraChannels) { - for (int i = 0; i < numSamples/2; i++) { - + for (int i = 0; i < numSamples / 2; i++) { // read 2 samples int16_t left = *source++; int16_t right = *source++; @@ -159,8 +172,7 @@ static void channelUpmix(int16_t* source, int16_t* dest, int numSamples, int num } static void channelDownmix(int16_t* source, int16_t* dest, int numSamples) { - for (int i = 0; i < numSamples/2; i++) { - + for (int i = 0; i < numSamples / 2; i++) { // read 2 samples int16_t left = *source++; int16_t right = *source++; @@ -171,9 +183,8 @@ static void channelDownmix(int16_t* source, int16_t* dest, int numSamples) { } static bool detectClipping(int16_t* samples, int numSamples, int numChannels) { - - const int32_t CLIPPING_THRESHOLD = 32392; // -0.1 dBFS - const int CLIPPING_DETECTION = 3; // consecutive samples over threshold + const int32_t CLIPPING_THRESHOLD = 32392; // -0.1 dBFS + const int CLIPPING_DETECTION = 3; // consecutive samples over threshold bool isClipping = false; @@ -181,9 +192,9 @@ static bool detectClipping(int16_t* samples, int numSamples, int numChannels) { int oversLeft = 0; int oversRight = 0; - for (int i = 0; i < numSamples/2; i++) { - int32_t left = std::abs((int32_t)samples[2*i+0]); - int32_t right = std::abs((int32_t)samples[2*i+1]); + for (int i = 0; i < numSamples / 2; i++) { + int32_t left = std::abs((int32_t)samples[2 * i + 0]); + int32_t right = std::abs((int32_t)samples[2 * i + 1]); if (left > CLIPPING_THRESHOLD) { isClipping |= (++oversLeft >= CLIPPING_DETECTION); @@ -214,7 +225,6 @@ static bool detectClipping(int16_t* samples, int numSamples, int numChannels) { } static float computeLoudness(int16_t* samples, int numSamples) { - float scale = numSamples ? 1.0f / numSamples : 0.0f; int32_t loudness = 0; @@ -226,7 +236,6 @@ static float computeLoudness(int16_t* samples, int numSamples) { template static void applyGainSmoothing(float* buffer, int numFrames, float gain0, float gain1) { - // fast path for unity gain if (gain0 == 1.0f && gain1 == 1.0f) { return; @@ -241,14 +250,13 @@ static void applyGainSmoothing(float* buffer, int numFrames, float gain0, float float tStep = 1.0f / numFrames; for (int i = 0; i < numFrames; i++) { - // evaluate poly over t=[0,1) float gain = (c3 * t + c2) * t * t + c0; t += tStep; // apply gain to all channels for (int ch = 0; ch < NUM_CHANNELS; ch++) { - buffer[NUM_CHANNELS*i + ch] *= gain; + buffer[NUM_CHANNELS * i + ch] *= gain; } } } @@ -258,52 +266,22 @@ static inline float convertToFloat(int16_t sample) { } AudioClient::AudioClient() : - AbstractAudioInterface(), - _gate(this), - _audioInput(NULL), - _dummyAudioInput(NULL), - _desiredInputFormat(), - _inputFormat(), - _numInputCallbackBytes(0), - _audioOutput(NULL), - _desiredOutputFormat(), - _outputFormat(), - _outputFrameSize(0), - _numOutputCallbackBytes(0), - _loopbackAudioOutput(NULL), - _loopbackOutputDevice(NULL), - _inputRingBuffer(0), - _localInjectorsStream(0, 1), - _receivedAudioStream(RECEIVED_AUDIO_STREAM_CAPACITY_FRAMES), - _isStereoInput(false), - _outputStarveDetectionStartTimeMsec(0), - _outputStarveDetectionCount(0), + AbstractAudioInterface(), _gate(this), _audioInput(NULL), _dummyAudioInput(NULL), _desiredInputFormat(), _inputFormat(), + _numInputCallbackBytes(0), _audioOutput(NULL), _desiredOutputFormat(), _outputFormat(), _outputFrameSize(0), + _numOutputCallbackBytes(0), _loopbackAudioOutput(NULL), _loopbackOutputDevice(NULL), _inputRingBuffer(0), + _localInjectorsStream(0, 1), _receivedAudioStream(RECEIVED_AUDIO_STREAM_CAPACITY_FRAMES), _isStereoInput(false), + _outputStarveDetectionStartTimeMsec(0), _outputStarveDetectionCount(0), _outputBufferSizeFrames("audioOutputBufferFrames", DEFAULT_BUFFER_FRAMES), _sessionOutputBufferSizeFrames(_outputBufferSizeFrames.get()), _outputStarveDetectionEnabled("audioOutputStarveDetectionEnabled", DEFAULT_STARVE_DETECTION_ENABLED), - _lastRawInputLoudness(0.0f), - _lastSmoothedRawInputLoudness(0.0f), - _lastInputLoudness(0.0f), - _timeSinceLastClip(-1.0f), - _muted(false), - _shouldEchoLocally(false), - _shouldEchoToServer(false), - _isNoiseGateEnabled(true), - _isAECEnabled(true), - _reverb(false), - _reverbOptions(&_scriptReverbOptions), - _inputToNetworkResampler(NULL), - _networkToOutputResampler(NULL), - _localToOutputResampler(NULL), - _loopbackResampler(NULL), - _audioLimiter(AudioConstants::SAMPLE_RATE, OUTPUT_CHANNEL_COUNT), - _outgoingAvatarAudioSequenceNumber(0), - _audioOutputIODevice(_localInjectorsStream, _receivedAudioStream, this), - _stats(&_receivedAudioStream), - _positionGetter(DEFAULT_POSITION_GETTER), + _lastRawInputLoudness(0.0f), _lastSmoothedRawInputLoudness(0.0f), _lastInputLoudness(0.0f), _timeSinceLastClip(-1.0f), + _muted(false), _shouldEchoLocally(false), _shouldEchoToServer(false), _isNoiseGateEnabled(true), _isAECEnabled(true), + _reverb(false), _reverbOptions(&_scriptReverbOptions), _inputToNetworkResampler(NULL), _networkToOutputResampler(NULL), + _localToOutputResampler(NULL), _loopbackResampler(NULL), _audioLimiter(AudioConstants::SAMPLE_RATE, OUTPUT_CHANNEL_COUNT), + _outgoingAvatarAudioSequenceNumber(0), _audioOutputIODevice(_localInjectorsStream, _receivedAudioStream, this), + _stats(&_receivedAudioStream), _positionGetter(DEFAULT_POSITION_GETTER), #if defined(Q_OS_ANDROID) - _checkInputTimer(this), - _isHeadsetPluggedIn(false), + _checkInputTimer(this), _isHeadsetPluggedIn(false), #endif _orientationGetter(DEFAULT_ORIENTATION_GETTER) { @@ -314,17 +292,21 @@ AudioClient::AudioClient() : { Setting::Handle::Deprecated("maxFramesOverDesired", InboundAudioStream::MAX_FRAMES_OVER_DESIRED); Setting::Handle::Deprecated("windowStarveThreshold", InboundAudioStream::WINDOW_STARVE_THRESHOLD); - Setting::Handle::Deprecated("windowSecondsForDesiredCalcOnTooManyStarves", InboundAudioStream::WINDOW_SECONDS_FOR_DESIRED_CALC_ON_TOO_MANY_STARVES); - Setting::Handle::Deprecated("windowSecondsForDesiredReduction", InboundAudioStream::WINDOW_SECONDS_FOR_DESIRED_REDUCTION); + Setting::Handle::Deprecated("windowSecondsForDesiredCalcOnTooManyStarves", + InboundAudioStream::WINDOW_SECONDS_FOR_DESIRED_CALC_ON_TOO_MANY_STARVES); + Setting::Handle::Deprecated("windowSecondsForDesiredReduction", + InboundAudioStream::WINDOW_SECONDS_FOR_DESIRED_REDUCTION); Setting::Handle::Deprecated("useStDevForJitterCalc", InboundAudioStream::USE_STDEV_FOR_JITTER); Setting::Handle::Deprecated("repetitionWithFade", InboundAudioStream::REPETITION_WITH_FADE); } - connect(&_receivedAudioStream, &MixedProcessedAudioStream::processSamples, - this, &AudioClient::processReceivedSamples, Qt::DirectConnection); - connect(this, &AudioClient::changeDevice, this, [=](const QAudioDeviceInfo& outputDeviceInfo) { - qCDebug(audioclient) << "got AudioClient::changeDevice signal, about to call switchOutputToAudioDevice() outputDeviceInfo: [" << outputDeviceInfo.deviceName() << "]"; - switchOutputToAudioDevice(outputDeviceInfo); + connect(&_receivedAudioStream, &MixedProcessedAudioStream::processSamples, this, &AudioClient::processReceivedSamples, + Qt::DirectConnection); + connect(this, &AudioClient::changeDevice, this, [=](const HifiAudioDeviceInfo& outputDeviceInfo) { + qCDebug(audioclient) + << "got AudioClient::changeDevice signal, about to call switchOutputToAudioDevice() outputDeviceInfo: [" + << outputDeviceInfo.deviceName() << "]"; + switchOutputToAudioDevice(outputDeviceInfo.getDevice()); }); connect(&_receivedAudioStream, &InboundAudioStream::mismatchedAudioCodec, this, &AudioClient::handleMismatchAudioFormat); @@ -350,9 +332,8 @@ AudioClient::AudioClient() : // start a thread to detect peak value changes _checkPeakValuesTimer = new QTimer(this); - connect(_checkPeakValuesTimer, &QTimer::timeout, this, [this] { - QtConcurrent::run(QThreadPool::globalInstance(), [this] { checkPeakValues(); }); - }); + connect(_checkPeakValuesTimer, &QTimer::timeout, this, + [this] { QtConcurrent::run(QThreadPool::globalInstance(), [this] { checkPeakValues(); }); }); const unsigned long PEAK_VALUES_CHECK_INTERVAL_MSECS = 50; _checkPeakValuesTimer->start(PEAK_VALUES_CHECK_INTERVAL_MSECS); @@ -373,9 +354,7 @@ AudioClient::AudioClient() : packetReceiver.registerListener(PacketType::SelectedAudioFormat, this, "handleSelectedAudioFormat"); auto& domainHandler = nodeList->getDomainHandler(); - connect(&domainHandler, &DomainHandler::disconnectedFromDomain, this, [this] { - _solo.reset(); - }); + connect(&domainHandler, &DomainHandler::disconnectedFromDomain, this, [this] { _solo.reset(); }); connect(nodeList.data(), &NodeList::nodeActivated, this, [this](SharedNodePointer node) { if (node->getType() == NodeType::AudioMixer) { _solo.resend(); @@ -384,7 +363,6 @@ AudioClient::AudioClient() : } AudioClient::~AudioClient() { - stop(); if (_codec && _encoder) { @@ -401,11 +379,11 @@ void AudioClient::customDeleter() { } void AudioClient::handleMismatchAudioFormat(SharedNodePointer node, const QString& currentCodec, const QString& recievedCodec) { - qCDebug(audioclient) << __FUNCTION__ << "sendingNode:" << *node << "currentCodec:" << currentCodec << "recievedCodec:" << recievedCodec; + qCDebug(audioclient) << __FUNCTION__ << "sendingNode:" << *node << "currentCodec:" << currentCodec + << "recievedCodec:" << recievedCodec; selectAudioFormat(recievedCodec); } - void AudioClient::reset() { _receivedAudioStream.reset(); _stats.reset(); @@ -433,9 +411,9 @@ void AudioClient::setAudioPaused(bool pause) { QAudioDeviceInfo getNamedAudioDeviceForMode(QAudio::Mode mode, const QString& deviceName) { QAudioDeviceInfo result; - foreach(QAudioDeviceInfo audioDevice, getAvailableDevices(mode)) { + foreach (HifiAudioDeviceInfo audioDevice, getAvailableDevices(mode)) { if (audioDevice.deviceName().trimmed() == deviceName.trimmed()) { - result = audioDevice; + result = audioDevice.getDevice(); break; } } @@ -468,7 +446,8 @@ QString AudioClient::getWinDeviceName(wchar_t* guid) { HRESULT hr = S_OK; CoInitialize(nullptr); IMMDeviceEnumerator* pMMDeviceEnumerator = nullptr; - CoCreateInstance(__uuidof(MMDeviceEnumerator), nullptr, CLSCTX_ALL, __uuidof(IMMDeviceEnumerator), (void**)&pMMDeviceEnumerator); + CoCreateInstance(__uuidof(MMDeviceEnumerator), nullptr, CLSCTX_ALL, __uuidof(IMMDeviceEnumerator), + (void**)&pMMDeviceEnumerator); IMMDevice* pEndpoint; hr = pMMDeviceEnumerator->GetDevice(guid, &pEndpoint); if (hr == E_NOTFOUND) { @@ -492,34 +471,26 @@ QAudioDeviceInfo defaultAudioDeviceForMode(QAudio::Mode mode) { if (getAvailableDevices(mode).size() > 1) { AudioDeviceID defaultDeviceID = 0; uint32_t propertySize = sizeof(AudioDeviceID); - AudioObjectPropertyAddress propertyAddress = { - kAudioHardwarePropertyDefaultInputDevice, - kAudioObjectPropertyScopeGlobal, - kAudioObjectPropertyElementMaster - }; + AudioObjectPropertyAddress propertyAddress = { kAudioHardwarePropertyDefaultInputDevice, + kAudioObjectPropertyScopeGlobal, kAudioObjectPropertyElementMaster }; if (mode == QAudio::AudioOutput) { propertyAddress.mSelector = kAudioHardwarePropertyDefaultOutputDevice; } - - OSStatus getPropertyError = AudioObjectGetPropertyData(kAudioObjectSystemObject, - &propertyAddress, - 0, - NULL, - &propertySize, - &defaultDeviceID); + OSStatus getPropertyError = + AudioObjectGetPropertyData(kAudioObjectSystemObject, &propertyAddress, 0, NULL, &propertySize, &defaultDeviceID); if (!getPropertyError && propertySize) { CFStringRef deviceName = NULL; propertySize = sizeof(deviceName); propertyAddress.mSelector = kAudioDevicePropertyDeviceNameCFString; - getPropertyError = AudioObjectGetPropertyData(defaultDeviceID, &propertyAddress, 0, - NULL, &propertySize, &deviceName); + getPropertyError = + AudioObjectGetPropertyData(defaultDeviceID, &propertyAddress, 0, NULL, &propertySize, &deviceName); if (!getPropertyError && propertySize) { // find a device in the list that matches the name we have and return it - foreach(QAudioDeviceInfo audioDevice, getAvailableDevices(mode)) { + foreach (QAudioDeviceInfo audioDevice, getAvailableDevices(mode)) { if (audioDevice.deviceName() == CFStringGetCStringPtr(deviceName, kCFStringEncodingMacRoman)) { return audioDevice; } @@ -531,7 +502,7 @@ QAudioDeviceInfo defaultAudioDeviceForMode(QAudio::Mode mode) { #ifdef WIN32 QString deviceName; //Check for Windows Vista or higher, IMMDeviceEnumerator doesn't work below that. - if (!IsWindowsVistaOrGreater()) { // lower then vista + if (!IsWindowsVistaOrGreater()) { // lower then vista if (mode == QAudio::AudioInput) { WAVEINCAPS wic; // first use WAVE_MAPPER to get the default devices manufacturer ID @@ -553,9 +524,11 @@ QAudioDeviceInfo defaultAudioDeviceForMode(QAudio::Mode mode) { HRESULT hr = S_OK; CoInitialize(NULL); IMMDeviceEnumerator* pMMDeviceEnumerator = NULL; - CoCreateInstance(__uuidof(MMDeviceEnumerator), NULL, CLSCTX_ALL, __uuidof(IMMDeviceEnumerator), (void**)&pMMDeviceEnumerator); + CoCreateInstance(__uuidof(MMDeviceEnumerator), NULL, CLSCTX_ALL, __uuidof(IMMDeviceEnumerator), + (void**)&pMMDeviceEnumerator); IMMDevice* pEndpoint; - hr = pMMDeviceEnumerator->GetDefaultAudioEndpoint(mode == QAudio::AudioOutput ? eRender : eCapture, eMultimedia, &pEndpoint); + hr = pMMDeviceEnumerator->GetDefaultAudioEndpoint(mode == QAudio::AudioOutput ? eRender : eCapture, eMultimedia, + &pEndpoint); if (hr == E_NOTFOUND) { printf("Audio Error: device not found\n"); deviceName = QString("NONE"); @@ -569,22 +542,22 @@ QAudioDeviceInfo defaultAudioDeviceForMode(QAudio::Mode mode) { CoUninitialize(); } - qCDebug(audioclient) << "defaultAudioDeviceForMode mode: " << (mode == QAudio::AudioOutput ? "Output" : "Input") - << " [" << deviceName << "] [" << getNamedAudioDeviceForMode(mode, deviceName).deviceName() << "]"; + qCDebug(audioclient) << "defaultAudioDeviceForMode mode: " << (mode == QAudio::AudioOutput ? "Output" : "Input") << " [" + << deviceName << "] [" << getNamedAudioDeviceForMode(mode, deviceName).deviceName() << "]"; return getNamedAudioDeviceForMode(mode, deviceName); #endif -#if defined (Q_OS_ANDROID) +#if defined(Q_OS_ANDROID) if (mode == QAudio::AudioInput) { Setting::Handle enableAEC(SETTING_AEC_KEY, DEFAULT_AEC_ENABLED); bool aecEnabled = enableAEC.get(); auto audioClient = DependencyManager::get(); - bool headsetOn = audioClient? audioClient->isHeadsetPluggedIn() : false; + bool headsetOn = audioClient ? audioClient->isHeadsetPluggedIn() : false; auto inputDevices = QAudioDeviceInfo::availableDevices(QAudio::AudioInput); for (auto inputDevice : inputDevices) { if (((headsetOn || !aecEnabled) && inputDevice.deviceName() == VOICE_RECOGNITION) || - ((!headsetOn && aecEnabled) && inputDevice.deviceName() == VOICE_COMMUNICATION)) { + ((!headsetOn && aecEnabled) && inputDevice.deviceName() == VOICE_COMMUNICATION)) { return inputDevice; } } @@ -598,10 +571,8 @@ bool AudioClient::getNamedAudioDeviceForModeExists(QAudio::Mode mode, const QStr return (getNamedAudioDeviceForMode(mode, deviceName).deviceName() == deviceName); } - // attempt to use the native sample rate and channel count bool nativeFormatForAudioDevice(const QAudioDeviceInfo& audioDevice, QAudioFormat& audioFormat) { - audioFormat = audioDevice.preferredFormat(); // converting to/from this rate must produce an integral number of samples @@ -622,7 +593,6 @@ bool nativeFormatForAudioDevice(const QAudioDeviceInfo& audioDevice, QAudioForma // attempt the native sample rate, with channels forced to 2 audioFormat.setChannelCount(2); if (!audioDevice.isFormatSupported(audioFormat)) { - // attempt the native sample rate, with channels forced to 1 audioFormat.setChannelCount(1); if (!audioDevice.isFormatSupported(audioFormat)) { @@ -636,40 +606,38 @@ bool nativeFormatForAudioDevice(const QAudioDeviceInfo& audioDevice, QAudioForma bool adjustedFormatForAudioDevice(const QAudioDeviceInfo& audioDevice, const QAudioFormat& desiredAudioFormat, QAudioFormat& adjustedAudioFormat) { - qCDebug(audioclient) << "The desired format for audio I/O is" << desiredAudioFormat; #if defined(Q_OS_WIN) if (IsWindows8OrGreater()) { // On Windows using WASAPI shared-mode, returns the internal mix format return nativeFormatForAudioDevice(audioDevice, adjustedAudioFormat); - } // else enumerate formats + } // else enumerate formats #endif - + #if defined(Q_OS_MAC) // Mac OSX returns the preferred CoreAudio format return nativeFormatForAudioDevice(audioDevice, adjustedAudioFormat); #endif - + #if defined(Q_OS_ANDROID) // As of Qt5.6, Android returns the native OpenSLES sample rate when possible, else 48000 if (nativeFormatForAudioDevice(audioDevice, adjustedAudioFormat)) { return true; - } // else enumerate formats + } // else enumerate formats #endif - + adjustedAudioFormat = desiredAudioFormat; // // Attempt the device sample rate and channel count in decreasing order of preference. // const int sampleRates[] = { 48000, 44100, 32000, 24000, 16000, 96000, 192000, 88200, 176400 }; - const int inputChannels[] = { 1, 2, 4, 6, 8 }; // prefer mono - const int outputChannels[] = { 2, 4, 6, 8, 1 }; // prefer stereo, downmix as last resort + const int inputChannels[] = { 1, 2, 4, 6, 8 }; // prefer mono + const int outputChannels[] = { 2, 4, 6, 8, 1 }; // prefer stereo, downmix as last resort for (int channelCount : (desiredAudioFormat.channelCount() == 1 ? inputChannels : outputChannels)) { for (int sampleRate : sampleRates) { - adjustedAudioFormat.setChannelCount(channelCount); adjustedAudioFormat.setSampleRate(sampleRate); @@ -679,11 +647,14 @@ bool adjustedFormatForAudioDevice(const QAudioDeviceInfo& audioDevice, } } - return false; // a supported format could not be found + return false; // a supported format could not be found } -bool sampleChannelConversion(const int16_t* sourceSamples, int16_t* destinationSamples, int numSourceSamples, - const int sourceChannelCount, const int destinationChannelCount) { +bool sampleChannelConversion(const int16_t* sourceSamples, + int16_t* destinationSamples, + int numSourceSamples, + const int sourceChannelCount, + const int destinationChannelCount) { if (sourceChannelCount == 2 && destinationChannelCount == 1) { // loop through the stereo input audio samples and average every two samples for (int i = 0; i < numSourceSamples; i += 2) { @@ -692,7 +663,6 @@ bool sampleChannelConversion(const int16_t* sourceSamples, int16_t* destinationS return true; } else if (sourceChannelCount == 1 && destinationChannelCount == 2) { - // loop through the mono input audio and repeat each sample twice for (int i = 0; i < numSourceSamples; ++i) { destinationSamples[i * 2] = destinationSamples[(i * 2) + 1] = sourceSamples[i]; @@ -705,28 +675,29 @@ bool sampleChannelConversion(const int16_t* sourceSamples, int16_t* destinationS } int possibleResampling(AudioSRC* resampler, - const int16_t* sourceSamples, int16_t* destinationSamples, - int numSourceSamples, int maxDestinationSamples, - const int sourceChannelCount, const int destinationChannelCount) { - + const int16_t* sourceSamples, + int16_t* destinationSamples, + int numSourceSamples, + int maxDestinationSamples, + const int sourceChannelCount, + const int destinationChannelCount) { int numSourceFrames = numSourceSamples / sourceChannelCount; int numDestinationFrames = 0; if (numSourceSamples > 0) { if (!resampler) { - if (!sampleChannelConversion(sourceSamples, destinationSamples, numSourceSamples, - sourceChannelCount, destinationChannelCount)) { + if (!sampleChannelConversion(sourceSamples, destinationSamples, numSourceSamples, sourceChannelCount, + destinationChannelCount)) { // no conversion, we can copy the samples directly across memcpy(destinationSamples, sourceSamples, numSourceSamples * AudioConstants::SAMPLE_SIZE); } numDestinationFrames = numSourceFrames; } else { if (sourceChannelCount != destinationChannelCount) { - int16_t* channelConversionSamples = new int16_t[numSourceFrames * destinationChannelCount]; - sampleChannelConversion(sourceSamples, channelConversionSamples, numSourceSamples, - sourceChannelCount, destinationChannelCount); + sampleChannelConversion(sourceSamples, channelConversionSamples, numSourceSamples, sourceChannelCount, + destinationChannelCount); numDestinationFrames = resampler->render(channelConversionSamples, destinationSamples, numSourceFrames); @@ -746,7 +717,6 @@ int possibleResampling(AudioSRC* resampler, } void AudioClient::start() { - // set up the desired audio format _desiredInputFormat.setSampleRate(AudioConstants::SAMPLE_RATE); _desiredInputFormat.setSampleSize(16); @@ -835,7 +805,6 @@ void AudioClient::handleAudioDataPacket(QSharedPointer message) nodeList->flagTimeForConnectionStep(LimitedNodeList::ConnectionStep::ReceiveFirstAudioPacket); if (_audioOutput) { - if (!_hasReceivedFirstPacket) { _hasReceivedFirstPacket = true; @@ -852,8 +821,8 @@ void AudioClient::handleAudioDataPacket(QSharedPointer message) } } -AudioClient::Gate::Gate(AudioClient* audioClient) : - _audioClient(audioClient) {} +AudioClient::Gate::Gate(AudioClient* audioClient) : _audioClient(audioClient) { +} void AudioClient::Gate::setIsSimulatingJitter(bool enable) { std::lock_guard lock(_mutex); @@ -906,7 +875,6 @@ void AudioClient::Gate::flush() { _index = 0; } - void AudioClient::handleNoisyMutePacket(QSharedPointer message) { if (!_muted) { setMuted(true); @@ -952,7 +920,6 @@ void AudioClient::handleSelectedAudioFormat(QSharedPointer mess } void AudioClient::selectAudioFormat(const QString& selectedCodecName) { - _selectedCodecName = selectedCodecName; qCDebug(audioclient) << "Selected Codec:" << _selectedCodecName << "isStereoInput:" << _isStereoInput; @@ -970,12 +937,12 @@ void AudioClient::selectAudioFormat(const QString& selectedCodecName) { if (_selectedCodecName == plugin->getName()) { _codec = plugin; _receivedAudioStream.setupCodec(plugin, _selectedCodecName, AudioConstants::STEREO); - _encoder = plugin->createEncoder(AudioConstants::SAMPLE_RATE, _isStereoInput ? AudioConstants::STEREO : AudioConstants::MONO); + _encoder = plugin->createEncoder(AudioConstants::SAMPLE_RATE, + _isStereoInput ? AudioConstants::STEREO : AudioConstants::MONO); qCDebug(audioclient) << "Selected Codec Plugin:" << _codec.get(); break; } } - } bool AudioClient::switchAudioDevice(QAudio::Mode mode, const QAudioDeviceInfo& deviceInfo) { @@ -987,7 +954,7 @@ bool AudioClient::switchAudioDevice(QAudio::Mode mode, const QAudioDeviceInfo& d if (mode == QAudio::AudioInput) { return switchInputToAudioDevice(device); - } else { // if (mode == QAudio::AudioOutput) + } else { // if (mode == QAudio::AudioOutput) return switchOutputToAudioDevice(device); } } @@ -1029,8 +996,8 @@ void AudioClient::configureReverb() { p.sampleRate = _outputFormat.sampleRate(); p.wetDryMix = 100.0f; p.preDelay = 0.0f; - p.earlyGain = -96.0f; // disable ER - p.lateGain += _reverbOptions->getWetDryMix() * (24.0f/100.0f) - 24.0f; // -0dB to -24dB, based on wetDryMix + p.earlyGain = -96.0f; // disable ER + p.lateGain += _reverbOptions->getWetDryMix() * (24.0f / 100.0f) - 24.0f; // -0dB to -24dB, based on wetDryMix p.lateMixLeft = 0.0f; p.lateMixRight = 0.0f; @@ -1040,7 +1007,6 @@ void AudioClient::configureReverb() { void AudioClient::updateReverbOptions() { bool reverbChanged = false; if (_receivedAudioStream.hasReverb()) { - if (_zoneReverbOptions.getReverbTime() != _receivedAudioStream.getRevebTime()) { _zoneReverbOptions.setReverbTime(_receivedAudioStream.getRevebTime()); reverbChanged = true; @@ -1110,8 +1076,8 @@ static void deinterleaveToFloat(const int16_t* src, float* const* dst, int numFr for (int i = 0; i < numFrames; i++) { for (int ch = 0; ch < numChannels; ch++) { float f = *src++; - f *= (1/32768.0f); // scale - dst[ch][i] = f; // deinterleave + f *= (1 / 32768.0f); // scale + dst[ch][i] = f; // deinterleave } } } @@ -1120,10 +1086,10 @@ static void interleaveToInt16(const float* const* src, int16_t* dst, int numFram for (int i = 0; i < numFrames; i++) { for (int ch = 0; ch < numChannels; ch++) { float f = src[ch][i]; - f *= 32768.0f; // scale - f += (f < 0.0f) ? -0.5f : 0.5f; // round - f = std::max(std::min(f, 32767.0f), -32768.0f); // saturate - *dst++ = (int16_t)f; // interleave + f *= 32768.0f; // scale + f += (f < 0.0f) ? -0.5f : 0.5f; // round + f = std::max(std::min(f, 32767.0f), -32768.0f); // saturate + *dst++ = (int16_t)f; // interleave } } } @@ -1153,7 +1119,6 @@ void AudioClient::configureWebrtc() { // rebuffer into 10ms chunks void AudioClient::processWebrtcFarEnd(const int16_t* samples, int numFrames, int numChannels, int sampleRate) { - const webrtc::StreamConfig streamConfig = webrtc::StreamConfig(sampleRate, numChannels); const int numChunk = (int)streamConfig.num_frames(); @@ -1167,7 +1132,6 @@ void AudioClient::processWebrtcFarEnd(const int16_t* samples, int numFrames, int } while (numFrames > 0) { - // number of frames to fill int numFill = std::min(numFrames, numChunk - _numFifoFarEnd); @@ -1178,7 +1142,6 @@ void AudioClient::processWebrtcFarEnd(const int16_t* samples, int numFrames, int _numFifoFarEnd += numFill; if (_numFifoFarEnd == numChunk) { - // convert audio format float buffer[WEBRTC_CHANNELS_MAX][WEBRTC_FRAMES_MAX]; float* const buffers[WEBRTC_CHANNELS_MAX] = { buffer[0], buffer[1] }; @@ -1195,7 +1158,6 @@ void AudioClient::processWebrtcFarEnd(const int16_t* samples, int numFrames, int } void AudioClient::processWebrtcNearEnd(int16_t* samples, int numFrames, int numChannels, int sampleRate) { - const webrtc::StreamConfig streamConfig = webrtc::StreamConfig(sampleRate, numChannels); const int numChunk = (int)streamConfig.num_frames(); @@ -1227,7 +1189,7 @@ void AudioClient::processWebrtcNearEnd(int16_t* samples, int numFrames, int numC } } -#endif // WEBRTC_ENABLED +#endif // WEBRTC_ENABLED void AudioClient::handleLocalEchoAndReverb(QByteArray& inputByteArray) { // If there is server echo, reverb will be applied to the recieved audio stream so no need to have it here. @@ -1254,7 +1216,8 @@ void AudioClient::handleLocalEchoAndReverb(QByteArray& inputByteArray) { // if required, create loopback resampler if (_inputFormat.sampleRate() != _outputFormat.sampleRate() && !_loopbackResampler) { - qCDebug(audioclient) << "Resampling from" << _inputFormat.sampleRate() << "to" << _outputFormat.sampleRate() << "for audio loopback."; + qCDebug(audioclient) << "Resampling from" << _inputFormat.sampleRate() << "to" << _outputFormat.sampleRate() + << "for audio loopback."; _loopbackResampler = new AudioSRC(_inputFormat.sampleRate(), _outputFormat.sampleRate(), OUTPUT_CHANNEL_COUNT); } @@ -1270,27 +1233,23 @@ void AudioClient::handleLocalEchoAndReverb(QByteArray& inputByteArray) { int16_t* inputSamples = reinterpret_cast(inputByteArray.data()); int16_t* loopbackSamples = reinterpret_cast(loopBackByteArray.data()); - int numLoopbackSamples = possibleResampling(_loopbackResampler, - inputSamples, loopbackSamples, - numInputSamples, maxLoopbackSamples, - _inputFormat.channelCount(), OUTPUT_CHANNEL_COUNT); + int numLoopbackSamples = possibleResampling(_loopbackResampler, inputSamples, loopbackSamples, numInputSamples, + maxLoopbackSamples, _inputFormat.channelCount(), OUTPUT_CHANNEL_COUNT); loopBackByteArray.resize(numLoopbackSamples * AudioConstants::SAMPLE_SIZE); // apply stereo reverb at the source, to the loopback audio if (!_shouldEchoLocally && hasReverb) { updateReverbOptions(); - _sourceReverb.render(loopbackSamples, loopbackSamples, numLoopbackSamples/2); + _sourceReverb.render(loopbackSamples, loopbackSamples, numLoopbackSamples / 2); } // if required, upmix or downmix to deviceChannelCount int deviceChannelCount = _outputFormat.channelCount(); if (deviceChannelCount == OUTPUT_CHANNEL_COUNT) { - _loopbackOutputDevice->write(loopBackByteArray); } else { - static QByteArray deviceByteArray; int numDeviceSamples = (numLoopbackSamples * deviceChannelCount) / OUTPUT_CHANNEL_COUNT; @@ -1310,7 +1269,6 @@ void AudioClient::handleLocalEchoAndReverb(QByteArray& inputByteArray) { void AudioClient::handleAudioInput(QByteArray& audioBuffer) { if (!_audioPaused) { - bool audioGateOpen = false; if (!_muted) { @@ -1364,8 +1322,7 @@ void AudioClient::handleAudioInput(QByteArray& audioBuffer) { } emitAudioPacket(encodedBuffer.data(), encodedBuffer.size(), _outgoingAvatarAudioSequenceNumber, _isStereoInput, - audioTransform, avatarBoundingBoxCorner, avatarBoundingBoxScale, - packetType, _selectedCodecName); + audioTransform, avatarBoundingBoxCorner, avatarBoundingBoxScale, packetType, _selectedCodecName); _stats.sentPacket(); } } @@ -1380,9 +1337,10 @@ void AudioClient::handleMicAudioInput() { #endif // input samples required to produce exactly NETWORK_FRAME_SAMPLES of output - const int inputSamplesRequired = (_inputToNetworkResampler ? - _inputToNetworkResampler->getMinInput(AudioConstants::NETWORK_FRAME_SAMPLES_PER_CHANNEL) : - AudioConstants::NETWORK_FRAME_SAMPLES_PER_CHANNEL) * _inputFormat.channelCount(); + const int inputSamplesRequired = + (_inputToNetworkResampler ? _inputToNetworkResampler->getMinInput(AudioConstants::NETWORK_FRAME_SAMPLES_PER_CHANNEL) + : AudioConstants::NETWORK_FRAME_SAMPLES_PER_CHANNEL) * + _inputFormat.channelCount(); const auto inputAudioSamples = std::unique_ptr(new int16_t[inputSamplesRequired]); QByteArray inputByteArray = _inputDevice->readAll(); @@ -1394,17 +1352,14 @@ void AudioClient::handleMicAudioInput() { float audioInputMsecsRead = inputByteArray.size() / (float)(_inputFormat.bytesForDuration(USECS_PER_MSEC)); _stats.updateInputMsRead(audioInputMsecsRead); - const int numNetworkBytes = _isStereoInput - ? AudioConstants::NETWORK_FRAME_BYTES_STEREO - : AudioConstants::NETWORK_FRAME_BYTES_PER_CHANNEL; - const int numNetworkSamples = _isStereoInput - ? AudioConstants::NETWORK_FRAME_SAMPLES_STEREO - : AudioConstants::NETWORK_FRAME_SAMPLES_PER_CHANNEL; + const int numNetworkBytes = + _isStereoInput ? AudioConstants::NETWORK_FRAME_BYTES_STEREO : AudioConstants::NETWORK_FRAME_BYTES_PER_CHANNEL; + const int numNetworkSamples = + _isStereoInput ? AudioConstants::NETWORK_FRAME_SAMPLES_STEREO : AudioConstants::NETWORK_FRAME_SAMPLES_PER_CHANNEL; static int16_t networkAudioSamples[AudioConstants::NETWORK_FRAME_SAMPLES_STEREO]; while (_inputRingBuffer.samplesAvailable() >= inputSamplesRequired) { - _inputRingBuffer.readSamples(inputAudioSamples.get(), inputSamplesRequired); // detect clipping on the raw input @@ -1414,7 +1369,7 @@ void AudioClient::handleMicAudioInput() { } else if (_timeSinceLastClip >= 0.0f) { _timeSinceLastClip += AudioConstants::NETWORK_FRAME_SECS; } - isClipping = (_timeSinceLastClip >= 0.0f) && (_timeSinceLastClip < 2.0f); // 2 second hold time + isClipping = (_timeSinceLastClip >= 0.0f) && (_timeSinceLastClip < 2.0f); // 2 second hold time #if defined(WEBRTC_ENABLED) if (_isAECEnabled) { @@ -1434,10 +1389,8 @@ void AudioClient::handleMicAudioInput() { emit inputLoudnessChanged(_lastSmoothedRawInputLoudness, isClipping); if (!_muted) { - possibleResampling(_inputToNetworkResampler, - inputAudioSamples.get(), networkAudioSamples, - inputSamplesRequired, numNetworkSamples, - _inputFormat.channelCount(), _desiredInputFormat.channelCount()); + possibleResampling(_inputToNetworkResampler, inputAudioSamples.get(), networkAudioSamples, inputSamplesRequired, + numNetworkSamples, _inputFormat.channelCount(), _desiredInputFormat.channelCount()); } int bytesInInputRingBuffer = _inputRingBuffer.samplesAvailable() * AudioConstants::SAMPLE_SIZE; float msecsInInputRingBuffer = bytesInInputRingBuffer / (float)(_inputFormat.bytesForDuration(USECS_PER_MSEC)); @@ -1449,11 +1402,10 @@ void AudioClient::handleMicAudioInput() { } void AudioClient::handleDummyAudioInput() { - const int numNetworkBytes = _isStereoInput - ? AudioConstants::NETWORK_FRAME_BYTES_STEREO - : AudioConstants::NETWORK_FRAME_BYTES_PER_CHANNEL; + const int numNetworkBytes = + _isStereoInput ? AudioConstants::NETWORK_FRAME_BYTES_STEREO : AudioConstants::NETWORK_FRAME_BYTES_PER_CHANNEL; - QByteArray audioBuffer(numNetworkBytes, 0); // silent + QByteArray audioBuffer(numNetworkBytes, 0); // silent handleAudioInput(audioBuffer); } @@ -1484,9 +1436,8 @@ void AudioClient::prepareLocalAudioInjectors(std::unique_ptr localAudioLoc int bufferCapacity = _localInjectorsStream.getSampleCapacity(); int maxOutputSamples = AudioConstants::NETWORK_FRAME_SAMPLES_PER_CHANNEL * AudioConstants::STEREO; if (_localToOutputResampler) { - maxOutputSamples = - _localToOutputResampler->getMaxOutput(AudioConstants::NETWORK_FRAME_SAMPLES_PER_CHANNEL) * - AudioConstants::STEREO; + maxOutputSamples = _localToOutputResampler->getMaxOutput(AudioConstants::NETWORK_FRAME_SAMPLES_PER_CHANNEL) * + AudioConstants::STEREO; } samplesNeeded = bufferCapacity - _localSamplesAvailable.load(std::memory_order_relaxed); @@ -1509,7 +1460,7 @@ void AudioClient::prepareLocalAudioInjectors(std::unique_ptr localAudioLoc if (_localToOutputResampler) { // resample to output sample rate int frames = _localToOutputResampler->render(_localMixBuffer, _localOutputMixBuffer, - AudioConstants::NETWORK_FRAME_SAMPLES_PER_CHANNEL); + AudioConstants::NETWORK_FRAME_SAMPLES_PER_CHANNEL); // write to local injectors' ring buffer samples = frames * AudioConstants::STEREO; @@ -1518,8 +1469,7 @@ void AudioClient::prepareLocalAudioInjectors(std::unique_ptr localAudioLoc } else { // write to local injectors' ring buffer samples = AudioConstants::NETWORK_FRAME_SAMPLES_STEREO; - _localInjectorsStream.writeSamples(_localMixBuffer, - AudioConstants::NETWORK_FRAME_SAMPLES_STEREO); + _localInjectorsStream.writeSamples(_localMixBuffer, AudioConstants::NETWORK_FRAME_SAMPLES_STEREO); } _localSamplesAvailable.fetch_add(samples, std::memory_order_release); @@ -1544,26 +1494,23 @@ bool AudioClient::mixLocalAudioInjectors(float* mixBuffer) { // the lock guarantees that injectorBuffer, if found, is invariant auto injectorBuffer = injector->getLocalBuffer(); if (injectorBuffer) { - auto options = injector->getOptions(); static const int HRTF_DATASET_INDEX = 1; - int numChannels = options.ambisonic ? AudioConstants::AMBISONIC : (options.stereo ? AudioConstants::STEREO : AudioConstants::MONO); + int numChannels = options.ambisonic ? AudioConstants::AMBISONIC + : (options.stereo ? AudioConstants::STEREO : AudioConstants::MONO); size_t bytesToRead = numChannels * AudioConstants::NETWORK_FRAME_BYTES_PER_CHANNEL; // get one frame from the injector memset(_localScratchBuffer, 0, bytesToRead); if (0 < injectorBuffer->readData((char*)_localScratchBuffer, bytesToRead)) { - bool isSystemSound = !options.positionSet && !options.ambisonic; float gain = options.volume * (isSystemSound ? _systemInjectorGain : _localInjectorGain); if (options.ambisonic) { - if (options.positionSet) { - // distance attenuation glm::vec3 relativePosition = options.position - _positionGetter(); float distance = glm::max(glm::length(relativePosition), EPSILON); @@ -1583,12 +1530,10 @@ bool AudioClient::mixLocalAudioInjectors(float* mixBuffer) { float qz = relativeOrientation.y; // spatialize into mixBuffer - injector->getLocalFOA().render(_localScratchBuffer, mixBuffer, HRTF_DATASET_INDEX, - qw, qx, qy, qz, gain, AudioConstants::NETWORK_FRAME_SAMPLES_PER_CHANNEL); + injector->getLocalFOA().render(_localScratchBuffer, mixBuffer, HRTF_DATASET_INDEX, qw, qx, qy, qz, gain, + AudioConstants::NETWORK_FRAME_SAMPLES_PER_CHANNEL); } else if (options.stereo) { - if (options.positionSet) { - // distance attenuation glm::vec3 relativePosition = options.position - _positionGetter(); float distance = glm::max(glm::length(relativePosition), EPSILON); @@ -1598,10 +1543,9 @@ bool AudioClient::mixLocalAudioInjectors(float* mixBuffer) { // direct mix into mixBuffer injector->getLocalHRTF().mixStereo(_localScratchBuffer, mixBuffer, gain, AudioConstants::NETWORK_FRAME_SAMPLES_PER_CHANNEL); - } else { // injector is mono + } else { // injector is mono if (options.positionSet) { - // distance attenuation glm::vec3 relativePosition = options.position - _positionGetter(); float distance = glm::max(glm::length(relativePosition), EPSILON); @@ -1610,10 +1554,9 @@ bool AudioClient::mixLocalAudioInjectors(float* mixBuffer) { float azimuth = azimuthForSource(relativePosition); // spatialize into mixBuffer - injector->getLocalHRTF().render(_localScratchBuffer, mixBuffer, HRTF_DATASET_INDEX, - azimuth, distance, gain, AudioConstants::NETWORK_FRAME_SAMPLES_PER_CHANNEL); + injector->getLocalHRTF().render(_localScratchBuffer, mixBuffer, HRTF_DATASET_INDEX, azimuth, distance, + gain, AudioConstants::NETWORK_FRAME_SAMPLES_PER_CHANNEL); } else { - // direct mix into mixBuffer injector->getLocalHRTF().mixMono(_localScratchBuffer, mixBuffer, gain, AudioConstants::NETWORK_FRAME_SAMPLES_PER_CHANNEL); @@ -1621,14 +1564,12 @@ bool AudioClient::mixLocalAudioInjectors(float* mixBuffer) { } } else { - //qCDebug(audioclient) << "injector has no more data, marking finished for removal"; injector->finishLocalInjection(); injectorsToRemove.append(injector); } } else { - //qCDebug(audioclient) << "injector has no local buffer, marking as finished for removal"; injector->finishLocalInjection(); injectorsToRemove.append(injector); @@ -1647,7 +1588,6 @@ bool AudioClient::mixLocalAudioInjectors(float* mixBuffer) { } void AudioClient::processReceivedSamples(const QByteArray& decodedBuffer, QByteArray& outputBuffer) { - const int16_t* decodedSamples = reinterpret_cast(decodedBuffer.data()); assert(decodedBuffer.size() == AudioConstants::NETWORK_FRAME_BYTES_STEREO); @@ -1736,7 +1676,7 @@ void AudioClient::setAcousticEchoCancellation(bool enable, bool emitSignal) { bool AudioClient::setIsStereoInput(bool isStereoInput) { bool stereoInputChanged = false; - if (isStereoInput != _isStereoInput && _inputDeviceInfo.supportedChannelCounts().contains(2)) { + if (isStereoInput != _isStereoInput && _inputDeviceInfo.getDevice().supportedChannelCounts().contains(2)) { _isStereoInput = isStereoInput; stereoInputChanged = true; @@ -1751,12 +1691,13 @@ bool AudioClient::setIsStereoInput(bool isStereoInput) { if (_encoder) { _codec->releaseEncoder(_encoder); } - _encoder = _codec->createEncoder(AudioConstants::SAMPLE_RATE, _isStereoInput ? AudioConstants::STEREO : AudioConstants::MONO); + _encoder = _codec->createEncoder(AudioConstants::SAMPLE_RATE, + _isStereoInput ? AudioConstants::STEREO : AudioConstants::MONO); } qCDebug(audioclient) << "Reset Codec:" << _selectedCodecName << "isStereoInput:" << _isStereoInput; // restart the input device - switchInputToAudioDevice(_inputDeviceInfo); + switchInputToAudioDevice(_inputDeviceInfo.getDevice()); emit isStereoInputChanged(_isStereoInput); } @@ -1791,10 +1732,9 @@ int AudioClient::getNumLocalInjectors() { Lock lock(_injectorsMutex); return _activeLocalAudioInjectors.size(); } - void AudioClient::outputFormatChanged() { _outputFrameSize = (AudioConstants::NETWORK_FRAME_SAMPLES_PER_CHANNEL * OUTPUT_CHANNEL_COUNT * _outputFormat.sampleRate()) / - _desiredOutputFormat.sampleRate(); + _desiredOutputFormat.sampleRate(); _receivedAudioStream.outputFormatChanged(_outputFormat.sampleRate(), OUTPUT_CHANNEL_COUNT); } @@ -1808,7 +1748,7 @@ bool AudioClient::switchInputToAudioDevice(const QAudioDeviceInfo inputDeviceInf Lock lock(_deviceMutex); #if defined(Q_OS_ANDROID) - _shouldRestartInputSetup = false; // avoid a double call to _audioInput->start() from audioInputStateChanged + _shouldRestartInputSetup = false; // avoid a double call to _audioInput->start() from audioInputStateChanged #endif // cleanup any previously initialized device @@ -1823,7 +1763,7 @@ bool AudioClient::switchInputToAudioDevice(const QAudioDeviceInfo inputDeviceInf _audioInput = NULL; _numInputCallbackBytes = 0; - _inputDeviceInfo = QAudioDeviceInfo(); + _inputDeviceInfo = HifiAudioDeviceInfo(); //QAudioDeviceInfo(); } if (_dummyAudioInput) { @@ -1856,23 +1796,24 @@ bool AudioClient::switchInputToAudioDevice(const QAudioDeviceInfo inputDeviceInf if (!inputDeviceInfo.isNull()) { qCDebug(audioclient) << "The audio input device " << inputDeviceInfo.deviceName() << "is available."; - _inputDeviceInfo = inputDeviceInfo; - emit deviceChanged(QAudio::AudioInput, inputDeviceInfo); + _inputDeviceInfo = HifiAudioDeviceInfo(); + _inputDeviceInfo.setDevice(inputDeviceInfo); + emit deviceChanged(QAudio::AudioInput, _inputDeviceInfo); if (adjustedFormatForAudioDevice(inputDeviceInfo, _desiredInputFormat, _inputFormat)) { qCDebug(audioclient) << "The format to be used for audio input is" << _inputFormat; // we've got the best we can get for input // if required, setup a resampler for this input to our desired network format - if (_inputFormat != _desiredInputFormat - && _inputFormat.sampleRate() != _desiredInputFormat.sampleRate()) { + if (_inputFormat != _desiredInputFormat && _inputFormat.sampleRate() != _desiredInputFormat.sampleRate()) { qCDebug(audioclient) << "Attemping to create a resampler for input format to network format."; assert(_inputFormat.sampleSize() == 16); assert(_desiredInputFormat.sampleSize() == 16); int channelCount = (_inputFormat.channelCount() == 2 && _desiredInputFormat.channelCount() == 2) ? 2 : 1; - _inputToNetworkResampler = new AudioSRC(_inputFormat.sampleRate(), _desiredInputFormat.sampleRate(), channelCount); + _inputToNetworkResampler = + new AudioSRC(_inputFormat.sampleRate(), _desiredInputFormat.sampleRate(), channelCount); } else { qCDebug(audioclient) << "No resampling required for audio input to match desired network format."; @@ -1906,7 +1847,7 @@ bool AudioClient::switchInputToAudioDevice(const QAudioDeviceInfo inputDeviceInf connect(_inputDevice, SIGNAL(readyRead()), this, SLOT(handleMicAudioInput())); supportedFormat = true; } else { - qCDebug(audioclient) << "Error starting audio input -" << _audioInput->error(); + qCDebug(audioclient) << "Error starting audio input -" << _audioInput->error(); _audioInput->deleteLater(); _audioInput = NULL; } @@ -1919,7 +1860,7 @@ bool AudioClient::switchInputToAudioDevice(const QAudioDeviceInfo inputDeviceInf // This enables clients without a mic to still receive an audio stream from the mixer. if (!_audioInput) { qCDebug(audioclient) << "Audio input device is not available, using dummy input."; - _inputDeviceInfo = QAudioDeviceInfo(); + _inputDeviceInfo.setDevice(QAudioDeviceInfo()); emit deviceChanged(QAudio::AudioInput, _inputDeviceInfo); _inputFormat = _desiredInputFormat; @@ -1976,7 +1917,7 @@ void AudioClient::checkInputTimeout() { void AudioClient::setHeadsetPluggedIn(bool pluggedIn) { #if defined(Q_OS_ANDROID) if (pluggedIn == !_isHeadsetPluggedIn && !_inputDeviceInfo.isNull()) { - QAndroidJniObject brand = QAndroidJniObject::getStaticObjectField("android/os/Build", "BRAND"); + QAndroidJniObject brand = QAndroidJniObject::getStaticObjectField("android/os/Build", "BRAND"); // some samsung phones needs more time to shutdown the previous input device if (brand.toString().contains("samsung", Qt::CaseInsensitive)) { switchInputToAudioDevice(QAudioDeviceInfo(), true); @@ -2014,8 +1955,8 @@ void AudioClient::outputNotify() { int newOutputBufferSizeFrames = setOutputBufferSize(oldOutputBufferSizeFrames + 1, false); if (newOutputBufferSizeFrames > oldOutputBufferSizeFrames) { - qCDebug(audioclient, - "Starve threshold surpassed (%d starves in %d ms)", _outputStarveDetectionCount, dt); + qCDebug(audioclient, "Starve threshold surpassed (%d starves in %d ms)", _outputStarveDetectionCount, + dt); } _outputStarveDetectionStartTimeMsec = now; @@ -2029,7 +1970,8 @@ void AudioClient::outputNotify() { bool AudioClient::switchOutputToAudioDevice(const QAudioDeviceInfo outputDeviceInfo, bool isShutdownRequest) { Q_ASSERT_X(QThread::currentThread() == thread(), Q_FUNC_INFO, "Function invoked on wrong thread"); - qCDebug(audioclient) << "AudioClient::switchOutputToAudioDevice() outputDeviceInfo: [" << outputDeviceInfo.deviceName() << "]"; + qCDebug(audioclient) << "AudioClient::switchOutputToAudioDevice() outputDeviceInfo: [" << outputDeviceInfo.deviceName() + << "]"; bool supportedFormat = false; // NOTE: device start() uses the Qt internal device list @@ -2062,7 +2004,9 @@ bool AudioClient::switchOutputToAudioDevice(const QAudioDeviceInfo outputDeviceI delete[] _localOutputMixBuffer; _localOutputMixBuffer = NULL; - _outputDeviceInfo = QAudioDeviceInfo(); + //ctor call needed? + _outputDeviceInfo = HifiAudioDeviceInfo(); + _outputDeviceInfo.setDevice(QAudioDeviceInfo()); } // cleanup any resamplers @@ -2088,23 +2032,24 @@ bool AudioClient::switchOutputToAudioDevice(const QAudioDeviceInfo outputDeviceI if (!outputDeviceInfo.isNull()) { qCDebug(audioclient) << "The audio output device " << outputDeviceInfo.deviceName() << "is available."; - _outputDeviceInfo = outputDeviceInfo; - emit deviceChanged(QAudio::AudioOutput, outputDeviceInfo); + _outputDeviceInfo.setDevice(outputDeviceInfo); + emit deviceChanged(QAudio::AudioOutput, _outputDeviceInfo); if (adjustedFormatForAudioDevice(outputDeviceInfo, _desiredOutputFormat, _outputFormat)) { qCDebug(audioclient) << "The format to be used for audio output is" << _outputFormat; // we've got the best we can get for input // if required, setup a resampler for this input to our desired network format - if (_desiredOutputFormat != _outputFormat - && _desiredOutputFormat.sampleRate() != _outputFormat.sampleRate()) { + if (_desiredOutputFormat != _outputFormat && _desiredOutputFormat.sampleRate() != _outputFormat.sampleRate()) { qCDebug(audioclient) << "Attemping to create a resampler for network format to output format."; assert(_desiredOutputFormat.sampleSize() == 16); assert(_outputFormat.sampleSize() == 16); - _networkToOutputResampler = new AudioSRC(_desiredOutputFormat.sampleRate(), _outputFormat.sampleRate(), OUTPUT_CHANNEL_COUNT); - _localToOutputResampler = new AudioSRC(_desiredOutputFormat.sampleRate(), _outputFormat.sampleRate(), OUTPUT_CHANNEL_COUNT); + _networkToOutputResampler = + new AudioSRC(_desiredOutputFormat.sampleRate(), _outputFormat.sampleRate(), OUTPUT_CHANNEL_COUNT); + _localToOutputResampler = + new AudioSRC(_desiredOutputFormat.sampleRate(), _outputFormat.sampleRate(), OUTPUT_CHANNEL_COUNT); } else { qCDebug(audioclient) << "No resampling required for network output to match actual output format."; @@ -2116,7 +2061,9 @@ bool AudioClient::switchOutputToAudioDevice(const QAudioDeviceInfo outputDeviceI _audioOutput = new QAudioOutput(outputDeviceInfo, _outputFormat, this); int deviceChannelCount = _outputFormat.channelCount(); - int frameSize = (AudioConstants::NETWORK_FRAME_SAMPLES_PER_CHANNEL * deviceChannelCount * _outputFormat.sampleRate()) / _desiredOutputFormat.sampleRate(); + int frameSize = + (AudioConstants::NETWORK_FRAME_SAMPLES_PER_CHANNEL * deviceChannelCount * _outputFormat.sampleRate()) / + _desiredOutputFormat.sampleRate(); int requestedSize = _sessionOutputBufferSizeFrames * frameSize * AudioConstants::SAMPLE_SIZE; _audioOutput->setBufferSize(requestedSize); @@ -2137,7 +2084,9 @@ bool AudioClient::switchOutputToAudioDevice(const QAudioDeviceInfo outputDeviceI _outputScratchBuffer = new int16_t[_outputPeriod]; // size local output mix buffer based on resampled network frame size - int networkPeriod = _localToOutputResampler ? _localToOutputResampler->getMaxOutput(AudioConstants::NETWORK_FRAME_SAMPLES_STEREO) : AudioConstants::NETWORK_FRAME_SAMPLES_STEREO; + int networkPeriod = _localToOutputResampler + ? _localToOutputResampler->getMaxOutput(AudioConstants::NETWORK_FRAME_SAMPLES_STEREO) + : AudioConstants::NETWORK_FRAME_SAMPLES_STEREO; _localOutputMixBuffer = new float[networkPeriod]; // local period should be at least twice the output period, @@ -2180,7 +2129,8 @@ int AudioClient::setOutputBufferSize(int numFrames, bool persist) { qCDebug(audioclient) << __FUNCTION__ << "numFrames:" << numFrames << "persist:" << persist; numFrames = std::min(std::max(numFrames, MIN_BUFFER_FRAMES), MAX_BUFFER_FRAMES); - qCDebug(audioclient) << __FUNCTION__ << "clamped numFrames:" << numFrames << "_sessionOutputBufferSizeFrames:" << _sessionOutputBufferSizeFrames; + qCDebug(audioclient) << __FUNCTION__ << "clamped numFrames:" << numFrames + << "_sessionOutputBufferSizeFrames:" << _sessionOutputBufferSizeFrames; if (numFrames != _sessionOutputBufferSizeFrames) { qCInfo(audioclient, "Audio output buffer set to %d frames", numFrames); @@ -2211,10 +2161,10 @@ const float AudioClient::CALLBACK_ACCELERATOR_RATIO = 2.0f; #endif int AudioClient::calculateNumberOfInputCallbackBytes(const QAudioFormat& format) const { - int numInputCallbackBytes = (int)(((AudioConstants::NETWORK_FRAME_BYTES_PER_CHANNEL - * format.channelCount() - * ((float) format.sampleRate() / AudioConstants::SAMPLE_RATE)) - / CALLBACK_ACCELERATOR_RATIO) + 0.5f); + int numInputCallbackBytes = (int)(((AudioConstants::NETWORK_FRAME_BYTES_PER_CHANNEL * format.channelCount() * + ((float)format.sampleRate() / AudioConstants::SAMPLE_RATE)) / + CALLBACK_ACCELERATOR_RATIO) + + 0.5f); return numInputCallbackBytes; } @@ -2236,10 +2186,9 @@ float AudioClient::azimuthForSource(const glm::vec3& relativePosition) { float rotatedSourcePositionLength2 = glm::length2(rotatedSourcePosition); if (rotatedSourcePositionLength2 > SOURCE_DISTANCE_THRESHOLD) { - // produce an oriented angle about the y-axis glm::vec3 direction = rotatedSourcePosition * (1.0f / fastSqrtf(rotatedSourcePositionLength2)); - float angle = fastAcosf(glm::clamp(-direction.z, -1.0f, 1.0f)); // UNIT_NEG_Z is "forward" + float angle = fastAcosf(glm::clamp(-direction.z, -1.0f, 1.0f)); // UNIT_NEG_Z is "forward" return (direction.x < 0.0f) ? -angle : angle; } else { @@ -2249,7 +2198,6 @@ float AudioClient::azimuthForSource(const glm::vec3& relativePosition) { } float AudioClient::gainForSource(float distance, float volume) { - // attenuation = -6dB * log2(distance) // reference attenuation of 0dB at distance = ATTN_DISTANCE_REF float d = (1.0f / ATTN_DISTANCE_REF) * std::max(distance, HRTF_NEARFIELD_MIN); @@ -2259,8 +2207,7 @@ float AudioClient::gainForSource(float distance, float volume) { return gain; } -qint64 AudioClient::AudioOutputIODevice::readData(char * data, qint64 maxSize) { - +qint64 AudioClient::AudioOutputIODevice::readData(char* data, qint64 maxSize) { // lock-free wait for initialization to avoid races if (!_audio->_audioOutputInitialized.load(std::memory_order_acquire)) { memset(data, 0, maxSize); @@ -2279,7 +2226,8 @@ qint64 AudioClient::AudioOutputIODevice::readData(char * data, qint64 maxSize) { int samplesRequested = maxSamplesRequested; int networkSamplesPopped; if ((networkSamplesPopped = _receivedAudioStream.popSamples(samplesRequested, false)) > 0) { - qCDebug(audiostream, "Read %d samples from buffer (%d available, %d requested)", networkSamplesPopped, _receivedAudioStream.getSamplesAvailable(), samplesRequested); + qCDebug(audiostream, "Read %d samples from buffer (%d available, %d requested)", networkSamplesPopped, + _receivedAudioStream.getSamplesAvailable(), samplesRequested); AudioRingBuffer::ConstIterator lastPopOutput = _receivedAudioStream.getLastPopOutput(); lastPopOutput.readSamples(scratchBuffer, networkSamplesPopped); for (int i = 0; i < networkSamplesPopped; i++) { @@ -2311,14 +2259,13 @@ qint64 AudioClient::AudioOutputIODevice::readData(char * data, qint64 maxSize) { samplesRequested = std::min(samplesRequested, samplesAvailable); if ((injectorSamplesPopped = _localInjectorsStream.appendSamples(mixBuffer, samplesRequested, append)) > 0) { _audio->_localSamplesAvailable.fetch_sub(injectorSamplesPopped, std::memory_order_release); - qCDebug(audiostream, "Read %d samples from injectors (%d available, %d requested)", injectorSamplesPopped, _localInjectorsStream.samplesAvailable(), samplesRequested); + qCDebug(audiostream, "Read %d samples from injectors (%d available, %d requested)", injectorSamplesPopped, + _localInjectorsStream.samplesAvailable(), samplesRequested); } } // prepare injectors for the next callback - QtConcurrent::run(QThreadPool::globalInstance(), [this] { - _audio->prepareLocalAudioInjectors(); - }); + QtConcurrent::run(QThreadPool::globalInstance(), [this] { _audio->prepareLocalAudioInjectors(); }); int samplesPopped = std::max(networkSamplesPopped, injectorSamplesPopped); if (samplesPopped == 0) { @@ -2398,7 +2345,6 @@ void AudioClient::loadSettings() { for (auto& plugin : codecPlugins) { qCDebug(audioclient) << "Codec available:" << plugin->getName(); } - } void AudioClient::saveSettings() { @@ -2411,9 +2357,9 @@ void AudioClient::setAvatarBoundingBoxParameters(glm::vec3 corner, glm::vec3 sca avatarBoundingBoxScale = scale; } - void AudioClient::startThread() { - moveToNewNamedThread(this, "Audio Thread", [this] { start(); }, QThread::TimeCriticalPriority); + moveToNewNamedThread( + this, "Audio Thread", [this] { start(); }, QThread::TimeCriticalPriority); } void AudioClient::setInputVolume(float volume, bool emitSignal) { diff --git a/libraries/audio-client/src/AudioClient.h b/libraries/audio-client/src/AudioClient.h index ab12393ebf..fcea31a21b 100644 --- a/libraries/audio-client/src/AudioClient.h +++ b/libraries/audio-client/src/AudioClient.h @@ -53,18 +53,19 @@ #include "AudioIOStats.h" #include "AudioFileWav.h" +#include "HifiAudioDeviceInfo.h" #ifdef _WIN32 -#pragma warning( push ) -#pragma warning( disable : 4273 ) -#pragma warning( disable : 4305 ) +#pragma warning(push) +#pragma warning(disable : 4273) +#pragma warning(disable : 4305) #endif #ifdef _WIN32 -#pragma warning( pop ) +#pragma warning(pop) #endif -#if defined (Q_OS_ANDROID) +#if defined(Q_OS_ANDROID) #define VOICE_RECOGNITION "voicerecognition" #define VOICE_COMMUNICATION "voicecommunication" @@ -79,11 +80,14 @@ class QIODevice; class Transform; class NLPacket; + + class AudioClient : public AbstractAudioInterface, public Dependency { Q_OBJECT SINGLETON_DEPENDENCY using LocalInjectorsStream = AudioMixRingBuffer; + public: static const int MIN_BUFFER_FRAMES; static const int MAX_BUFFER_FRAMES; @@ -96,15 +100,21 @@ public: class AudioOutputIODevice : public QIODevice { public: - AudioOutputIODevice(LocalInjectorsStream& localInjectorsStream, MixedProcessedAudioStream& receivedAudioStream, - AudioClient* audio) : - _localInjectorsStream(localInjectorsStream), _receivedAudioStream(receivedAudioStream), - _audio(audio), _unfulfilledReads(0) {} + AudioOutputIODevice(LocalInjectorsStream& localInjectorsStream, + MixedProcessedAudioStream& receivedAudioStream, + AudioClient* audio) : + _localInjectorsStream(localInjectorsStream), + _receivedAudioStream(receivedAudioStream), _audio(audio), _unfulfilledReads(0) {} void start() { open(QIODevice::ReadOnly | QIODevice::Unbuffered); } - qint64 readData(char * data, qint64 maxSize) override; - qint64 writeData(const char * data, qint64 maxSize) override { return 0; } - int getRecentUnfulfilledReads() { int unfulfilledReads = _unfulfilledReads; _unfulfilledReads = 0; return unfulfilledReads; } + qint64 readData(char* data, qint64 maxSize) override; + qint64 writeData(const char* data, qint64 maxSize) override { return 0; } + int getRecentUnfulfilledReads() { + int unfulfilledReads = _unfulfilledReads; + _unfulfilledReads = 0; + return unfulfilledReads; + } + private: LocalInjectorsStream& _localInjectorsStream; MixedProcessedAudioStream& _receivedAudioStream; @@ -152,11 +162,11 @@ public: void setIsPlayingBackRecording(bool isPlayingBackRecording) { _isPlayingBackRecording = isPlayingBackRecording; } Q_INVOKABLE void setAvatarBoundingBoxParameters(glm::vec3 corner, glm::vec3 scale); - + bool outputLocalInjector(const AudioInjectorPointer& injector) override; - QAudioDeviceInfo getActiveAudioDevice(QAudio::Mode mode) const; - QList getAudioDevices(QAudio::Mode mode) const; + HifiAudioDeviceInfo getActiveAudioDevice(QAudio::Mode mode) const; + QList getAudioDevices(QAudio::Mode mode) const; void enablePeakValues(bool enable) { _enablePeakValues = enable; } bool peakValuesAvailable() const; @@ -269,10 +279,10 @@ signals: void noiseGateOpened(); void noiseGateClosed(); - void changeDevice(const QAudioDeviceInfo& outputDeviceInfo); + void changeDevice(const HifiAudioDeviceInfo& outputDeviceInfo); - void deviceChanged(QAudio::Mode mode, const QAudioDeviceInfo& device); - void devicesChanged(QAudio::Mode mode, const QList& devices); + void deviceChanged(QAudio::Mode mode, const HifiAudioDeviceInfo& device); + void devicesChanged(QAudio::Mode mode, const QList& devices); void peakValueListChanged(const QList peakValueList); void receivedFirstPacket(); @@ -345,7 +355,7 @@ private: QIODevice* _inputDevice; int _numInputCallbackBytes; QAudioOutput* _audioOutput; - std::atomic _audioOutputInitialized { false }; + std::atomic _audioOutputInitialized{ false }; QAudioFormat _desiredOutputFormat; QAudioFormat _outputFormat; int _outputFrameSize; @@ -356,11 +366,11 @@ private: LocalInjectorsStream _localInjectorsStream; // In order to use _localInjectorsStream as a lock-free pipe, // use it with a single producer/consumer, and track available samples and injectors - std::atomic _localSamplesAvailable { 0 }; - std::atomic _localInjectorsAvailable { false }; + std::atomic _localSamplesAvailable{ 0 }; + std::atomic _localInjectorsAvailable{ false }; MixedProcessedAudioStream _receivedAudioStream; bool _isStereoInput; - std::atomic _enablePeakValues { false }; + std::atomic _enablePeakValues{ false }; quint64 _outputStarveDetectionStartTimeMsec; int _outputStarveDetectionCount; @@ -371,9 +381,9 @@ private: StDev _stdev; QElapsedTimer _timeSinceLastReceived; - float _lastRawInputLoudness; // before mute/gate + float _lastRawInputLoudness; // before mute/gate float _lastSmoothedRawInputLoudness; - float _lastInputLoudness; // after mute/gate + float _lastInputLoudness; // after mute/gate float _timeSinceLastClip; int _totalInputAudioSamples; @@ -388,9 +398,9 @@ private: AudioEffectOptions _scriptReverbOptions; AudioEffectOptions _zoneReverbOptions; AudioEffectOptions* _reverbOptions; - AudioReverb _sourceReverb { AudioConstants::SAMPLE_RATE }; - AudioReverb _listenerReverb { AudioConstants::SAMPLE_RATE }; - AudioReverb _localReverb { AudioConstants::SAMPLE_RATE }; + AudioReverb _sourceReverb{ AudioConstants::SAMPLE_RATE }; + AudioReverb _listenerReverb{ AudioConstants::SAMPLE_RATE }; + AudioReverb _localReverb{ AudioConstants::SAMPLE_RATE }; // possible streams needed for resample AudioSRC* _inputToNetworkResampler; @@ -402,21 +412,21 @@ private: int16_t _networkScratchBuffer[AudioConstants::NETWORK_FRAME_SAMPLES_AMBISONIC]; // for output audio (used by this thread) - int _outputPeriod { 0 }; - float* _outputMixBuffer { NULL }; - int16_t* _outputScratchBuffer { NULL }; - std::atomic _outputGain { 1.0f }; - float _lastOutputGain { 1.0f }; + int _outputPeriod{ 0 }; + float* _outputMixBuffer{ NULL }; + int16_t* _outputScratchBuffer{ NULL }; + std::atomic _outputGain{ 1.0f }; + float _lastOutputGain{ 1.0f }; // for local audio (used by audio injectors thread) - std::atomic _localInjectorGain { 1.0f }; - std::atomic _systemInjectorGain { 1.0f }; + std::atomic _localInjectorGain{ 1.0f }; + std::atomic _systemInjectorGain{ 1.0f }; float _localMixBuffer[AudioConstants::NETWORK_FRAME_SAMPLES_STEREO]; int16_t _localScratchBuffer[AudioConstants::NETWORK_FRAME_SAMPLES_AMBISONIC]; - float* _localOutputMixBuffer { NULL }; + float* _localOutputMixBuffer{ NULL }; Mutex _localAudioMutex; AudioLimiter _audioLimiter; - + // Adds Reverb void configureReverb(); void updateReverbOptions(); @@ -427,10 +437,10 @@ private: static const int WEBRTC_CHANNELS_MAX = 2; static const int WEBRTC_FRAMES_MAX = webrtc::AudioProcessing::kChunkSizeMs * WEBRTC_SAMPLE_RATE_MAX / 1000; - webrtc::AudioProcessing* _apm { nullptr }; + webrtc::AudioProcessing* _apm{ nullptr }; - int16_t _fifoFarEnd[WEBRTC_CHANNELS_MAX * WEBRTC_FRAMES_MAX] {}; - int _numFifoFarEnd = 0; // numFrames saved in fifo + int16_t _fifoFarEnd[WEBRTC_CHANNELS_MAX * WEBRTC_FRAMES_MAX]{}; + int _numFifoFarEnd = 0; // numFrames saved in fifo void configureWebrtc(); void processWebrtcFarEnd(const int16_t* samples, int numFrames, int numChannels, int sampleRate); @@ -450,8 +460,8 @@ private: AudioIOStats _stats; - AudioGate* _audioGate { nullptr }; - bool _audioGateOpen { true }; + AudioGate* _audioGate{ nullptr }; + bool _audioGateOpen{ true }; AudioPositionGetter _positionGetter; AudioOrientationGetter _orientationGetter; @@ -459,24 +469,30 @@ private: glm::vec3 avatarBoundingBoxCorner; glm::vec3 avatarBoundingBoxScale; - QAudioDeviceInfo _inputDeviceInfo; - QAudioDeviceInfo _outputDeviceInfo; + HifiAudioDeviceInfo _inputDeviceInfo; + HifiAudioDeviceInfo _outputDeviceInfo; - QList _inputDevices; - QList _outputDevices; + QList _inputDevices; + QList _outputDevices; + + //QAudioDeviceInfo _inputDeviceInfo; + // QAudioDeviceInfo _outputDeviceInfo; + + // QList _inputDevices; + /// QList _outputDevices; AudioFileWav _audioFileWav; - bool _hasReceivedFirstPacket { false }; + bool _hasReceivedFirstPacket{ false }; QVector _activeLocalAudioInjectors; - bool _isPlayingBackRecording { false }; - bool _audioPaused { false }; + bool _isPlayingBackRecording{ false }; + bool _audioPaused{ false }; CodecPluginPointer _codec; QString _selectedCodecName; - Encoder* _encoder { nullptr }; // for outbound mic stream + Encoder* _encoder{ nullptr }; // for outbound mic stream RateCounter<> _silentOutbound; RateCounter<> _audioOutbound; @@ -484,18 +500,17 @@ private: RateCounter<> _audioInbound; #if defined(Q_OS_ANDROID) - bool _shouldRestartInputSetup { true }; // Should we restart the input device because of an unintended stop? + bool _shouldRestartInputSetup{ true }; // Should we restart the input device because of an unintended stop? #endif AudioSolo _solo; - - Mutex _checkDevicesMutex; - QTimer* _checkDevicesTimer { nullptr }; - Mutex _checkPeakValuesMutex; - QTimer* _checkPeakValuesTimer { nullptr }; - bool _isRecording { false }; + Mutex _checkDevicesMutex; + QTimer* _checkDevicesTimer{ nullptr }; + Mutex _checkPeakValuesMutex; + QTimer* _checkPeakValuesTimer{ nullptr }; + + bool _isRecording{ false }; }; - -#endif // hifi_AudioClient_h +#endif // hifi_AudioClient_h diff --git a/libraries/ui/src/ui/OffscreenQmlSurface.cpp b/libraries/ui/src/ui/OffscreenQmlSurface.cpp index 8beca598a2..5dfdce0c63 100644 --- a/libraries/ui/src/ui/OffscreenQmlSurface.cpp +++ b/libraries/ui/src/ui/OffscreenQmlSurface.cpp @@ -356,7 +356,7 @@ void OffscreenQmlSurface::onRootCreated() { getSurfaceContext()->setContextProperty("offscreenWindow", QVariant::fromValue(getWindow())); // Connect with the audio client and listen for audio device changes - connect(DependencyManager::get().data(), &AudioClient::deviceChanged, this, [this](QAudio::Mode mode, const QAudioDeviceInfo& device) { + connect(DependencyManager::get().data(), &AudioClient::deviceChanged, this, [this](QAudio::Mode mode, const HifiAudioDeviceInfo& device) { if (mode == QAudio::Mode::AudioOutput) { QMetaObject::invokeMethod(this, "changeAudioOutputDevice", Qt::QueuedConnection, Q_ARG(QString, device.deviceName())); }