diff --git a/assignment-client/src/audio/AudioMixerSlave.cpp b/assignment-client/src/audio/AudioMixerSlave.cpp index ed63bbc298..a131e266d2 100644 --- a/assignment-client/src/audio/AudioMixerSlave.cpp +++ b/assignment-client/src/audio/AudioMixerSlave.cpp @@ -558,7 +558,7 @@ float computeAzimuth(const AvatarAudioStream& listeningNodeStream, const Positio // produce an oriented angle about the y-axis glm::vec3 direction = rotatedSourcePosition * (1.0f / fastSqrtf(rotatedSourcePositionLength2)); - float angle = fastAcosf(glm::clamp(-direction.z, -1.0f, 1.0f)); // UNIT_NEG_Z is "forward" + float angle = fastAcosf(glm::clamp(-direction.z, -1.0f, 1.0f)); // UNIT_NEG_Z is "forward" return (direction.x < 0.0f) ? -angle : angle; } else { diff --git a/interface/resources/qml/hifi/audio/Audio.qml b/interface/resources/qml/hifi/audio/Audio.qml index 93742e39a5..a819e032eb 100644 --- a/interface/resources/qml/hifi/audio/Audio.qml +++ b/interface/resources/qml/hifi/audio/Audio.qml @@ -36,6 +36,24 @@ Rectangle { return (root.parent !== null) && root.parent.objectName == "loader"; } + property bool showPeaks: true; + function enablePeakValues() { + Audio.devices.input.peakValuesEnabled = true; + Audio.devices.input.peakValuesEnabledChanged.connect(function(enabled) { + if (!enabled && root.showPeaks) { + Audio.devices.input.peakValuesEnabled = true; + } + }); + } + function disablePeakValues() { + root.showPeaks = false; + Audio.devices.input.peakValuesEnabled = false; + } + + Component.onCompleted: enablePeakValues(); + Component.onDestruction: disablePeakValues(); + onVisibleChanged: visible ? enablePeakValues() : disablePeakValues(); + Column { y: 16; // padding does not work spacing: 16; @@ -133,12 +151,13 @@ Rectangle { onClicked: Audio.setInputDevice(info); } - InputLevel { - id: level; + InputPeak { + id: inputPeak; + visible: Audio.devices.input.peakValuesAvailable; + peak: model.peak; anchors.verticalCenter: parent.verticalCenter anchors.right: parent.right anchors.rightMargin: 30 - visible: selected; } } } diff --git a/interface/resources/qml/hifi/audio/InputLevel.qml b/interface/resources/qml/hifi/audio/InputPeak.qml similarity index 95% rename from interface/resources/qml/hifi/audio/InputLevel.qml rename to interface/resources/qml/hifi/audio/InputPeak.qml index 58ce3a1ec7..4ff49091b1 100644 --- a/interface/resources/qml/hifi/audio/InputLevel.qml +++ b/interface/resources/qml/hifi/audio/InputPeak.qml @@ -1,5 +1,5 @@ // -// InputLevel.qml +// InputPeak.qml // qml/hifi/audio // // Created by Zach Pomerantz on 6/20/2017 @@ -15,7 +15,7 @@ import QtQuick.Layouts 1.3 import QtGraphicalEffects 1.0 Rectangle { - readonly property var level: Audio.inputLevel; + property var peak; width: 70; height: 8; @@ -65,7 +65,7 @@ Rectangle { Rectangle { // mask id: mask; - width: parent.width * level; + width: parent.width * peak; radius: 5; anchors { bottom: parent.bottom; diff --git a/interface/src/scripting/AudioDevices.cpp b/interface/src/scripting/AudioDevices.cpp index f2e6dbf4d7..e5cc43f9df 100644 --- a/interface/src/scripting/AudioDevices.cpp +++ b/interface/src/scripting/AudioDevices.cpp @@ -37,6 +37,20 @@ Setting::Handle<QString>& getSetting(bool contextIsHMD, QAudio::Mode mode) { } } +enum AudioDeviceRole { + DisplayRole = Qt::DisplayRole, + CheckStateRole = Qt::CheckStateRole, + PeakRole = Qt::UserRole, + InfoRole = Qt::UserRole + 1 +}; + +QHash<int, QByteArray> AudioDeviceList::_roles { + { DisplayRole, "display" }, + { CheckStateRole, "selected" }, + { PeakRole, "peak" }, + { InfoRole, "info" } +}; + static QString getTargetDevice(bool hmd, QAudio::Mode mode) { QString deviceName; auto& setting = getSetting(hmd, mode); @@ -52,29 +66,36 @@ static QString getTargetDevice(bool hmd, QAudio::Mode mode) { return deviceName; } -QHash<int, QByteArray> AudioDeviceList::_roles { - { Qt::DisplayRole, "display" }, - { Qt::CheckStateRole, "selected" }, - { Qt::UserRole, "info" } -}; Qt::ItemFlags AudioDeviceList::_flags { Qt::ItemIsSelectable | Qt::ItemIsEnabled }; QVariant AudioDeviceList::data(const QModelIndex& index, int role) const { - if (!index.isValid() || index.row() >= _devices.size()) { + if (!index.isValid() || index.row() >= rowCount()) { return QVariant(); } - if (role == Qt::DisplayRole) { - return _devices.at(index.row()).display; - } else if (role == Qt::CheckStateRole) { - return _devices.at(index.row()).selected; - } else if (role == Qt::UserRole) { - return QVariant::fromValue<QAudioDeviceInfo>(_devices.at(index.row()).info); + if (role == DisplayRole) { + return _devices.at(index.row())->display; + } else if (role == CheckStateRole) { + return _devices.at(index.row())->selected; + } else if (role == InfoRole) { + return QVariant::fromValue<QAudioDeviceInfo>(_devices.at(index.row())->info); } else { return QVariant(); } } +QVariant AudioInputDeviceList::data(const QModelIndex& index, int role) const { + if (!index.isValid() || index.row() >= rowCount()) { + return QVariant(); + } + + if (role == PeakRole) { + return std::static_pointer_cast<AudioInputDevice>(_devices.at(index.row()))->peak; + } else { + return AudioDeviceList::data(index, role); + } +} + void AudioDeviceList::resetDevice(bool contextIsHMD) { auto client = DependencyManager::get<AudioClient>().data(); QString deviceName = getTargetDevice(contextIsHMD, _mode); @@ -113,8 +134,9 @@ void AudioDeviceList::onDeviceChanged(const QAudioDeviceInfo& device) { auto oldDevice = _selectedDevice; _selectedDevice = device; - for (auto i = 0; i < _devices.size(); ++i) { - AudioDevice& device = _devices[i]; + for (auto i = 0; i < rowCount(); ++i) { + AudioDevice& device = *_devices[i]; + if (device.selected && device.info != _selectedDevice) { device.selected = false; } else if (device.info == _selectedDevice) { @@ -139,17 +161,47 @@ void AudioDeviceList::onDevicesChanged(const QList<QAudioDeviceInfo>& devices) { .remove("Device") .replace(" )", ")"); device.selected = (device.info == _selectedDevice); - _devices.push_back(device); + _devices.push_back(newDevice(device)); } endResetModel(); } +bool AudioInputDeviceList::peakValuesAvailable() { + std::call_once(_peakFlag, [&] { + _peakValuesAvailable = DependencyManager::get<AudioClient>()->peakValuesAvailable(); + }); + return _peakValuesAvailable; +} + +void AudioInputDeviceList::setPeakValuesEnabled(bool enable) { + if (peakValuesAvailable() && (enable != _peakValuesEnabled)) { + DependencyManager::get<AudioClient>()->enablePeakValues(enable); + _peakValuesEnabled = enable; + emit peakValuesEnabledChanged(_peakValuesEnabled); + } +} + +void AudioInputDeviceList::onPeakValueListChanged(const QList<float>& peakValueList) { + assert(_mode == QAudio::AudioInput); + + if (peakValueList.length() != rowCount()) { + qWarning() << "AudioDeviceList" << __FUNCTION__ << "length mismatch"; + } + + for (auto i = 0; i < rowCount(); ++i) { + std::static_pointer_cast<AudioInputDevice>(_devices[i])->peak = peakValueList[i]; + } + + emit dataChanged(createIndex(0, 0), createIndex(rowCount() - 1, 0), { PeakRole }); +} + AudioDevices::AudioDevices(bool& contextIsHMD) : _contextIsHMD(contextIsHMD) { auto client = DependencyManager::get<AudioClient>(); connect(client.data(), &AudioClient::deviceChanged, this, &AudioDevices::onDeviceChanged, Qt::QueuedConnection); connect(client.data(), &AudioClient::devicesChanged, this, &AudioDevices::onDevicesChanged, Qt::QueuedConnection); + connect(client.data(), &AudioClient::peakValueListChanged, &_inputs, &AudioInputDeviceList::onPeakValueListChanged, Qt::QueuedConnection); // connections are made after client is initialized, so we must also fetch the devices _inputs.onDeviceChanged(client->getActiveAudioDevice(QAudio::AudioInput)); diff --git a/interface/src/scripting/AudioDevices.h b/interface/src/scripting/AudioDevices.h index 3278a53374..4c1820e9c8 100644 --- a/interface/src/scripting/AudioDevices.h +++ b/interface/src/scripting/AudioDevices.h @@ -12,6 +12,9 @@ #ifndef hifi_scripting_AudioDevices_h #define hifi_scripting_AudioDevices_h +#include <memory> +#include <mutex> + #include <QObject> #include <QAbstractListModel> #include <QAudioDeviceInfo> @@ -29,7 +32,11 @@ class AudioDeviceList : public QAbstractListModel { Q_OBJECT public: - AudioDeviceList(QAudio::Mode mode) : _mode(mode) {} + AudioDeviceList(QAudio::Mode mode = QAudio::AudioOutput) : _mode(mode) {} + ~AudioDeviceList() = default; + + virtual std::shared_ptr<AudioDevice> newDevice(const AudioDevice& device) + { return std::make_shared<AudioDevice>(device); } int rowCount(const QModelIndex& parent = QModelIndex()) const override { Q_UNUSED(parent); return _devices.size(); } QHash<int, QByteArray> roleNames() const override { return _roles; } @@ -44,25 +51,63 @@ public: signals: void deviceChanged(const QAudioDeviceInfo& device); -private slots: +protected slots: void onDeviceChanged(const QAudioDeviceInfo& device); void onDevicesChanged(const QList<QAudioDeviceInfo>& devices); -private: +protected: friend class AudioDevices; static QHash<int, QByteArray> _roles; static Qt::ItemFlags _flags; const QAudio::Mode _mode; QAudioDeviceInfo _selectedDevice; - QList<AudioDevice> _devices; + QList<std::shared_ptr<AudioDevice>> _devices; +}; + +class AudioInputDevice : public AudioDevice { +public: + AudioInputDevice(const AudioDevice& device) : AudioDevice(device) {} + float peak { 0.0f }; +}; + +class AudioInputDeviceList : public AudioDeviceList { + Q_OBJECT + Q_PROPERTY(bool peakValuesAvailable READ peakValuesAvailable) + Q_PROPERTY(bool peakValuesEnabled READ peakValuesEnabled WRITE setPeakValuesEnabled NOTIFY peakValuesEnabledChanged) + +public: + AudioInputDeviceList() : AudioDeviceList(QAudio::AudioInput) {} + virtual ~AudioInputDeviceList() = default; + + virtual std::shared_ptr<AudioDevice> newDevice(const AudioDevice& device) override + { return std::make_shared<AudioInputDevice>(device); } + + QVariant data(const QModelIndex& index, int role) const override; + +signals: + void peakValuesEnabledChanged(bool enabled); + +protected slots: + void onPeakValueListChanged(const QList<float>& peakValueList); + +protected: + friend class AudioDevices; + + bool peakValuesAvailable(); + std::once_flag _peakFlag; + bool _peakValuesAvailable; + + bool peakValuesEnabled() const { return _peakValuesEnabled; } + void setPeakValuesEnabled(bool enable); + bool _peakValuesEnabled { false }; }; class Audio; class AudioDevices : public QObject { Q_OBJECT - Q_PROPERTY(AudioDeviceList* input READ getInputList NOTIFY nop) + Q_PROPERTY(AudioInputDeviceList* input READ getInputList NOTIFY nop) Q_PROPERTY(AudioDeviceList* output READ getOutputList NOTIFY nop) public: @@ -82,10 +127,10 @@ private slots: private: friend class Audio; - AudioDeviceList* getInputList() { return &_inputs; } + AudioInputDeviceList* getInputList() { return &_inputs; } AudioDeviceList* getOutputList() { return &_outputs; } - AudioDeviceList _inputs { QAudio::AudioInput }; + AudioInputDeviceList _inputs; AudioDeviceList _outputs { QAudio::AudioOutput }; QAudioDeviceInfo _requestedOutputDevice; QAudioDeviceInfo _requestedInputDevice; diff --git a/libraries/audio-client/src/AudioClient.cpp b/libraries/audio-client/src/AudioClient.cpp index 27bab687d5..4e7cc08919 100644 --- a/libraries/audio-client/src/AudioClient.cpp +++ b/libraries/audio-client/src/AudioClient.cpp @@ -78,7 +78,7 @@ Setting::Handle<int> staticJitterBufferFrames("staticJitterBufferFrames", // protect the Qt internal device list using Mutex = std::mutex; using Lock = std::unique_lock<Mutex>; -static Mutex _deviceMutex; +Mutex _deviceMutex; // thread-safe QList<QAudioDeviceInfo> getAvailableDevices(QAudio::Mode mode) { @@ -227,13 +227,18 @@ AudioClient::AudioClient() : // start a thread to detect any device changes _checkDevicesTimer = new QTimer(this); connect(_checkDevicesTimer, &QTimer::timeout, [this] { - QtConcurrent::run(QThreadPool::globalInstance(), [this] { - checkDevices(); - }); + QtConcurrent::run(QThreadPool::globalInstance(), [this] { checkDevices(); }); }); const unsigned long DEVICE_CHECK_INTERVAL_MSECS = 2 * 1000; _checkDevicesTimer->start(DEVICE_CHECK_INTERVAL_MSECS); + // start a thread to detect peak value changes + _checkPeakValuesTimer = new QTimer(this); + connect(_checkPeakValuesTimer, &QTimer::timeout, [this] { + QtConcurrent::run(QThreadPool::globalInstance(), [this] { checkPeakValues(); }); + }); + const unsigned long PEAK_VALUES_CHECK_INTERVAL_MSECS = 50; + _checkPeakValuesTimer->start(PEAK_VALUES_CHECK_INTERVAL_MSECS); configureReverb(); @@ -275,6 +280,7 @@ void AudioClient::cleanupBeforeQuit() { stop(); _checkDevicesTimer->stop(); + _checkPeakValuesTimer->stop(); guard.trigger(); } @@ -316,8 +322,6 @@ QString getWinDeviceName(IMMDevice* pEndpoint) { QString deviceName; IPropertyStore* pPropertyStore; pEndpoint->OpenPropertyStore(STGM_READ, &pPropertyStore); - pEndpoint->Release(); - pEndpoint = nullptr; PROPVARIANT pv; PropVariantInit(&pv); HRESULT hr = pPropertyStore->GetValue(PKEY_Device_FriendlyName, &pv); @@ -346,6 +350,8 @@ QString AudioClient::getWinDeviceName(wchar_t* guid) { deviceName = QString("NONE"); } else { deviceName = ::getWinDeviceName(pEndpoint); + pEndpoint->Release(); + pEndpoint = nullptr; } pMMDeviceEnumerator->Release(); pMMDeviceEnumerator = nullptr; @@ -429,6 +435,8 @@ QAudioDeviceInfo defaultAudioDeviceForMode(QAudio::Mode mode) { deviceName = QString("NONE"); } else { deviceName = getWinDeviceName(pEndpoint); + pEndpoint->Release(); + pEndpoint = nullptr; } pMMDeviceEnumerator->Release(); pMMDeviceEnumerator = NULL; diff --git a/libraries/audio-client/src/AudioClient.h b/libraries/audio-client/src/AudioClient.h index 26a07235d5..ff0ea968a8 100644 --- a/libraries/audio-client/src/AudioClient.h +++ b/libraries/audio-client/src/AudioClient.h @@ -148,6 +148,9 @@ public: QAudioDeviceInfo getActiveAudioDevice(QAudio::Mode mode) const; QList<QAudioDeviceInfo> getAudioDevices(QAudio::Mode mode) const; + void enablePeakValues(bool enable) { _enablePeakValues = enable; } + bool peakValuesAvailable() const; + static const float CALLBACK_ACCELERATOR_RATIO; bool getNamedAudioDeviceForModeExists(QAudio::Mode mode, const QString& deviceName); @@ -224,6 +227,7 @@ signals: void deviceChanged(QAudio::Mode mode, const QAudioDeviceInfo& device); void devicesChanged(QAudio::Mode mode, const QList<QAudioDeviceInfo>& devices); + void peakValueListChanged(const QList<float> peakValueList); void receivedFirstPacket(); void disconnected(); @@ -242,9 +246,12 @@ private: friend class CheckDevicesThread; friend class LocalInjectorsThread; + // background tasks + void checkDevices(); + void checkPeakValues(); + void outputFormatChanged(); void handleAudioInput(QByteArray& audioBuffer); - void checkDevices(); void prepareLocalAudioInjectors(std::unique_ptr<Lock> localAudioLock = nullptr); bool mixLocalAudioInjectors(float* mixBuffer); float azimuthForSource(const glm::vec3& relativePosition); @@ -298,6 +305,7 @@ private: std::atomic<bool> _localInjectorsAvailable { false }; MixedProcessedAudioStream _receivedAudioStream; bool _isStereoInput; + std::atomic<bool> _enablePeakValues { false }; quint64 _outputStarveDetectionStartTimeMsec; int _outputStarveDetectionCount; @@ -396,6 +404,7 @@ private: RateCounter<> _audioInbound; QTimer* _checkDevicesTimer { nullptr }; + QTimer* _checkPeakValuesTimer { nullptr }; }; diff --git a/libraries/audio-client/src/AudioPeakValues.cpp b/libraries/audio-client/src/AudioPeakValues.cpp new file mode 100644 index 0000000000..3df469b830 --- /dev/null +++ b/libraries/audio-client/src/AudioPeakValues.cpp @@ -0,0 +1,176 @@ +// +// AudioPeakValues.cpp +// interface/src +// +// Created by Zach Pomerantz on 6/26/2017 +// Copyright 2013 High Fidelity, Inc. +// +// Distributed under the Apache License, Version 2.0. +// See the accompanying file LICENSE or http://www.apache.org/licenses/LICENSE-2.0.html +// + +#include "AudioClient.h" + +#ifdef Q_OS_WIN + +#include <windows.h> +#include <mmdeviceapi.h> +#include <endpointvolume.h> +#include <audioclient.h> + +#include <QString> + +#define RETURN_ON_FAIL(result) if (FAILED(result)) { return; } +#define CONTINUE_ON_FAIL(result) if (FAILED(result)) { continue; } + +extern QString getWinDeviceName(IMMDevice* pEndpoint); +extern std::mutex _deviceMutex; + +std::map<std::wstring, std::shared_ptr<IAudioClient>> activeClients; + +template <class T> +void release(T* t) { + t->Release(); +} + +template <> +void release(IAudioClient* audioClient) { + audioClient->Stop(); + audioClient->Release(); +} + +void AudioClient::checkPeakValues() { + // prepare the windows environment + CoInitialize(NULL); + + // if disabled, clean up active clients + if (!_enablePeakValues) { + activeClients.clear(); + return; + } + + // lock the devices so the _inputDevices list is static + std::unique_lock<std::mutex> lock(_deviceMutex); + HRESULT result; + + // initialize the payload + QList<float> peakValueList; + for (int i = 0; i < _inputDevices.size(); ++i) { + peakValueList.push_back(0.0f); + } + + std::shared_ptr<IMMDeviceEnumerator> enumerator; + { + IMMDeviceEnumerator* pEnumerator; + result = CoCreateInstance( + __uuidof(MMDeviceEnumerator), NULL, CLSCTX_INPROC_SERVER, + __uuidof(IMMDeviceEnumerator), (void**)&pEnumerator); + RETURN_ON_FAIL(result); + enumerator = std::shared_ptr<IMMDeviceEnumerator>(pEnumerator, &release<IMMDeviceEnumerator>); + } + + std::shared_ptr<IMMDeviceCollection> endpoints; + { + IMMDeviceCollection* pEndpoints; + result = enumerator->EnumAudioEndpoints(eCapture, DEVICE_STATE_ACTIVE, &pEndpoints); + RETURN_ON_FAIL(result); + endpoints = std::shared_ptr<IMMDeviceCollection>(pEndpoints, &release<IMMDeviceCollection>); + } + + UINT count; + { + result = endpoints->GetCount(&count); + RETURN_ON_FAIL(result); + } + + IMMDevice* pDevice; + std::shared_ptr<IMMDevice> device; + IAudioMeterInformation* pMeterInfo; + std::shared_ptr<IAudioMeterInformation> meterInfo; + IAudioClient* pAudioClient; + std::shared_ptr<IAudioClient> audioClient; + DWORD hardwareSupport; + LPWSTR pDeviceId = NULL; + LPWAVEFORMATEX format; + float peakValue; + QString deviceName; + int deviceIndex; + for (UINT i = 0; i < count; ++i) { + result = endpoints->Item(i, &pDevice); + CONTINUE_ON_FAIL(result); + device = std::shared_ptr<IMMDevice>(pDevice, &release<IMMDevice>); + + // if the device isn't listed through Qt, skip it + deviceName = ::getWinDeviceName(pDevice); + deviceIndex = 0; + for (; deviceIndex < _inputDevices.size(); ++deviceIndex) { + if (deviceName == _inputDevices[deviceIndex].deviceName()) { + break; + } + } + if (deviceIndex >= _inputDevices.size()) { + continue; + } + + //continue; + + result = device->Activate(__uuidof(IAudioMeterInformation), CLSCTX_ALL, NULL, (void**)&pMeterInfo); + CONTINUE_ON_FAIL(result); + meterInfo = std::shared_ptr<IAudioMeterInformation>(pMeterInfo, &release<IAudioMeterInformation>); + + //continue; + + hardwareSupport; + result = meterInfo->QueryHardwareSupport(&hardwareSupport); + CONTINUE_ON_FAIL(result); + + //continue; + + // if the device has no hardware support (USB)... + if (!(hardwareSupport & ENDPOINT_HARDWARE_SUPPORT_METER)) { + result = device->GetId(&pDeviceId); + CONTINUE_ON_FAIL(result); + std::wstring deviceId(pDeviceId); + CoTaskMemFree(pDeviceId); + + //continue; + + // ...and no active client... + if (activeClients.find(deviceId) == activeClients.end()) { + result = device->Activate(__uuidof(IAudioClient), CLSCTX_ALL, NULL, (void**)&pAudioClient); + CONTINUE_ON_FAIL(result); + audioClient = std::shared_ptr<IAudioClient>(pAudioClient, &release<IAudioClient>); + + //continue; + + // ...activate a client + audioClient->GetMixFormat(&format); + audioClient->Initialize(AUDCLNT_SHAREMODE_SHARED, 0, 0, 0, format, NULL); + audioClient->Start(); + + //continue; + + activeClients[deviceId] = audioClient; + } + } + + // get the peak value and put it in the payload + meterInfo->GetPeakValue(&peakValue); + peakValueList[deviceIndex] = peakValue; + } + + emit peakValueListChanged(peakValueList); +} + +bool AudioClient::peakValuesAvailable() const { + return true; +} + +#else +void AudioClient::checkPeakValues() { +} + +bool AudioClient::peakValuesAvailable() const { + return false; +} +#endif \ No newline at end of file