Merge pull request #10937 from kencooke/zzmp-audio/peak-values

Add peak values for all input devices in selection dialog

assignment-client/src/audio/AudioMixerSlave.cpp

@@ -558,7 +558,7 @@ float computeAzimuth(const AvatarAudioStream& listeningNodeStream, const Positio
         
         // produce an oriented angle about the y-axis
         glm::vec3 direction = rotatedSourcePosition * (1.0f / fastSqrtf(rotatedSourcePositionLength2));
-		float angle = fastAcosf(glm::clamp(-direction.z, -1.0f, 1.0f)); // UNIT_NEG_Z is "forward"
+        float angle = fastAcosf(glm::clamp(-direction.z, -1.0f, 1.0f)); // UNIT_NEG_Z is "forward"
         return (direction.x < 0.0f) ? -angle : angle;
 
     } else {   

interface/resources/qml/hifi/audio/Audio.qml

@@ -36,6 +36,24 @@ Rectangle {
         return (root.parent !== null) && root.parent.objectName == "loader";
     }
 
+    property bool showPeaks: true;
+    function enablePeakValues() {
+        Audio.devices.input.peakValuesEnabled = true;
+        Audio.devices.input.peakValuesEnabledChanged.connect(function(enabled) {
+            if (!enabled && root.showPeaks) {
+                Audio.devices.input.peakValuesEnabled = true;
+            }
+        });
+    }
+    function disablePeakValues() {
+        root.showPeaks = false;
+        Audio.devices.input.peakValuesEnabled = false;
+    }
+
+    Component.onCompleted: enablePeakValues();
+    Component.onDestruction: disablePeakValues();
+    onVisibleChanged: visible ? enablePeakValues() : disablePeakValues();
+
     Column {
         y: 16; // padding does not work
         spacing: 16;
@@ -133,12 +151,13 @@ Rectangle {
                     onClicked: Audio.setInputDevice(info);
                 }
 
-                InputLevel {
-                    id: level;
+                InputPeak {
+                    id: inputPeak;
+                    visible: Audio.devices.input.peakValuesAvailable;
+                    peak: model.peak;
                     anchors.verticalCenter: parent.verticalCenter
                     anchors.right: parent.right
                     anchors.rightMargin: 30
-                    visible: selected;
                 }
             }
         }

interface/resources/qml/hifi/audio/InputPeak.qml

@@ -1,5 +1,5 @@
 //
-//  InputLevel.qml
+//  InputPeak.qml
 //  qml/hifi/audio
 //
 //  Created by Zach Pomerantz on 6/20/2017
@@ -15,7 +15,7 @@ import QtQuick.Layouts 1.3
 import QtGraphicalEffects 1.0
 
 Rectangle {
-    readonly property var level: Audio.inputLevel;
+    property var peak;
 
     width: 70;
     height: 8;
@@ -65,7 +65,7 @@ Rectangle {
 
         Rectangle { // mask
             id: mask;
-            width: parent.width * level;
+            width: parent.width * peak;
             radius: 5;
             anchors {
                 bottom: parent.bottom;

interface/src/scripting/AudioDevices.cpp

@@ -37,6 +37,20 @@ Setting::Handle<QString>& getSetting(bool contextIsHMD, QAudio::Mode mode) {
     }
 }
 
+enum AudioDeviceRole {
+    DisplayRole = Qt::DisplayRole,
+    CheckStateRole = Qt::CheckStateRole,
+    PeakRole = Qt::UserRole,
+    InfoRole = Qt::UserRole + 1
+};
+
+QHash<int, QByteArray> AudioDeviceList::_roles {
+    { DisplayRole, "display" },
+    { CheckStateRole, "selected" },
+    { PeakRole, "peak" },
+    { InfoRole, "info" }
+};
+
 static QString getTargetDevice(bool hmd, QAudio::Mode mode) {
     QString deviceName;
     auto& setting = getSetting(hmd, mode);
@@ -52,29 +66,36 @@ static QString getTargetDevice(bool hmd, QAudio::Mode mode) {
     return deviceName;
 }
 
-QHash<int, QByteArray> AudioDeviceList::_roles {
-    { Qt::DisplayRole, "display" },
-    { Qt::CheckStateRole, "selected" },
-    { Qt::UserRole, "info" }
-};
 Qt::ItemFlags AudioDeviceList::_flags { Qt::ItemIsSelectable | Qt::ItemIsEnabled };
 
 QVariant AudioDeviceList::data(const QModelIndex& index, int role) const {
-    if (!index.isValid() || index.row() >= _devices.size()) {
+    if (!index.isValid() || index.row() >= rowCount()) {
         return QVariant();
     }
 
-    if (role == Qt::DisplayRole) {
-        return _devices.at(index.row()).display;
-    } else if (role == Qt::CheckStateRole) {
-        return _devices.at(index.row()).selected;
-    } else if (role == Qt::UserRole) {
-        return QVariant::fromValue<QAudioDeviceInfo>(_devices.at(index.row()).info);
+    if (role == DisplayRole) {
+        return _devices.at(index.row())->display;
+    } else if (role == CheckStateRole) {
+        return _devices.at(index.row())->selected;
+    } else if (role == InfoRole) {
+        return QVariant::fromValue<QAudioDeviceInfo>(_devices.at(index.row())->info);
     } else {
         return QVariant();
     }
 }
 
+QVariant AudioInputDeviceList::data(const QModelIndex& index, int role) const {
+    if (!index.isValid() || index.row() >= rowCount()) {
+        return QVariant();
+    }
+
+    if (role == PeakRole) {
+        return std::static_pointer_cast<AudioInputDevice>(_devices.at(index.row()))->peak;
+    } else {
+        return AudioDeviceList::data(index, role);
+    }
+}
+
 void AudioDeviceList::resetDevice(bool contextIsHMD) {
     auto client = DependencyManager::get<AudioClient>().data();
     QString deviceName = getTargetDevice(contextIsHMD, _mode);
@@ -113,8 +134,9 @@ void AudioDeviceList::onDeviceChanged(const QAudioDeviceInfo& device) {
     auto oldDevice = _selectedDevice;
     _selectedDevice = device;
 
-    for (auto i = 0; i < _devices.size(); ++i) {
-        AudioDevice& device = _devices[i];
+    for (auto i = 0; i < rowCount(); ++i) {
+        AudioDevice& device = *_devices[i];
+
         if (device.selected && device.info != _selectedDevice) {
             device.selected = false;
         } else if (device.info == _selectedDevice) {
@@ -139,17 +161,47 @@ void AudioDeviceList::onDevicesChanged(const QList<QAudioDeviceInfo>& devices) {
             .remove("Device")
             .replace(" )", ")");
         device.selected = (device.info == _selectedDevice);
-        _devices.push_back(device);
+        _devices.push_back(newDevice(device));
     }
 
     endResetModel();
 }
 
+bool AudioInputDeviceList::peakValuesAvailable() {
+    std::call_once(_peakFlag, [&] {
+        _peakValuesAvailable = DependencyManager::get<AudioClient>()->peakValuesAvailable();
+    });
+    return _peakValuesAvailable;
+}
+
+void AudioInputDeviceList::setPeakValuesEnabled(bool enable) {
+    if (peakValuesAvailable() && (enable != _peakValuesEnabled)) {
+        DependencyManager::get<AudioClient>()->enablePeakValues(enable);
+        _peakValuesEnabled = enable;
+        emit peakValuesEnabledChanged(_peakValuesEnabled);
+    }
+}
+
+void AudioInputDeviceList::onPeakValueListChanged(const QList<float>& peakValueList) {
+    assert(_mode == QAudio::AudioInput);
+
+    if (peakValueList.length() != rowCount()) {
+        qWarning() << "AudioDeviceList" << __FUNCTION__ << "length mismatch";
+    }
+
+    for (auto i = 0; i < rowCount(); ++i) {
+        std::static_pointer_cast<AudioInputDevice>(_devices[i])->peak = peakValueList[i];
+    }
+
+    emit dataChanged(createIndex(0, 0), createIndex(rowCount() - 1, 0), { PeakRole });
+}
+
 AudioDevices::AudioDevices(bool& contextIsHMD) : _contextIsHMD(contextIsHMD) {
     auto client = DependencyManager::get<AudioClient>();
 
     connect(client.data(), &AudioClient::deviceChanged, this, &AudioDevices::onDeviceChanged, Qt::QueuedConnection);
     connect(client.data(), &AudioClient::devicesChanged, this, &AudioDevices::onDevicesChanged, Qt::QueuedConnection);
+    connect(client.data(), &AudioClient::peakValueListChanged, &_inputs, &AudioInputDeviceList::onPeakValueListChanged, Qt::QueuedConnection);
 
     // connections are made after client is initialized, so we must also fetch the devices
     _inputs.onDeviceChanged(client->getActiveAudioDevice(QAudio::AudioInput));

interface/src/scripting/AudioDevices.h

@@ -12,6 +12,9 @@
 #ifndef hifi_scripting_AudioDevices_h
 #define hifi_scripting_AudioDevices_h
 
+#include <memory>
+#include <mutex>
+
 #include <QObject>
 #include <QAbstractListModel>
 #include <QAudioDeviceInfo>
@@ -29,7 +32,11 @@ class AudioDeviceList : public QAbstractListModel {
     Q_OBJECT
 
 public:
-    AudioDeviceList(QAudio::Mode mode) : _mode(mode) {}
+    AudioDeviceList(QAudio::Mode mode = QAudio::AudioOutput) : _mode(mode) {}
+    ~AudioDeviceList() = default;
+
+    virtual std::shared_ptr<AudioDevice> newDevice(const AudioDevice& device)
+        { return std::make_shared<AudioDevice>(device); }
 
     int rowCount(const QModelIndex& parent = QModelIndex()) const override { Q_UNUSED(parent); return _devices.size(); }
     QHash<int, QByteArray> roleNames() const override { return _roles; }
@@ -44,25 +51,63 @@ public:
 signals:
     void deviceChanged(const QAudioDeviceInfo& device);
 
-private slots:
+protected slots:
     void onDeviceChanged(const QAudioDeviceInfo& device);
     void onDevicesChanged(const QList<QAudioDeviceInfo>& devices);
 
-private:
+protected:
     friend class AudioDevices;
 
     static QHash<int, QByteArray> _roles;
     static Qt::ItemFlags _flags;
     const QAudio::Mode _mode;
     QAudioDeviceInfo _selectedDevice;
-    QList<AudioDevice> _devices;
+    QList<std::shared_ptr<AudioDevice>> _devices;
+};
+
+class AudioInputDevice : public AudioDevice {
+public:
+    AudioInputDevice(const AudioDevice& device) : AudioDevice(device) {}
+    float peak { 0.0f };
+};
+
+class AudioInputDeviceList : public AudioDeviceList {
+    Q_OBJECT
+    Q_PROPERTY(bool peakValuesAvailable READ peakValuesAvailable)
+    Q_PROPERTY(bool peakValuesEnabled READ peakValuesEnabled WRITE setPeakValuesEnabled NOTIFY peakValuesEnabledChanged)
+
+public:
+    AudioInputDeviceList() : AudioDeviceList(QAudio::AudioInput) {}
+    virtual ~AudioInputDeviceList() = default;
+
+    virtual std::shared_ptr<AudioDevice> newDevice(const AudioDevice& device) override
+        { return std::make_shared<AudioInputDevice>(device); }
+
+    QVariant data(const QModelIndex& index, int role) const override;
+
+signals:
+    void peakValuesEnabledChanged(bool enabled);
+
+protected slots:
+    void onPeakValueListChanged(const QList<float>& peakValueList);
+
+protected:
+    friend class AudioDevices;
+
+    bool peakValuesAvailable();
+    std::once_flag _peakFlag;
+    bool _peakValuesAvailable;
+
+    bool peakValuesEnabled() const { return _peakValuesEnabled; }
+    void setPeakValuesEnabled(bool enable);
+    bool _peakValuesEnabled { false };
 };
 
 class Audio;
 
 class AudioDevices : public QObject {
     Q_OBJECT
-    Q_PROPERTY(AudioDeviceList* input READ getInputList NOTIFY nop)
+    Q_PROPERTY(AudioInputDeviceList* input READ getInputList NOTIFY nop)
     Q_PROPERTY(AudioDeviceList* output READ getOutputList NOTIFY nop)
 
 public:
@@ -82,10 +127,10 @@ private slots:
 private:
     friend class Audio;
 
-    AudioDeviceList* getInputList() { return &_inputs; }
+    AudioInputDeviceList* getInputList() { return &_inputs; }
     AudioDeviceList* getOutputList() { return &_outputs; }
 
-    AudioDeviceList _inputs { QAudio::AudioInput };
+    AudioInputDeviceList _inputs;
     AudioDeviceList _outputs { QAudio::AudioOutput };
     QAudioDeviceInfo _requestedOutputDevice;
     QAudioDeviceInfo _requestedInputDevice;

libraries/audio-client/src/AudioClient.cpp

@@ -78,7 +78,7 @@ Setting::Handle<int> staticJitterBufferFrames("staticJitterBufferFrames",
 // protect the Qt internal device list
 using Mutex = std::mutex;
 using Lock = std::unique_lock<Mutex>;
-static Mutex _deviceMutex;
+Mutex _deviceMutex;
 
 // thread-safe
 QList<QAudioDeviceInfo> getAvailableDevices(QAudio::Mode mode) {
@@ -227,13 +227,18 @@ AudioClient::AudioClient() :
     // start a thread to detect any device changes
     _checkDevicesTimer = new QTimer(this);
     connect(_checkDevicesTimer, &QTimer::timeout, [this] {
-        QtConcurrent::run(QThreadPool::globalInstance(), [this] {
-            checkDevices();
-        });
+        QtConcurrent::run(QThreadPool::globalInstance(), [this] { checkDevices(); });
     });
     const unsigned long DEVICE_CHECK_INTERVAL_MSECS = 2 * 1000;
     _checkDevicesTimer->start(DEVICE_CHECK_INTERVAL_MSECS);
 
+    // start a thread to detect peak value changes
+    _checkPeakValuesTimer = new QTimer(this);
+    connect(_checkPeakValuesTimer, &QTimer::timeout, [this] {
+        QtConcurrent::run(QThreadPool::globalInstance(), [this] { checkPeakValues(); });
+    });
+    const unsigned long PEAK_VALUES_CHECK_INTERVAL_MSECS = 50;
+    _checkPeakValuesTimer->start(PEAK_VALUES_CHECK_INTERVAL_MSECS);
 
     configureReverb();
 
@@ -275,6 +280,7 @@ void AudioClient::cleanupBeforeQuit() {
 
     stop();
     _checkDevicesTimer->stop();
+    _checkPeakValuesTimer->stop();
     guard.trigger();
 }
 
@@ -316,8 +322,6 @@ QString getWinDeviceName(IMMDevice* pEndpoint) {
     QString deviceName;
     IPropertyStore* pPropertyStore;
     pEndpoint->OpenPropertyStore(STGM_READ, &pPropertyStore);
-    pEndpoint->Release();
-    pEndpoint = nullptr;
     PROPVARIANT pv;
     PropVariantInit(&pv);
     HRESULT hr = pPropertyStore->GetValue(PKEY_Device_FriendlyName, &pv);
@@ -346,6 +350,8 @@ QString AudioClient::getWinDeviceName(wchar_t* guid) {
         deviceName = QString("NONE");
     } else {
         deviceName = ::getWinDeviceName(pEndpoint);
+        pEndpoint->Release();
+        pEndpoint = nullptr;
     }
     pMMDeviceEnumerator->Release();
     pMMDeviceEnumerator = nullptr;
@@ -429,6 +435,8 @@ QAudioDeviceInfo defaultAudioDeviceForMode(QAudio::Mode mode) {
             deviceName = QString("NONE");
         } else {
             deviceName = getWinDeviceName(pEndpoint);
+            pEndpoint->Release();
+            pEndpoint = nullptr;
         }
         pMMDeviceEnumerator->Release();
         pMMDeviceEnumerator = NULL;

libraries/audio-client/src/AudioClient.h

@@ -148,6 +148,9 @@ public:
     QAudioDeviceInfo getActiveAudioDevice(QAudio::Mode mode) const;
     QList<QAudioDeviceInfo> getAudioDevices(QAudio::Mode mode) const;
 
+    void enablePeakValues(bool enable) { _enablePeakValues = enable; }
+    bool peakValuesAvailable() const;
+
     static const float CALLBACK_ACCELERATOR_RATIO;
 
     bool getNamedAudioDeviceForModeExists(QAudio::Mode mode, const QString& deviceName);
@@ -224,6 +227,7 @@ signals:
 
     void deviceChanged(QAudio::Mode mode, const QAudioDeviceInfo& device);
     void devicesChanged(QAudio::Mode mode, const QList<QAudioDeviceInfo>& devices);
+    void peakValueListChanged(const QList<float> peakValueList);
 
     void receivedFirstPacket();
     void disconnected();
@@ -242,9 +246,12 @@ private:
     friend class CheckDevicesThread;
     friend class LocalInjectorsThread;
 
+    // background tasks
+    void checkDevices();
+    void checkPeakValues();
+
     void outputFormatChanged();
     void handleAudioInput(QByteArray& audioBuffer);
-    void checkDevices();
     void prepareLocalAudioInjectors(std::unique_ptr<Lock> localAudioLock = nullptr);
     bool mixLocalAudioInjectors(float* mixBuffer);
     float azimuthForSource(const glm::vec3& relativePosition);
@@ -298,6 +305,7 @@ private:
     std::atomic<bool> _localInjectorsAvailable { false };
     MixedProcessedAudioStream _receivedAudioStream;
     bool _isStereoInput;
+    std::atomic<bool> _enablePeakValues { false };
 
     quint64 _outputStarveDetectionStartTimeMsec;
     int _outputStarveDetectionCount;
@@ -396,6 +404,7 @@ private:
     RateCounter<> _audioInbound;
 
     QTimer* _checkDevicesTimer { nullptr };
+    QTimer* _checkPeakValuesTimer { nullptr };
 };
 
 

libraries/audio-client/src/AudioPeakValues.cpp

@@ -0,0 +1,176 @@
+//
+//  AudioPeakValues.cpp
+//  interface/src
+//
+//  Created by Zach Pomerantz on 6/26/2017
+//  Copyright 2013 High Fidelity, Inc.
+//
+//  Distributed under the Apache License, Version 2.0.
+//  See the accompanying file LICENSE or http://www.apache.org/licenses/LICENSE-2.0.html
+//
+
+#include "AudioClient.h"
+
+#ifdef Q_OS_WIN
+
+#include <windows.h>
+#include <mmdeviceapi.h>
+#include <endpointvolume.h>
+#include <audioclient.h>
+
+#include <QString>
+
+#define RETURN_ON_FAIL(result) if (FAILED(result)) { return; }
+#define CONTINUE_ON_FAIL(result) if (FAILED(result)) { continue; }
+
+extern QString getWinDeviceName(IMMDevice* pEndpoint);
+extern std::mutex _deviceMutex;
+
+std::map<std::wstring, std::shared_ptr<IAudioClient>> activeClients;
+
+template <class T>
+void release(T* t) {
+    t->Release();
+}
+
+template <>
+void release(IAudioClient* audioClient) {
+    audioClient->Stop();
+    audioClient->Release();
+}
+
+void AudioClient::checkPeakValues() {
+    // prepare the windows environment
+    CoInitialize(NULL);
+
+    // if disabled, clean up active clients
+    if (!_enablePeakValues) {
+        activeClients.clear();
+        return;
+    }
+
+    // lock the devices so the _inputDevices list is static
+    std::unique_lock<std::mutex> lock(_deviceMutex);
+    HRESULT result;
+
+    // initialize the payload
+    QList<float> peakValueList;
+    for (int i = 0; i < _inputDevices.size(); ++i) {
+        peakValueList.push_back(0.0f);
+    }
+
+    std::shared_ptr<IMMDeviceEnumerator> enumerator;
+    {
+        IMMDeviceEnumerator* pEnumerator;
+        result = CoCreateInstance(
+            __uuidof(MMDeviceEnumerator), NULL, CLSCTX_INPROC_SERVER,
+            __uuidof(IMMDeviceEnumerator), (void**)&pEnumerator);
+        RETURN_ON_FAIL(result);
+        enumerator = std::shared_ptr<IMMDeviceEnumerator>(pEnumerator, &release<IMMDeviceEnumerator>);
+    }
+
+    std::shared_ptr<IMMDeviceCollection> endpoints;
+    {
+        IMMDeviceCollection* pEndpoints;
+        result = enumerator->EnumAudioEndpoints(eCapture, DEVICE_STATE_ACTIVE, &pEndpoints);
+        RETURN_ON_FAIL(result);
+        endpoints = std::shared_ptr<IMMDeviceCollection>(pEndpoints, &release<IMMDeviceCollection>);
+    }
+
+    UINT count;
+    {
+        result = endpoints->GetCount(&count);
+        RETURN_ON_FAIL(result);
+    }
+
+    IMMDevice* pDevice;
+    std::shared_ptr<IMMDevice> device;
+    IAudioMeterInformation* pMeterInfo;
+    std::shared_ptr<IAudioMeterInformation> meterInfo;
+    IAudioClient* pAudioClient;
+    std::shared_ptr<IAudioClient> audioClient;
+    DWORD hardwareSupport;
+    LPWSTR pDeviceId = NULL;
+    LPWAVEFORMATEX format;
+    float peakValue;
+    QString deviceName;
+    int deviceIndex;
+    for (UINT i = 0; i < count; ++i) {
+        result = endpoints->Item(i, &pDevice);
+        CONTINUE_ON_FAIL(result);
+        device = std::shared_ptr<IMMDevice>(pDevice, &release<IMMDevice>);
+
+        // if the device isn't listed through Qt, skip it
+        deviceName = ::getWinDeviceName(pDevice);
+        deviceIndex = 0;
+        for (;  deviceIndex < _inputDevices.size(); ++deviceIndex) {
+            if (deviceName == _inputDevices[deviceIndex].deviceName()) {
+                break;
+            }
+        }
+        if (deviceIndex >= _inputDevices.size()) {
+            continue;
+        }
+
+        //continue;
+
+        result = device->Activate(__uuidof(IAudioMeterInformation), CLSCTX_ALL, NULL, (void**)&pMeterInfo);
+        CONTINUE_ON_FAIL(result);
+        meterInfo = std::shared_ptr<IAudioMeterInformation>(pMeterInfo, &release<IAudioMeterInformation>);
+
+        //continue;
+
+        hardwareSupport;
+        result = meterInfo->QueryHardwareSupport(&hardwareSupport);
+        CONTINUE_ON_FAIL(result);
+
+        //continue;
+
+        // if the device has no hardware support (USB)...
+        if (!(hardwareSupport & ENDPOINT_HARDWARE_SUPPORT_METER)) {
+            result = device->GetId(&pDeviceId);
+            CONTINUE_ON_FAIL(result);
+            std::wstring deviceId(pDeviceId);
+            CoTaskMemFree(pDeviceId);
+
+            //continue;
+
+            // ...and no active client...
+            if (activeClients.find(deviceId) == activeClients.end()) {
+                result = device->Activate(__uuidof(IAudioClient), CLSCTX_ALL, NULL, (void**)&pAudioClient);
+                CONTINUE_ON_FAIL(result);
+                audioClient = std::shared_ptr<IAudioClient>(pAudioClient, &release<IAudioClient>);
+
+                //continue;
+
+                // ...activate a client
+                audioClient->GetMixFormat(&format);
+                audioClient->Initialize(AUDCLNT_SHAREMODE_SHARED, 0, 0, 0, format, NULL);
+                audioClient->Start();
+
+                //continue;
+
+                activeClients[deviceId] = audioClient;
+            }
+        }
+
+        // get the peak value and put it in the payload
+        meterInfo->GetPeakValue(&peakValue);
+        peakValueList[deviceIndex] = peakValue;
+    }
+
+    emit peakValueListChanged(peakValueList);
+}
+
+bool AudioClient::peakValuesAvailable() const {
+    return true;
+}
+
+#else
+void AudioClient::checkPeakValues() {
+}
+
+bool AudioClient::peakValuesAvailable() const {
+    return false;
+}
+#endif