From 803be053e98e650623b2e964bfc88e49035dcb3b Mon Sep 17 00:00:00 2001 From: Karol Suprynowicz Date: Sat, 4 May 2024 00:31:17 +0200 Subject: [PATCH] Initial script engine audio listener --- libraries/audio-client/src/AudioClient.cpp | 40 +++++++++++++++++++ libraries/audio-client/src/AudioClient.h | 8 ++++ libraries/audio/src/AbstractAudioInterface.h | 4 ++ .../audio/src/AudioScriptingInterface.cpp | 38 ++++++++++++++++++ libraries/audio/src/AudioScriptingInterface.h | 17 ++++++++ 5 files changed, 107 insertions(+) diff --git a/libraries/audio-client/src/AudioClient.cpp b/libraries/audio-client/src/AudioClient.cpp index 16b217bc53..6dc5b1380d 100644 --- a/libraries/audio-client/src/AudioClient.cpp +++ b/libraries/audio-client/src/AudioClient.cpp @@ -448,6 +448,34 @@ void AudioClient::setAudioPaused(bool pause) { } } +QUuid AudioClient::registerScriptListener() { + std::lock_guard lock(_scriptListenersMutex); + QUuid listenerID = QUuid::createUuid(); + while (_scriptListeners.contains(listenerID)) { + listenerID = QUuid::createUuid(); + } + _scriptListeners.insert(listenerID, std::make_shared()); + return listenerID; +} + +void AudioClient::unregisterScriptListener(const QUuid& listener) { + std::lock_guard lock(_scriptListenersMutex); +} + +QByteArray AudioClient::getPCMData(const QUuid& listenerID) { + std::shared_ptr listener; + { + std::lock_guard lock(_scriptListenersMutex); + if (_scriptListeners.contains(listenerID)) { + listener = _scriptListeners[listenerID]; + } else { + qDebug() << "AudioClient::getPCMData: Script listener doesn't exist: " << listenerID; + return QByteArray(); + } + } + return listener->getData(); +} + HifiAudioDeviceInfo getNamedAudioDeviceForMode(QAudio::Mode mode, const QString& deviceName, const QString& hmdName, bool isHmd=false) { HifiAudioDeviceInfo result; foreach (HifiAudioDeviceInfo audioDevice, getAvailableDevices(mode,hmdName)) { @@ -2450,6 +2478,18 @@ qint64 AudioClient::AudioOutputIODevice::readData(char * data, qint64 maxSize) { _audio->_audioFileWav.addRawAudioChunk(data, bytesWritten); } + // Send audio data to script engines. + // Hash table needs to be copied to avoid deadlocks and improve performance + QHash> scriptListeners; + { + std::lock_guard lock(_audio->_scriptListenersMutex); + scriptListeners = _audio->_scriptListeners; + } + QByteArray dataArray(data, bytesWritten); + for (auto listener : _audio->_scriptListeners) { + listener->putData(dataArray); + } + int bytesAudioOutputUnplayed = _audio->_audioOutput->bufferSize() - _audio->_audioOutput->bytesFree(); float msecsAudioOutputUnplayed = bytesAudioOutputUnplayed / (float)_audio->_outputFormat.bytesForDuration(USECS_PER_MSEC); _audio->_stats.updateOutputMsUnplayed(msecsAudioOutputUnplayed); diff --git a/libraries/audio-client/src/AudioClient.h b/libraries/audio-client/src/AudioClient.h index 9c25c2d4f2..a356cf9ed1 100644 --- a/libraries/audio-client/src/AudioClient.h +++ b/libraries/audio-client/src/AudioClient.h @@ -54,6 +54,7 @@ #include +#include "AudioScriptingInterface.h" #include "AudioIOStats.h" #include "AudioFileWav.h" #include "HifiAudioDeviceInfo.h" @@ -187,6 +188,10 @@ public: AudioSolo& getAudioSolo() override { return _solo; } + QUuid registerScriptListener() override; + void unregisterScriptListener(const QUuid& listenerID) override; + QByteArray getPCMData(const QUuid& listener) override; + #ifdef Q_OS_WIN static QString getWinDeviceName(wchar_t* guid); #endif @@ -535,6 +540,9 @@ private: Mutex _checkPeakValuesMutex; QTimer* _checkPeakValuesTimer { nullptr }; + Mutex _scriptListenersMutex; + QHash> _scriptListeners; + bool _isRecording { false }; }; diff --git a/libraries/audio/src/AbstractAudioInterface.h b/libraries/audio/src/AbstractAudioInterface.h index e9e40e95f9..11cebda877 100644 --- a/libraries/audio/src/AbstractAudioInterface.h +++ b/libraries/audio/src/AbstractAudioInterface.h @@ -41,6 +41,10 @@ public: virtual AudioSolo& getAudioSolo() = 0; + virtual QUuid registerScriptListener() = 0; + virtual void unregisterScriptListener(const QUuid& listener) = 0; + virtual QByteArray getPCMData(const QUuid& listener) = 0; + public slots: virtual bool shouldLoopbackInjectors() { return false; } diff --git a/libraries/audio/src/AudioScriptingInterface.cpp b/libraries/audio/src/AudioScriptingInterface.cpp index cc90c909ff..9500c32df2 100644 --- a/libraries/audio/src/AudioScriptingInterface.cpp +++ b/libraries/audio/src/AudioScriptingInterface.cpp @@ -40,6 +40,22 @@ void registerAudioMetaTypes(ScriptEngine* engine) { scriptRegisterMetaType(engine); } +QByteArray ScriptAudioListener::getData() { + QByteArray result; + { + std::lock_guard lock(_dataMutex); + while (!_data.empty()) { + result.append(_data.front()); + _data.pop(); + } + } + return result; +} + +void ScriptAudioListener::putData(const QByteArray &data) { + std::lock_guard lock(_dataMutex); + _data.push(data); +} void AudioScriptingInterface::setLocalAudioInterface(AbstractAudioInterface* audioInterface) { if (_localAudioInterface) { @@ -96,6 +112,28 @@ bool AudioScriptingInterface::isStereoInput() { return stereoEnabled; } +QUuid AudioScriptingInterface::registerScriptListener() { + if (_localAudioInterface) { + return _localAudioInterface->registerScriptListener(); + } else { + return QUuid(); + } +} + +void AudioScriptingInterface::unregisterScriptListener(const QUuid& listener) { + if (_localAudioInterface) { + _localAudioInterface->unregisterScriptListener(listener); + } +} + +QByteArray AudioScriptingInterface::getPCMData(const QUuid& listener) { + if (_localAudioInterface) { + return _localAudioInterface->getPCMData(listener); + } else { + return QByteArray(); + } +} + bool AudioScriptingInterface::getServerEcho() { bool serverEchoEnabled = false; if (_localAudioInterface) { diff --git a/libraries/audio/src/AudioScriptingInterface.h b/libraries/audio/src/AudioScriptingInterface.h index 1f43e5f45a..eaa942ee4b 100644 --- a/libraries/audio/src/AudioScriptingInterface.h +++ b/libraries/audio/src/AudioScriptingInterface.h @@ -17,6 +17,8 @@ #ifndef hifi_AudioScriptingInterface_h #define hifi_AudioScriptingInterface_h +#include + #include "AbstractAudioInterface.h" #include "AudioInjector.h" #include @@ -25,6 +27,17 @@ class ScriptAudioInjector; class ScriptEngine; +class ScriptAudioListener { + //TODO: add total data size limit + //TODO: unregister on script engine destruction +public: + QByteArray getData(); + void putData(const QByteArray &data); +private: + std::mutex _dataMutex; + std::queue _data; +}; + /// Provides the Audio scripting API class AudioScriptingInterface : public QObject, public Dependency { Q_OBJECT @@ -224,6 +237,10 @@ protected: */ Q_INVOKABLE bool isStereoInput(); + Q_INVOKABLE QUuid registerScriptListener(); + Q_INVOKABLE void unregisterScriptListener(const QUuid& listener); + Q_INVOKABLE QByteArray getPCMData(const QUuid& listener); + signals: /*@jsdoc