From f1a238a6c2280cc7d1f403cf8ce63b2ce694fd09 Mon Sep 17 00:00:00 2001 From: Stephen Birarda Date: Fri, 7 Nov 2014 10:08:09 -0800 Subject: [PATCH 1/2] piping for local UI sounds --- interface/src/Application.cpp | 3 +++ interface/src/Audio.cpp | 3 ++- interface/src/Audio.h | 2 +- libraries/audio/src/AbstractAudioInterface.h | 4 +++- libraries/audio/src/AudioInjector.cpp | 10 +--------- libraries/audio/src/AudioInjectorOptions.cpp | 5 +---- libraries/audio/src/AudioInjectorOptions.h | 8 ++------ .../audio/src/AudioScriptingInterface.cpp | 18 ++++++++++++++++++ libraries/audio/src/AudioScriptingInterface.h | 10 +++++++++- 9 files changed, 40 insertions(+), 23 deletions(-) diff --git a/interface/src/Application.cpp b/interface/src/Application.cpp index a62ce0e209..e58f79cb5e 100644 --- a/interface/src/Application.cpp +++ b/interface/src/Application.cpp @@ -416,6 +416,9 @@ Application::Application(int& argc, char** argv, QElapsedTimer &startup_time) : _trayIcon->show(); + // set the local loopback interface for local sounds from audio scripts + AudioScriptingInterface::getInstance().setLocalLoopbackInterface(&_audio); + #ifdef HAVE_RTMIDI // setup the MIDIManager MIDIManager& midiManagerInstance = MIDIManager::getInstance(); diff --git a/interface/src/Audio.cpp b/interface/src/Audio.cpp index 8c207c3d21..4bd9f35887 100644 --- a/interface/src/Audio.cpp +++ b/interface/src/Audio.cpp @@ -1331,9 +1331,10 @@ void Audio::startDrumSound(float volume, float frequency, float duration, float _drumSoundSample = 0; } -void Audio::handleAudioByteArray(const QByteArray& audioByteArray) { +void Audio::handleAudioByteArray(const QByteArray& audioByteArray, const AudioInjectorOptions& injectorOptions) { // TODO: either create a new audio device (up to the limit of the sound card or a hard limit) // or send to the mixer and use delayed loopback + } void Audio::renderToolBox(int x, int y, bool boxed) { diff --git a/interface/src/Audio.h b/interface/src/Audio.h index 0924f65fab..bec2ce32a0 100644 --- a/interface/src/Audio.h +++ b/interface/src/Audio.h @@ -155,7 +155,7 @@ public slots: void selectAudioFilterBassCut(); void selectAudioFilterSmiley(); - virtual void handleAudioByteArray(const QByteArray& audioByteArray); + virtual void handleAudioByteArray(const QByteArray& audioByteArray, const AudioInjectorOptions& options); void sendDownstreamAudioStatsPacket(); diff --git a/libraries/audio/src/AbstractAudioInterface.h b/libraries/audio/src/AbstractAudioInterface.h index f950eefcbd..1ee71ee32d 100644 --- a/libraries/audio/src/AbstractAudioInterface.h +++ b/libraries/audio/src/AbstractAudioInterface.h @@ -14,6 +14,8 @@ #include +#include "AudioInjectorOptions.h" + class AbstractAudioInterface : public QObject { Q_OBJECT public: @@ -22,7 +24,7 @@ public: virtual void startCollisionSound(float magnitude, float frequency, float noise, float duration, bool flashScreen) = 0; virtual void startDrumSound(float volume, float frequency, float duration, float decay) = 0; public slots: - virtual void handleAudioByteArray(const QByteArray& audioByteArray) = 0; + virtual void handleAudioByteArray(const QByteArray& audioByteArray, const AudioInjectorOptions& options) = 0; }; Q_DECLARE_METATYPE(AbstractAudioInterface*) diff --git a/libraries/audio/src/AudioInjector.cpp b/libraries/audio/src/AudioInjector.cpp index c61a00d218..45e1eb57e1 100644 --- a/libraries/audio/src/AudioInjector.cpp +++ b/libraries/audio/src/AudioInjector.cpp @@ -62,14 +62,6 @@ void AudioInjector::injectAudio() { // make sure we actually have samples downloaded to inject if (soundByteArray.size()) { - // give our sample byte array to the local audio interface, if we have it, so it can be handled locally - if (_options.getLoopbackAudioInterface()) { - // assume that localAudioInterface could be on a separate thread, use Qt::AutoConnection to handle properly - QMetaObject::invokeMethod(_options.getLoopbackAudioInterface(), "handleAudioByteArray", - Qt::AutoConnection, - Q_ARG(QByteArray, soundByteArray)); - - } // setup the packet for injected audio QByteArray injectAudioPacket = byteArrayWithPopulatedHeader(PacketTypeInjectAudio); @@ -86,7 +78,7 @@ void AudioInjector::injectAudio() { packetStream << _options.isStereo(); // pack the flag for loopback - uchar loopbackFlag = (uchar) (!_options.getLoopbackAudioInterface()); + uchar loopbackFlag = (uchar) true; packetStream << loopbackFlag; // pack the position for injected audio diff --git a/libraries/audio/src/AudioInjectorOptions.cpp b/libraries/audio/src/AudioInjectorOptions.cpp index a930e761ab..670bea2fef 100644 --- a/libraries/audio/src/AudioInjectorOptions.cpp +++ b/libraries/audio/src/AudioInjectorOptions.cpp @@ -18,8 +18,7 @@ AudioInjectorOptions::AudioInjectorOptions(QObject* parent) : _loop(false), _orientation(glm::vec3(0.0f, 0.0f, 0.0f)), _isStereo(false), - _ignorePenumbra(false), - _loopbackAudioInterface(NULL) + _ignorePenumbra(false) { } @@ -30,7 +29,6 @@ AudioInjectorOptions::AudioInjectorOptions(const AudioInjectorOptions& other) { _orientation = other._orientation; _isStereo = other._isStereo; _ignorePenumbra = other._ignorePenumbra; - _loopbackAudioInterface = other._loopbackAudioInterface; } void AudioInjectorOptions::operator=(const AudioInjectorOptions& other) { @@ -40,5 +38,4 @@ void AudioInjectorOptions::operator=(const AudioInjectorOptions& other) { _orientation = other._orientation; _isStereo = other._isStereo; _ignorePenumbra = other._ignorePenumbra; - _loopbackAudioInterface = other._loopbackAudioInterface; } \ No newline at end of file diff --git a/libraries/audio/src/AudioInjectorOptions.h b/libraries/audio/src/AudioInjectorOptions.h index e59517c10b..1ccd85be7e 100644 --- a/libraries/audio/src/AudioInjectorOptions.h +++ b/libraries/audio/src/AudioInjectorOptions.h @@ -19,8 +19,6 @@ #include -#include "AbstractAudioInterface.h" - class AudioInjectorOptions : public QObject { Q_OBJECT @@ -53,9 +51,6 @@ public: const bool ignorePenumbra() const {return _ignorePenumbra; } void setIgnorePenumbra(bool ignorePenumbra) { _ignorePenumbra = ignorePenumbra; } - AbstractAudioInterface* getLoopbackAudioInterface() const { return _loopbackAudioInterface; } - void setLoopbackAudioInterface(AbstractAudioInterface* loopbackAudioInterface) - { _loopbackAudioInterface = loopbackAudioInterface; } private: glm::vec3 _position; float _volume; @@ -63,7 +58,8 @@ private: glm::quat _orientation; bool _isStereo; bool _ignorePenumbra; - AbstractAudioInterface* _loopbackAudioInterface; }; +Q_DECLARE_METATYPE(AudioInjectorOptions) + #endif // hifi_AudioInjectorOptions_h diff --git a/libraries/audio/src/AudioScriptingInterface.cpp b/libraries/audio/src/AudioScriptingInterface.cpp index 418d25a563..33afcdb095 100644 --- a/libraries/audio/src/AudioScriptingInterface.cpp +++ b/libraries/audio/src/AudioScriptingInterface.cpp @@ -16,6 +16,12 @@ AudioScriptingInterface& AudioScriptingInterface::getInstance() { return staticInstance; } +AudioScriptingInterface::AudioScriptingInterface() : + _localLoopbackInterface(NULL) +{ + qRegisterMetaType("AudioInjectorOptions"); +} + void AudioScriptingInterface::stopAllInjectors() { QList >::iterator injector = _activeInjectors.begin(); while (injector != _activeInjectors.end()) { @@ -31,6 +37,18 @@ void AudioScriptingInterface::stopAllInjectors() { } } +void AudioScriptingInterface::playLocalSound(Sound* sound, const AudioInjectorOptions* injectorOptions) { + if (sound->isStereo()) { + const_cast(injectorOptions)->setIsStereo(true); + } + + // assume that localAudioInterface could be on a separate thread, use Qt::AutoConnection to handle properly + QMetaObject::invokeMethod(_localLoopbackInterface, "handleAudioByteArray", + Qt::AutoConnection, + Q_ARG(QByteArray, sound->getByteArray()), + Q_ARG(const AudioInjectorOptions&, *injectorOptions)); +} + AudioInjector* AudioScriptingInterface::playSound(Sound* sound, const AudioInjectorOptions* injectorOptions) { if (sound->isStereo()) { diff --git a/libraries/audio/src/AudioScriptingInterface.h b/libraries/audio/src/AudioScriptingInterface.h index 8a49befe7b..709b82323c 100644 --- a/libraries/audio/src/AudioScriptingInterface.h +++ b/libraries/audio/src/AudioScriptingInterface.h @@ -14,9 +14,12 @@ #include +#include "AbstractAudioInterface.h" #include "AudioInjector.h" #include "Sound.h" +class AbstractAudioInterface; + const AudioInjectorOptions DEFAULT_INJECTOR_OPTIONS; class AudioScriptingInterface : public QObject { @@ -25,19 +28,24 @@ public: static AudioScriptingInterface& getInstance(); void stopAllInjectors(); + + void setLocalLoopbackInterface(AbstractAudioInterface* audioInterface) { _localLoopbackInterface = audioInterface; } public slots: static float getLoudness(AudioInjector* injector); + void playLocalSound(Sound *sound, const AudioInjectorOptions* injectorOptions = NULL); AudioInjector* playSound(Sound* sound, const AudioInjectorOptions* injectorOptions = NULL); + void stopInjector(AudioInjector* injector); bool isInjectorPlaying(AudioInjector* injector); void injectorStopped(); private: - AudioScriptingInterface() {}; + AudioScriptingInterface(); QList< QPointer > _activeInjectors; + AbstractAudioInterface* _localLoopbackInterface; }; From bd8b71a02af9b34a7359ad87ad3559692ecbe6f5 Mon Sep 17 00:00:00 2001 From: Stephen Birarda Date: Fri, 7 Nov 2014 11:05:01 -0800 Subject: [PATCH 2/2] complete handling of local audio loopback --- interface/src/Audio.cpp | 22 +++++++++++++++---- libraries/audio/src/AudioScriptingInterface.h | 4 ---- 2 files changed, 18 insertions(+), 8 deletions(-) diff --git a/interface/src/Audio.cpp b/interface/src/Audio.cpp index 4bd9f35887..3039b7adfb 100644 --- a/interface/src/Audio.cpp +++ b/interface/src/Audio.cpp @@ -184,9 +184,9 @@ QAudioDeviceInfo getNamedAudioDeviceForMode(QAudio::Mode mode, const QString& de QAudioDeviceInfo::defaultOutputDevice(); #else foreach(QAudioDeviceInfo audioDevice, QAudioDeviceInfo::availableDevices(mode)) { - qDebug() << audioDevice.deviceName() << " " << deviceName; if (audioDevice.deviceName().trimmed() == deviceName.trimmed()) { result = audioDevice; + break; } } #endif @@ -1332,9 +1332,21 @@ void Audio::startDrumSound(float volume, float frequency, float duration, float } void Audio::handleAudioByteArray(const QByteArray& audioByteArray, const AudioInjectorOptions& injectorOptions) { - // TODO: either create a new audio device (up to the limit of the sound card or a hard limit) - // or send to the mixer and use delayed loopback - + if (audioByteArray.size() > 0) { + QAudioFormat localFormat = _outputFormat; + + if (!injectorOptions.isStereo()) { + localFormat.setChannelCount(1); + } + + QAudioOutput* localSoundOutput = new QAudioOutput(getNamedAudioDeviceForMode(QAudio::AudioOutput, _outputAudioDeviceName), localFormat, this); + + QIODevice* localIODevice = localSoundOutput->start(); + qDebug() << "Writing" << audioByteArray.size() << "to" << localIODevice; + localIODevice->write(audioByteArray); + } else { + qDebug() << "Audio::handleAudioByteArray called with an empty byte array. Sound is likely still downloading."; + } } void Audio::renderToolBox(int x, int y, bool boxed) { @@ -1904,6 +1916,7 @@ bool Audio::switchInputToAudioDevice(const QAudioDeviceInfo& inputDeviceInfo) { } } } + return supportedFormat; } @@ -1962,6 +1975,7 @@ bool Audio::switchOutputToAudioDevice(const QAudioDeviceInfo& outputDeviceInfo) supportedFormat = true; } } + return supportedFormat; } diff --git a/libraries/audio/src/AudioScriptingInterface.h b/libraries/audio/src/AudioScriptingInterface.h index 709b82323c..5cd0f9e99a 100644 --- a/libraries/audio/src/AudioScriptingInterface.h +++ b/libraries/audio/src/AudioScriptingInterface.h @@ -18,8 +18,6 @@ #include "AudioInjector.h" #include "Sound.h" -class AbstractAudioInterface; - const AudioInjectorOptions DEFAULT_INJECTOR_OPTIONS; class AudioScriptingInterface : public QObject { @@ -46,7 +44,5 @@ private: AudioScriptingInterface(); QList< QPointer > _activeInjectors; AbstractAudioInterface* _localLoopbackInterface; - - }; #endif // hifi_AudioScriptingInterface_h