From aa77c4894c3ad5dfd39b3240ed327fd0a303683a Mon Sep 17 00:00:00 2001 From: Stephen Birarda Date: Mon, 16 Nov 2015 16:04:10 -0800 Subject: [PATCH] add an AudioInjectorManager for more efficient threading --- assignment-client/src/Agent.cpp | 2 +- examples/tests/injectorTest.js | 42 +++++++ interface/src/Application.cpp | 2 + libraries/audio/src/AudioInjector.cpp | 126 ++++++++++--------- libraries/audio/src/AudioInjector.h | 23 ++-- libraries/audio/src/AudioInjectorManager.cpp | 96 ++++++++++++++ libraries/audio/src/AudioInjectorManager.h | 53 ++++++++ 7 files changed, 276 insertions(+), 68 deletions(-) create mode 100644 examples/tests/injectorTest.js create mode 100644 libraries/audio/src/AudioInjectorManager.cpp create mode 100644 libraries/audio/src/AudioInjectorManager.h diff --git a/assignment-client/src/Agent.cpp b/assignment-client/src/Agent.cpp index 471de33602..7ee696693d 100644 --- a/assignment-client/src/Agent.cpp +++ b/assignment-client/src/Agent.cpp @@ -406,7 +406,7 @@ void Agent::sendPingRequests() { case NodeType::AvatarMixer: case NodeType::AudioMixer: case NodeType::EntityServer: - case NodeType::AssetClient: + case NodeType::AssetServer: return true; default: return false; diff --git a/examples/tests/injectorTest.js b/examples/tests/injectorTest.js new file mode 100644 index 0000000000..0ad4aad84c --- /dev/null +++ b/examples/tests/injectorTest.js @@ -0,0 +1,42 @@ +// +// injectorTests.js +// examples +// +// Created by Stephen Birarda on 11/16/15. +// Copyright 2014 High Fidelity, Inc. +// +// Distributed under the Apache License, Version 2.0. +// See the accompanying file LICENSE or http://www.apache.org/licenses/LICENSE-2.0.html +// + +var soundURL = "atp:44a83a788ccfd2924e35c902c34808b24dbd0309d000299ce01a355f91cf8115.wav"; +var audioOptions = { + position: MyAvatar.position, + volume: 0.5 +}; + +var sound = SoundCache.getSound(soundURL); +var injector = null; +var restarting = false; + +Script.update.connect(function(){ + if (sound.downloaded) { + if (!injector) { + injector = Audio.playSound(sound, audioOptions); + } else if (!injector.isPlaying && !restarting) { + restarting = true; + + Script.setTimeout(function(){ + print("Calling restart for a stopped injector from script."); + injector.restart(); + }, 1000); + } else if (injector.isPlaying) { + restarting = false; + + if (Math.random() < 0.0001) { + print("Calling restart for a running injector from script."); + injector.restart(); + } + } + } +}) diff --git a/interface/src/Application.cpp b/interface/src/Application.cpp index 7a564bbbf0..9bc0bf0179 100644 --- a/interface/src/Application.cpp +++ b/interface/src/Application.cpp @@ -50,6 +50,7 @@ #include #include #include +#include #include #include #include @@ -332,6 +333,7 @@ bool setupEssentials(int& argc, char** argv) { DependencyManager::set(); DependencyManager::set(); DependencyManager::set(); + DependencyManager::set(); DependencyManager::set(); DependencyManager::set(); return true; diff --git a/libraries/audio/src/AudioInjector.cpp b/libraries/audio/src/AudioInjector.cpp index bd807f8dbd..1a00611ee0 100644 --- a/libraries/audio/src/AudioInjector.cpp +++ b/libraries/audio/src/AudioInjector.cpp @@ -18,6 +18,7 @@ #include #include "AbstractAudioInterface.h" +#include "AudioInjectorManager.h" #include "AudioRingBuffer.h" #include "AudioLogging.h" #include "SoundCache.h" @@ -35,6 +36,7 @@ AudioInjector::AudioInjector(Sound* sound, const AudioInjectorOptions& injectorO _audioData(sound->getByteArray()), _options(injectorOptions) { + } AudioInjector::AudioInjector(const QByteArray& audioData, const AudioInjectorOptions& injectorOptions) : @@ -44,32 +46,27 @@ AudioInjector::AudioInjector(const QByteArray& audioData, const AudioInjectorOpt } -void AudioInjector::setIsFinished(bool isFinished) { - _isFinished = isFinished; - // In all paths, regardless of isFinished argument. restart() passes false to prepare for new play, and injectToMixer() needs _shouldStop reset. - _shouldStop = false; - - if (_isFinished) { - emit finished(); - - if (_localBuffer) { - _localBuffer->stop(); - _localBuffer->deleteLater(); - _localBuffer = NULL; - } - - _isStarted = false; - - if (_shouldDeleteAfterFinish) { - // we've been asked to delete after finishing, trigger a queued deleteLater here - QMetaObject::invokeMethod(this, "deleteLater", Qt::QueuedConnection); - } +void AudioInjector::finish() { + State oldState = std::atomic_exchange(&_state, State::Finished); + bool shouldDelete = (oldState == State::NotFinishedWithPendingDelete); + + emit finished(); + + if (_localBuffer) { + _localBuffer->stop(); + _localBuffer->deleteLater(); + _localBuffer = NULL; + } + + if (shouldDelete) { + // we've been asked to delete after finishing, trigger a queued deleteLater here + QMetaObject::invokeMethod(this, "deleteLater", Qt::QueuedConnection); } } void AudioInjector::injectAudio() { - if (!_isStarted) { - _isStarted = true; + if (!_hasStarted) { + _hasStarted = true; // check if we need to offset the sound by some number of seconds if (_options.secondOffset > 0.0f) { @@ -85,6 +82,7 @@ void AudioInjector::injectAudio() { if (_options.localOnly) { injectLocally(); } else { + qDebug() << "Calling inject to mixer from" << QThread::currentThread(); injectToMixer(); } } else { @@ -93,18 +91,26 @@ void AudioInjector::injectAudio() { } void AudioInjector::restart() { - _isPlaying = true; - connect(this, &AudioInjector::finished, this, &AudioInjector::restartPortionAfterFinished); - if (!_isStarted || _isFinished) { - emit finished(); - } else { - stop(); + if (thread() != QThread::currentThread()) { + QMetaObject::invokeMethod(this, "restart"); + return; + } + + // reset the current send offset to zero + _currentSendOffset = 0; + + // check our state to decide if we need extra handling for the restart request + if (!isPlaying()) { + // we finished playing, need to reset state so we can get going again + _hasStarted = false; + _shouldStop = false; + _state = State::NotFinished; + + qDebug() << "Calling inject audio again to restart an injector"; + + // call inject audio to start injection over again + injectAudio(); } -} -void AudioInjector::restartPortionAfterFinished() { - disconnect(this, &AudioInjector::finished, this, &AudioInjector::restartPortionAfterFinished); - setIsFinished(false); - QMetaObject::invokeMethod(this, "injectAudio", Qt::QueuedConnection); } void AudioInjector::injectLocally() { @@ -252,16 +258,17 @@ void AudioInjector::injectToMixer() { if (_currentSendOffset != bytesToCopy && _currentSendOffset < _audioData.size()) { - // process events in case we have been told to stop and be deleted - QCoreApplication::processEvents(); + if (_shouldStop) { break; } - + // not the first packet and not done // sleep for the appropriate time - int usecToSleep = (++nextFrame * (_options.stereo ? 2 : 1) * AudioConstants::NETWORK_FRAME_USECS) - timer.nsecsElapsed() / 1000; + int usecToSleep = (++nextFrame * AudioConstants::NETWORK_FRAME_USECS) - timer.nsecsElapsed() / 1000; + + qDebug() << "AudioInjector" << this << "will sleep on thread" << QThread::currentThread() << "for" << usecToSleep; if (usecToSleep > 0) { usleep(usecToSleep); @@ -274,8 +281,7 @@ void AudioInjector::injectToMixer() { } } - setIsFinished(true); - _isPlaying = !_isFinished; // Which can be false if a restart was requested + finish(); } void AudioInjector::stop() { @@ -283,8 +289,14 @@ void AudioInjector::stop() { if (_options.localOnly) { // we're only a local injector, so we can say we are finished right away too - _isPlaying = false; - setIsFinished(true); + finish(); + } +} + +void AudioInjector::triggerDeleteAfterFinish() { + auto expectedState = State::NotFinished; + if (!_state.compare_exchange_strong(expectedState, State::NotFinishedWithPendingDelete)) { + stopAndDeleteLater(); } } @@ -336,28 +348,26 @@ AudioInjector* AudioInjector::playSound(const QString& soundUrl, const float vol AudioInjector* AudioInjector::playSoundAndDelete(const QByteArray& buffer, const AudioInjectorOptions options, AbstractAudioInterface* localInterface) { AudioInjector* sound = playSound(buffer, options, localInterface); - sound->triggerDeleteAfterFinish(); + sound->_state = AudioInjector::State::NotFinishedWithPendingDelete; return sound; } AudioInjector* AudioInjector::playSound(const QByteArray& buffer, const AudioInjectorOptions options, AbstractAudioInterface* localInterface) { - QThread* injectorThread = new QThread(); - injectorThread->setObjectName("Audio Injector Thread"); - AudioInjector* injector = new AudioInjector(buffer, options); - injector->_isPlaying = true; injector->setLocalAudioInterface(localInterface); - - injector->moveToThread(injectorThread); - - // start injecting when the injector thread starts - connect(injectorThread, &QThread::started, injector, &AudioInjector::injectAudio); - - // connect the right slots and signals for AudioInjector and thread cleanup - connect(injector, &AudioInjector::destroyed, injectorThread, &QThread::quit); - connect(injectorThread, &QThread::finished, injectorThread, &QThread::deleteLater); - - injectorThread->start(); - return injector; + + // grab the AudioInjectorManager + auto injectorManager = DependencyManager::get(); + + // attempt to thread the new injector + if (injectorManager->threadInjector(injector)) { + // call inject audio on the correct thread + QMetaObject::invokeMethod(injector, "injectAudio", Qt::QueuedConnection); + + return injector; + } else { + // we failed to thread the new injector (we are at the max number of injector threads) + return nullptr; + } } diff --git a/libraries/audio/src/AudioInjector.h b/libraries/audio/src/AudioInjector.h index 0e98fe1682..25b814f0f6 100644 --- a/libraries/audio/src/AudioInjector.h +++ b/libraries/audio/src/AudioInjector.h @@ -12,6 +12,8 @@ #ifndef hifi_AudioInjector_h #define hifi_AudioInjector_h +#include + #include #include #include @@ -32,11 +34,17 @@ class AudioInjector : public QObject { Q_OBJECT public: + enum class State : uint8_t { + NotFinished, + NotFinishedWithPendingDelete, + Finished + }; + AudioInjector(QObject* parent); AudioInjector(Sound* sound, const AudioInjectorOptions& injectorOptions); AudioInjector(const QByteArray& audioData, const AudioInjectorOptions& injectorOptions); - bool isFinished() const { return _isFinished; } + bool isFinished() const { return _state == State::Finished; } int getCurrentSendOffset() const { return _currentSendOffset; } void setCurrentSendOffset(int currentSendOffset) { _currentSendOffset = currentSendOffset; } @@ -55,15 +63,14 @@ public slots: void restart(); void stop(); - void triggerDeleteAfterFinish() { _shouldDeleteAfterFinish = true; } + void triggerDeleteAfterFinish(); void stopAndDeleteLater(); const AudioInjectorOptions& getOptions() const { return _options; } void setOptions(const AudioInjectorOptions& options) { _options = options; } float getLoudness() const { return _loudness; } - bool isPlaying() const { return _isPlaying; } - void restartPortionAfterFinished(); + bool isPlaying() const { return _state == State::NotFinished || _state == State::NotFinishedWithPendingDelete; } signals: void finished(); @@ -72,16 +79,14 @@ private: void injectToMixer(); void injectLocally(); - void setIsFinished(bool isFinished); + void finish(); QByteArray _audioData; AudioInjectorOptions _options; + std::atomic _state { State::NotFinished }; + bool _hasStarted = false; bool _shouldStop = false; float _loudness = 0.0f; - bool _isPlaying = false; - bool _isStarted = false; - bool _isFinished = false; - bool _shouldDeleteAfterFinish = false; int _currentSendOffset = 0; AbstractAudioInterface* _localAudioInterface = NULL; AudioInjectorLocalBuffer* _localBuffer = NULL; diff --git a/libraries/audio/src/AudioInjectorManager.cpp b/libraries/audio/src/AudioInjectorManager.cpp new file mode 100644 index 0000000000..8eda670ddf --- /dev/null +++ b/libraries/audio/src/AudioInjectorManager.cpp @@ -0,0 +1,96 @@ +// +// AudioInjectorManager.cpp +// libraries/audio/src +// +// Created by Stephen Birarda on 2015-11-16. +// Copyright 2015 High Fidelity, Inc. +// +// Distributed under the Apache License, Version 2.0. +// See the accompanying file LICENSE or http://www.apache.org/licenses/LICENSE-2.0.html +// + +#include "AudioInjectorManager.h" + +#include + +#include "AudioInjector.h" + +AudioInjectorManager::~AudioInjectorManager() { + _shouldStop = true; + + // make sure any still living injectors are stopped and deleted + for (auto injector : _injectors) { + injector->stopAndDeleteLater(); + } + + // quit and wait on the injector thread, if we ever created it + if (_thread) { + _thread->quit(); + _thread->wait(); + } +} + +void AudioInjectorManager::createThread() { + _thread = new QThread; + _thread->setObjectName("Audio Injector Thread"); + + // when the thread is started, have it call our run to handle injection of audio + connect(_thread, &QThread::started, this, &AudioInjectorManager::run); + + // start the thread + _thread->start(); +} + +void AudioInjectorManager::run() { + while (!_shouldStop) { + // process events in case we have been told to stop or our injectors have been told to stop + QCoreApplication::processEvents(); + } +} + +static const int MAX_INJECTORS_PER_THREAD = 50; // calculated based on AudioInjector while loop time, with sufficient padding + +bool AudioInjectorManager::threadInjector(AudioInjector* injector) { + // guard the injectors vector with a mutex + std::unique_lock lock(_injectorsMutex); + + // check if we'll be able to thread this injector (do we have < MAX concurrent injectors) + if (_injectors.size() < MAX_INJECTORS_PER_THREAD) { + if (!_thread) { + createThread(); + } + + auto it = nextInjectorIterator(); + qDebug() << "Inserting injector at" << it - _injectors.begin(); + + // store a QPointer to this injector + _injectors.insert(it, QPointer(injector)); + + qDebug() << "Moving injector to thread" << _thread; + + // move the injector to the QThread + injector->moveToThread(_thread); + + return true; + } else { + // unable to thread this injector, at the max + qDebug() << "AudioInjectorManager::threadInjector could not thread AudioInjector - at max of" + << MAX_INJECTORS_PER_THREAD << "current audio injectors."; + return false; + } +} + +AudioInjectorVector::iterator AudioInjectorManager::nextInjectorIterator() { + // find the next usable iterator for an injector + auto it = _injectors.begin(); + + while (it != _injectors.end()) { + if (it->isNull()) { + return it; + } else { + ++it; + } + } + + return it; +} diff --git a/libraries/audio/src/AudioInjectorManager.h b/libraries/audio/src/AudioInjectorManager.h new file mode 100644 index 0000000000..4e571d2d8d --- /dev/null +++ b/libraries/audio/src/AudioInjectorManager.h @@ -0,0 +1,53 @@ +// +// AudioInjectorManager.h +// libraries/audio/src +// +// Created by Stephen Birarda on 2015-11-16. +// Copyright 2015 High Fidelity, Inc. +// +// Distributed under the Apache License, Version 2.0. +// See the accompanying file LICENSE or http://www.apache.org/licenses/LICENSE-2.0.html +// + +#pragma once + +#ifndef hifi_AudioInjectorManager_h +#define hifi_AudioInjectorManager_h + +#include +#include + +#include +#include + +#include + +class AudioInjector; +using AudioInjectorVector = std::vector>; + +class AudioInjectorManager : public QObject, public Dependency { + Q_OBJECT + SINGLETON_DEPENDENCY +public: + ~AudioInjectorManager(); +private slots: + void run(); +private: + bool threadInjector(AudioInjector* injector); + + AudioInjectorManager() {}; + AudioInjectorManager(const AudioInjectorManager&) = delete; + AudioInjectorManager& operator=(const AudioInjectorManager&) = delete; + + void createThread(); + AudioInjectorVector::iterator nextInjectorIterator(); + + QThread* _thread { nullptr }; + bool _shouldStop { false }; + AudioInjectorVector _injectors; + std::mutex _injectorsMutex; + + friend class AudioInjector; +}; + +#endif // hifi_AudioInjectorManager_h