From d941be8d0bcd1c59367f42159f072aafe8c5b60b Mon Sep 17 00:00:00 2001 From: Stephen Birarda Date: Tue, 20 Oct 2015 17:11:37 -0700 Subject: [PATCH 01/25] allow assignment client to use AssetClient for ATP downloads --- assignment-client/src/Agent.cpp | 15 +++++++++------ 1 file changed, 9 insertions(+), 6 deletions(-) diff --git a/assignment-client/src/Agent.cpp b/assignment-client/src/Agent.cpp index 063bf24de8..cfc641f12b 100644 --- a/assignment-client/src/Agent.cpp +++ b/assignment-client/src/Agent.cpp @@ -16,6 +16,7 @@ #include #include +#include #include #include #include @@ -42,7 +43,8 @@ Agent::Agent(NLPacket& packet) : DEFAULT_WINDOW_SECONDS_FOR_DESIRED_REDUCTION, false)) { DependencyManager::get()->setPacketSender(&_entityEditSender); - + + DependencyManager::set(); DependencyManager::set(); DependencyManager::set(); @@ -110,11 +112,12 @@ void Agent::run() { ThreadedAssignment::commonInit(AGENT_LOGGING_NAME, NodeType::Agent); auto nodeList = DependencyManager::get(); - nodeList->addSetOfNodeTypesToNodeInterestSet(NodeSet() - << NodeType::AudioMixer - << NodeType::AvatarMixer - << NodeType::EntityServer - ); + nodeList->addSetOfNodeTypesToNodeInterestSet({ + NodeType::AudioMixer, + NodeType::AvatarMixer, + NodeType::EntityServer, + NodeType::AssetServer + }); _pingTimer = new QTimer(this); connect(_pingTimer, SIGNAL(timeout()), SLOT(sendPingRequests())); From 5908f2d398952cf5b487ff6093b5e5d3303cf533 Mon Sep 17 00:00:00 2001 From: Stephen Birarda Date: Mon, 26 Oct 2015 12:06:25 -0700 Subject: [PATCH 02/25] put Agent AssetClient on sep thread, cleanup --- assignment-client/src/Agent.cpp | 16 +++++++++++++++- 1 file changed, 15 insertions(+), 1 deletion(-) diff --git a/assignment-client/src/Agent.cpp b/assignment-client/src/Agent.cpp index cfc641f12b..0d69a645c3 100644 --- a/assignment-client/src/Agent.cpp +++ b/assignment-client/src/Agent.cpp @@ -44,7 +44,14 @@ Agent::Agent(NLPacket& packet) : { DependencyManager::get()->setPacketSender(&_entityEditSender); - DependencyManager::set(); + auto assetClient = DependencyManager::set(); + + QThread* assetThread = new QThread; + assetThread->setObjectName("Asset Thread"); + assetClient->moveToThread(assetThread); + connect(assetThread, &QThread::started, assetClient.data(), &AssetClient::init); + assetThread->start(); + DependencyManager::set(); DependencyManager::set(); @@ -371,6 +378,7 @@ void Agent::processAgentAvatarAndAudio(float deltaTime) { void Agent::aboutToFinish() { setIsAvatar(false);// will stop timers for sending billboards and identity packets + if (_scriptEngine) { _scriptEngine->stop(); } @@ -382,6 +390,12 @@ void Agent::aboutToFinish() { // our entity tree is going to go away so tell that to the EntityScriptingInterface DependencyManager::get()->setEntityTree(NULL); + + // cleanup the AssetClient thread + QThread* assetThread = DependencyManager::get()->thread(); + DependencyManager::destroy(); + assetThread->quit(); + assetThread->wait(); } void Agent::sendPingRequests() { From 3fa0bcbcd9d56a0bcd025e1de36fac06c62dabc5 Mon Sep 17 00:00:00 2001 From: Stephen Birarda Date: Mon, 26 Oct 2015 16:06:12 -0700 Subject: [PATCH 03/25] ping the AssetClient from the agent --- assignment-client/src/Agent.cpp | 29 +++++++++++++++-------------- 1 file changed, 15 insertions(+), 14 deletions(-) diff --git a/assignment-client/src/Agent.cpp b/assignment-client/src/Agent.cpp index 0d69a645c3..471de33602 100644 --- a/assignment-client/src/Agent.cpp +++ b/assignment-client/src/Agent.cpp @@ -43,20 +43,20 @@ Agent::Agent(NLPacket& packet) : DEFAULT_WINDOW_SECONDS_FOR_DESIRED_REDUCTION, false)) { DependencyManager::get()->setPacketSender(&_entityEditSender); - + auto assetClient = DependencyManager::set(); - + QThread* assetThread = new QThread; assetThread->setObjectName("Asset Thread"); assetClient->moveToThread(assetThread); connect(assetThread, &QThread::started, assetClient.data(), &AssetClient::init); assetThread->start(); - + DependencyManager::set(); DependencyManager::set(); auto& packetReceiver = DependencyManager::get()->getPacketReceiver(); - + packetReceiver.registerListenerForTypes( { PacketType::MixedAudio, PacketType::SilentAudioFrame }, this, "handleAudioPacket"); @@ -75,7 +75,7 @@ void Agent::handleOctreePacket(QSharedPointer packet, SharedNodePointe if (packet->getPayloadSize() > statsMessageLength) { // pull out the piggybacked packet and create a new QSharedPointer for it int piggyBackedSizeWithHeader = packet->getPayloadSize() - statsMessageLength; - + auto buffer = std::unique_ptr(new char[piggyBackedSizeWithHeader]); memcpy(buffer.get(), packet->getPayload() + statsMessageLength, piggyBackedSizeWithHeader); @@ -102,7 +102,7 @@ void Agent::handleJurisdictionPacket(QSharedPointer packet, SharedNode DependencyManager::get()->getJurisdictionListener()-> queueReceivedPacket(packet, senderNode); } -} +} void Agent::handleAudioPacket(QSharedPointer packet) { _receivedAudioStream.parseData(*packet); @@ -176,7 +176,7 @@ void Agent::run() { // give this AvatarData object to the script engine setAvatarData(&scriptedAvatar, "Avatar"); - + auto avatarHashMap = DependencyManager::set(); _scriptEngine->registerGlobalObject("AvatarList", avatarHashMap.data()); @@ -189,7 +189,7 @@ void Agent::run() { // register ourselves to the script engine _scriptEngine->registerGlobalObject("Agent", this); - // FIXME -we shouldn't be calling this directly, it's normally called by run(), not sure why + // FIXME -we shouldn't be calling this directly, it's normally called by run(), not sure why // viewers would need this called. //_scriptEngine->init(); // must be done before we set up the viewers @@ -201,14 +201,14 @@ void Agent::run() { auto entityScriptingInterface = DependencyManager::get(); _scriptEngine->registerGlobalObject("EntityViewer", &_entityViewer); - + // we need to make sure that init has been called for our EntityScriptingInterface // so that it actually has a jurisdiction listener when we ask it for it next entityScriptingInterface->init(); _entityViewer.setJurisdictionListener(entityScriptingInterface->getJurisdictionListener()); - + _entityViewer.init(); - + entityScriptingInterface->setEntityTree(_entityViewer.getTree()); // wire up our additional agent related processing to the update signal @@ -241,7 +241,7 @@ void Agent::setIsAvatar(bool isAvatar) { delete _avatarIdentityTimer; _avatarIdentityTimer = nullptr; } - + if (_avatarBillboardTimer) { _avatarBillboardTimer->stop(); delete _avatarBillboardTimer; @@ -378,7 +378,7 @@ void Agent::processAgentAvatarAndAudio(float deltaTime) { void Agent::aboutToFinish() { setIsAvatar(false);// will stop timers for sending billboards and identity packets - + if (_scriptEngine) { _scriptEngine->stop(); } @@ -390,7 +390,7 @@ void Agent::aboutToFinish() { // our entity tree is going to go away so tell that to the EntityScriptingInterface DependencyManager::get()->setEntityTree(NULL); - + // cleanup the AssetClient thread QThread* assetThread = DependencyManager::get()->thread(); DependencyManager::destroy(); @@ -406,6 +406,7 @@ void Agent::sendPingRequests() { case NodeType::AvatarMixer: case NodeType::AudioMixer: case NodeType::EntityServer: + case NodeType::AssetClient: return true; default: return false; From 52716b8f9558c0fd1264fce416131a50b3ea02f8 Mon Sep 17 00:00:00 2001 From: Stephen Birarda Date: Mon, 16 Nov 2015 10:55:25 -0800 Subject: [PATCH 04/25] emit a ready signal if Sound has downloaded --- libraries/audio/src/Sound.cpp | 1 + libraries/audio/src/Sound.h | 3 +++ 2 files changed, 4 insertions(+) diff --git a/libraries/audio/src/Sound.cpp b/libraries/audio/src/Sound.cpp index 2ce2c47fef..12f63e0a12 100644 --- a/libraries/audio/src/Sound.cpp +++ b/libraries/audio/src/Sound.cpp @@ -86,6 +86,7 @@ void Sound::downloadFinished(const QByteArray& data) { } _isReady = true; + emit ready(); } void Sound::downSample(const QByteArray& rawAudioByteArray) { diff --git a/libraries/audio/src/Sound.h b/libraries/audio/src/Sound.h index 842c395a7d..91dbef8c6a 100644 --- a/libraries/audio/src/Sound.h +++ b/libraries/audio/src/Sound.h @@ -30,6 +30,9 @@ public: const QByteArray& getByteArray() { return _byteArray; } +signals: + void ready(); + private: QByteArray _byteArray; bool _isStereo; From aa77c4894c3ad5dfd39b3240ed327fd0a303683a Mon Sep 17 00:00:00 2001 From: Stephen Birarda Date: Mon, 16 Nov 2015 16:04:10 -0800 Subject: [PATCH 05/25] add an AudioInjectorManager for more efficient threading --- assignment-client/src/Agent.cpp | 2 +- examples/tests/injectorTest.js | 42 +++++++ interface/src/Application.cpp | 2 + libraries/audio/src/AudioInjector.cpp | 126 ++++++++++--------- libraries/audio/src/AudioInjector.h | 23 ++-- libraries/audio/src/AudioInjectorManager.cpp | 96 ++++++++++++++ libraries/audio/src/AudioInjectorManager.h | 53 ++++++++ 7 files changed, 276 insertions(+), 68 deletions(-) create mode 100644 examples/tests/injectorTest.js create mode 100644 libraries/audio/src/AudioInjectorManager.cpp create mode 100644 libraries/audio/src/AudioInjectorManager.h diff --git a/assignment-client/src/Agent.cpp b/assignment-client/src/Agent.cpp index 471de33602..7ee696693d 100644 --- a/assignment-client/src/Agent.cpp +++ b/assignment-client/src/Agent.cpp @@ -406,7 +406,7 @@ void Agent::sendPingRequests() { case NodeType::AvatarMixer: case NodeType::AudioMixer: case NodeType::EntityServer: - case NodeType::AssetClient: + case NodeType::AssetServer: return true; default: return false; diff --git a/examples/tests/injectorTest.js b/examples/tests/injectorTest.js new file mode 100644 index 0000000000..0ad4aad84c --- /dev/null +++ b/examples/tests/injectorTest.js @@ -0,0 +1,42 @@ +// +// injectorTests.js +// examples +// +// Created by Stephen Birarda on 11/16/15. +// Copyright 2014 High Fidelity, Inc. +// +// Distributed under the Apache License, Version 2.0. +// See the accompanying file LICENSE or http://www.apache.org/licenses/LICENSE-2.0.html +// + +var soundURL = "atp:44a83a788ccfd2924e35c902c34808b24dbd0309d000299ce01a355f91cf8115.wav"; +var audioOptions = { + position: MyAvatar.position, + volume: 0.5 +}; + +var sound = SoundCache.getSound(soundURL); +var injector = null; +var restarting = false; + +Script.update.connect(function(){ + if (sound.downloaded) { + if (!injector) { + injector = Audio.playSound(sound, audioOptions); + } else if (!injector.isPlaying && !restarting) { + restarting = true; + + Script.setTimeout(function(){ + print("Calling restart for a stopped injector from script."); + injector.restart(); + }, 1000); + } else if (injector.isPlaying) { + restarting = false; + + if (Math.random() < 0.0001) { + print("Calling restart for a running injector from script."); + injector.restart(); + } + } + } +}) diff --git a/interface/src/Application.cpp b/interface/src/Application.cpp index 7a564bbbf0..9bc0bf0179 100644 --- a/interface/src/Application.cpp +++ b/interface/src/Application.cpp @@ -50,6 +50,7 @@ #include #include #include +#include #include #include #include @@ -332,6 +333,7 @@ bool setupEssentials(int& argc, char** argv) { DependencyManager::set(); DependencyManager::set(); DependencyManager::set(); + DependencyManager::set(); DependencyManager::set(); DependencyManager::set(); return true; diff --git a/libraries/audio/src/AudioInjector.cpp b/libraries/audio/src/AudioInjector.cpp index bd807f8dbd..1a00611ee0 100644 --- a/libraries/audio/src/AudioInjector.cpp +++ b/libraries/audio/src/AudioInjector.cpp @@ -18,6 +18,7 @@ #include #include "AbstractAudioInterface.h" +#include "AudioInjectorManager.h" #include "AudioRingBuffer.h" #include "AudioLogging.h" #include "SoundCache.h" @@ -35,6 +36,7 @@ AudioInjector::AudioInjector(Sound* sound, const AudioInjectorOptions& injectorO _audioData(sound->getByteArray()), _options(injectorOptions) { + } AudioInjector::AudioInjector(const QByteArray& audioData, const AudioInjectorOptions& injectorOptions) : @@ -44,32 +46,27 @@ AudioInjector::AudioInjector(const QByteArray& audioData, const AudioInjectorOpt } -void AudioInjector::setIsFinished(bool isFinished) { - _isFinished = isFinished; - // In all paths, regardless of isFinished argument. restart() passes false to prepare for new play, and injectToMixer() needs _shouldStop reset. - _shouldStop = false; - - if (_isFinished) { - emit finished(); - - if (_localBuffer) { - _localBuffer->stop(); - _localBuffer->deleteLater(); - _localBuffer = NULL; - } - - _isStarted = false; - - if (_shouldDeleteAfterFinish) { - // we've been asked to delete after finishing, trigger a queued deleteLater here - QMetaObject::invokeMethod(this, "deleteLater", Qt::QueuedConnection); - } +void AudioInjector::finish() { + State oldState = std::atomic_exchange(&_state, State::Finished); + bool shouldDelete = (oldState == State::NotFinishedWithPendingDelete); + + emit finished(); + + if (_localBuffer) { + _localBuffer->stop(); + _localBuffer->deleteLater(); + _localBuffer = NULL; + } + + if (shouldDelete) { + // we've been asked to delete after finishing, trigger a queued deleteLater here + QMetaObject::invokeMethod(this, "deleteLater", Qt::QueuedConnection); } } void AudioInjector::injectAudio() { - if (!_isStarted) { - _isStarted = true; + if (!_hasStarted) { + _hasStarted = true; // check if we need to offset the sound by some number of seconds if (_options.secondOffset > 0.0f) { @@ -85,6 +82,7 @@ void AudioInjector::injectAudio() { if (_options.localOnly) { injectLocally(); } else { + qDebug() << "Calling inject to mixer from" << QThread::currentThread(); injectToMixer(); } } else { @@ -93,18 +91,26 @@ void AudioInjector::injectAudio() { } void AudioInjector::restart() { - _isPlaying = true; - connect(this, &AudioInjector::finished, this, &AudioInjector::restartPortionAfterFinished); - if (!_isStarted || _isFinished) { - emit finished(); - } else { - stop(); + if (thread() != QThread::currentThread()) { + QMetaObject::invokeMethod(this, "restart"); + return; + } + + // reset the current send offset to zero + _currentSendOffset = 0; + + // check our state to decide if we need extra handling for the restart request + if (!isPlaying()) { + // we finished playing, need to reset state so we can get going again + _hasStarted = false; + _shouldStop = false; + _state = State::NotFinished; + + qDebug() << "Calling inject audio again to restart an injector"; + + // call inject audio to start injection over again + injectAudio(); } -} -void AudioInjector::restartPortionAfterFinished() { - disconnect(this, &AudioInjector::finished, this, &AudioInjector::restartPortionAfterFinished); - setIsFinished(false); - QMetaObject::invokeMethod(this, "injectAudio", Qt::QueuedConnection); } void AudioInjector::injectLocally() { @@ -252,16 +258,17 @@ void AudioInjector::injectToMixer() { if (_currentSendOffset != bytesToCopy && _currentSendOffset < _audioData.size()) { - // process events in case we have been told to stop and be deleted - QCoreApplication::processEvents(); + if (_shouldStop) { break; } - + // not the first packet and not done // sleep for the appropriate time - int usecToSleep = (++nextFrame * (_options.stereo ? 2 : 1) * AudioConstants::NETWORK_FRAME_USECS) - timer.nsecsElapsed() / 1000; + int usecToSleep = (++nextFrame * AudioConstants::NETWORK_FRAME_USECS) - timer.nsecsElapsed() / 1000; + + qDebug() << "AudioInjector" << this << "will sleep on thread" << QThread::currentThread() << "for" << usecToSleep; if (usecToSleep > 0) { usleep(usecToSleep); @@ -274,8 +281,7 @@ void AudioInjector::injectToMixer() { } } - setIsFinished(true); - _isPlaying = !_isFinished; // Which can be false if a restart was requested + finish(); } void AudioInjector::stop() { @@ -283,8 +289,14 @@ void AudioInjector::stop() { if (_options.localOnly) { // we're only a local injector, so we can say we are finished right away too - _isPlaying = false; - setIsFinished(true); + finish(); + } +} + +void AudioInjector::triggerDeleteAfterFinish() { + auto expectedState = State::NotFinished; + if (!_state.compare_exchange_strong(expectedState, State::NotFinishedWithPendingDelete)) { + stopAndDeleteLater(); } } @@ -336,28 +348,26 @@ AudioInjector* AudioInjector::playSound(const QString& soundUrl, const float vol AudioInjector* AudioInjector::playSoundAndDelete(const QByteArray& buffer, const AudioInjectorOptions options, AbstractAudioInterface* localInterface) { AudioInjector* sound = playSound(buffer, options, localInterface); - sound->triggerDeleteAfterFinish(); + sound->_state = AudioInjector::State::NotFinishedWithPendingDelete; return sound; } AudioInjector* AudioInjector::playSound(const QByteArray& buffer, const AudioInjectorOptions options, AbstractAudioInterface* localInterface) { - QThread* injectorThread = new QThread(); - injectorThread->setObjectName("Audio Injector Thread"); - AudioInjector* injector = new AudioInjector(buffer, options); - injector->_isPlaying = true; injector->setLocalAudioInterface(localInterface); - - injector->moveToThread(injectorThread); - - // start injecting when the injector thread starts - connect(injectorThread, &QThread::started, injector, &AudioInjector::injectAudio); - - // connect the right slots and signals for AudioInjector and thread cleanup - connect(injector, &AudioInjector::destroyed, injectorThread, &QThread::quit); - connect(injectorThread, &QThread::finished, injectorThread, &QThread::deleteLater); - - injectorThread->start(); - return injector; + + // grab the AudioInjectorManager + auto injectorManager = DependencyManager::get(); + + // attempt to thread the new injector + if (injectorManager->threadInjector(injector)) { + // call inject audio on the correct thread + QMetaObject::invokeMethod(injector, "injectAudio", Qt::QueuedConnection); + + return injector; + } else { + // we failed to thread the new injector (we are at the max number of injector threads) + return nullptr; + } } diff --git a/libraries/audio/src/AudioInjector.h b/libraries/audio/src/AudioInjector.h index 0e98fe1682..25b814f0f6 100644 --- a/libraries/audio/src/AudioInjector.h +++ b/libraries/audio/src/AudioInjector.h @@ -12,6 +12,8 @@ #ifndef hifi_AudioInjector_h #define hifi_AudioInjector_h +#include + #include #include #include @@ -32,11 +34,17 @@ class AudioInjector : public QObject { Q_OBJECT public: + enum class State : uint8_t { + NotFinished, + NotFinishedWithPendingDelete, + Finished + }; + AudioInjector(QObject* parent); AudioInjector(Sound* sound, const AudioInjectorOptions& injectorOptions); AudioInjector(const QByteArray& audioData, const AudioInjectorOptions& injectorOptions); - bool isFinished() const { return _isFinished; } + bool isFinished() const { return _state == State::Finished; } int getCurrentSendOffset() const { return _currentSendOffset; } void setCurrentSendOffset(int currentSendOffset) { _currentSendOffset = currentSendOffset; } @@ -55,15 +63,14 @@ public slots: void restart(); void stop(); - void triggerDeleteAfterFinish() { _shouldDeleteAfterFinish = true; } + void triggerDeleteAfterFinish(); void stopAndDeleteLater(); const AudioInjectorOptions& getOptions() const { return _options; } void setOptions(const AudioInjectorOptions& options) { _options = options; } float getLoudness() const { return _loudness; } - bool isPlaying() const { return _isPlaying; } - void restartPortionAfterFinished(); + bool isPlaying() const { return _state == State::NotFinished || _state == State::NotFinishedWithPendingDelete; } signals: void finished(); @@ -72,16 +79,14 @@ private: void injectToMixer(); void injectLocally(); - void setIsFinished(bool isFinished); + void finish(); QByteArray _audioData; AudioInjectorOptions _options; + std::atomic _state { State::NotFinished }; + bool _hasStarted = false; bool _shouldStop = false; float _loudness = 0.0f; - bool _isPlaying = false; - bool _isStarted = false; - bool _isFinished = false; - bool _shouldDeleteAfterFinish = false; int _currentSendOffset = 0; AbstractAudioInterface* _localAudioInterface = NULL; AudioInjectorLocalBuffer* _localBuffer = NULL; diff --git a/libraries/audio/src/AudioInjectorManager.cpp b/libraries/audio/src/AudioInjectorManager.cpp new file mode 100644 index 0000000000..8eda670ddf --- /dev/null +++ b/libraries/audio/src/AudioInjectorManager.cpp @@ -0,0 +1,96 @@ +// +// AudioInjectorManager.cpp +// libraries/audio/src +// +// Created by Stephen Birarda on 2015-11-16. +// Copyright 2015 High Fidelity, Inc. +// +// Distributed under the Apache License, Version 2.0. +// See the accompanying file LICENSE or http://www.apache.org/licenses/LICENSE-2.0.html +// + +#include "AudioInjectorManager.h" + +#include + +#include "AudioInjector.h" + +AudioInjectorManager::~AudioInjectorManager() { + _shouldStop = true; + + // make sure any still living injectors are stopped and deleted + for (auto injector : _injectors) { + injector->stopAndDeleteLater(); + } + + // quit and wait on the injector thread, if we ever created it + if (_thread) { + _thread->quit(); + _thread->wait(); + } +} + +void AudioInjectorManager::createThread() { + _thread = new QThread; + _thread->setObjectName("Audio Injector Thread"); + + // when the thread is started, have it call our run to handle injection of audio + connect(_thread, &QThread::started, this, &AudioInjectorManager::run); + + // start the thread + _thread->start(); +} + +void AudioInjectorManager::run() { + while (!_shouldStop) { + // process events in case we have been told to stop or our injectors have been told to stop + QCoreApplication::processEvents(); + } +} + +static const int MAX_INJECTORS_PER_THREAD = 50; // calculated based on AudioInjector while loop time, with sufficient padding + +bool AudioInjectorManager::threadInjector(AudioInjector* injector) { + // guard the injectors vector with a mutex + std::unique_lock lock(_injectorsMutex); + + // check if we'll be able to thread this injector (do we have < MAX concurrent injectors) + if (_injectors.size() < MAX_INJECTORS_PER_THREAD) { + if (!_thread) { + createThread(); + } + + auto it = nextInjectorIterator(); + qDebug() << "Inserting injector at" << it - _injectors.begin(); + + // store a QPointer to this injector + _injectors.insert(it, QPointer(injector)); + + qDebug() << "Moving injector to thread" << _thread; + + // move the injector to the QThread + injector->moveToThread(_thread); + + return true; + } else { + // unable to thread this injector, at the max + qDebug() << "AudioInjectorManager::threadInjector could not thread AudioInjector - at max of" + << MAX_INJECTORS_PER_THREAD << "current audio injectors."; + return false; + } +} + +AudioInjectorVector::iterator AudioInjectorManager::nextInjectorIterator() { + // find the next usable iterator for an injector + auto it = _injectors.begin(); + + while (it != _injectors.end()) { + if (it->isNull()) { + return it; + } else { + ++it; + } + } + + return it; +} diff --git a/libraries/audio/src/AudioInjectorManager.h b/libraries/audio/src/AudioInjectorManager.h new file mode 100644 index 0000000000..4e571d2d8d --- /dev/null +++ b/libraries/audio/src/AudioInjectorManager.h @@ -0,0 +1,53 @@ +// +// AudioInjectorManager.h +// libraries/audio/src +// +// Created by Stephen Birarda on 2015-11-16. +// Copyright 2015 High Fidelity, Inc. +// +// Distributed under the Apache License, Version 2.0. +// See the accompanying file LICENSE or http://www.apache.org/licenses/LICENSE-2.0.html +// + +#pragma once + +#ifndef hifi_AudioInjectorManager_h +#define hifi_AudioInjectorManager_h + +#include +#include + +#include +#include + +#include + +class AudioInjector; +using AudioInjectorVector = std::vector>; + +class AudioInjectorManager : public QObject, public Dependency { + Q_OBJECT + SINGLETON_DEPENDENCY +public: + ~AudioInjectorManager(); +private slots: + void run(); +private: + bool threadInjector(AudioInjector* injector); + + AudioInjectorManager() {}; + AudioInjectorManager(const AudioInjectorManager&) = delete; + AudioInjectorManager& operator=(const AudioInjectorManager&) = delete; + + void createThread(); + AudioInjectorVector::iterator nextInjectorIterator(); + + QThread* _thread { nullptr }; + bool _shouldStop { false }; + AudioInjectorVector _injectors; + std::mutex _injectorsMutex; + + friend class AudioInjector; +}; + +#endif // hifi_AudioInjectorManager_h From 80115d38e9335ab125c36d502bf168c8ba2422a6 Mon Sep 17 00:00:00 2001 From: Stephen Birarda Date: Mon, 16 Nov 2015 18:53:09 -0800 Subject: [PATCH 06/25] tweak the AudioInjectorManager for injector threading --- libraries/audio/src/AudioInjector.cpp | 279 ++++++++++--------- libraries/audio/src/AudioInjector.h | 14 +- libraries/audio/src/AudioInjectorManager.cpp | 98 +++++-- libraries/audio/src/AudioInjectorManager.h | 12 +- 4 files changed, 234 insertions(+), 169 deletions(-) diff --git a/libraries/audio/src/AudioInjector.cpp b/libraries/audio/src/AudioInjector.cpp index 1a00611ee0..e88a858608 100644 --- a/libraries/audio/src/AudioInjector.cpp +++ b/libraries/audio/src/AudioInjector.cpp @@ -64,9 +64,10 @@ void AudioInjector::finish() { } } -void AudioInjector::injectAudio() { +void AudioInjector::setupInjection() { if (!_hasStarted) { _hasStarted = true; + // check if we need to offset the sound by some number of seconds if (_options.secondOffset > 0.0f) { @@ -81,12 +82,9 @@ void AudioInjector::injectAudio() { if (_options.localOnly) { injectLocally(); - } else { - qDebug() << "Calling inject to mixer from" << QThread::currentThread(); - injectToMixer(); } } else { - qCDebug(audio) << "AudioInjector::injectAudio called but already started."; + qCDebug(audio) << "AudioInjector::setupInjection called but already started."; } } @@ -109,7 +107,10 @@ void AudioInjector::restart() { qDebug() << "Calling inject audio again to restart an injector"; // call inject audio to start injection over again - injectAudio(); + setupInjection(); + + // emit our restarted signal, this allows the AudioInjectorManager to start considering us again + emit restartedWhileFinished(); } } @@ -148,140 +149,148 @@ void AudioInjector::injectLocally() { } const uchar MAX_INJECTOR_VOLUME = 0xFF; +static const uint64_t NEXT_FRAME_DELTA_ERROR_OR_FINISHED = 0; +static const uint64_t NEXT_FRAME_DELTA_IMMEDIATELY = 1; -void AudioInjector::injectToMixer() { - if (_currentSendOffset < 0 || - _currentSendOffset >= _audioData.size()) { - _currentSendOffset = 0; +uint64_t AudioInjector::injectNextFrame() { + + if (_state == AudioInjector::State::Finished) { + qDebug() << "AudioInjector::injectNextFrame called but AudioInjector has finished and was not restarted. Returning."; + return NEXT_FRAME_DELTA_ERROR_OR_FINISHED; } - - auto nodeList = DependencyManager::get(); - - // make sure we actually have samples downloaded to inject - if (_audioData.size()) { - - auto audioPacket = NLPacket::create(PacketType::InjectAudio); - - // setup the packet for injected audio - QDataStream audioPacketStream(audioPacket.get()); - - // pack some placeholder sequence number for now - audioPacketStream << (quint16) 0; - - // pack stream identifier (a generated UUID) - audioPacketStream << QUuid::createUuid(); - - // pack the stereo/mono type of the stream - audioPacketStream << _options.stereo; - - // pack the flag for loopback - uchar loopbackFlag = (uchar) true; - audioPacketStream << loopbackFlag; - - // pack the position for injected audio - int positionOptionOffset = audioPacket->pos(); - audioPacketStream.writeRawData(reinterpret_cast(&_options.position), - sizeof(_options.position)); - - // pack our orientation for injected audio - audioPacketStream.writeRawData(reinterpret_cast(&_options.orientation), - sizeof(_options.orientation)); - - // pack zero for radius - float radius = 0; - audioPacketStream << radius; - - // pack 255 for attenuation byte - int volumeOptionOffset = audioPacket->pos(); - quint8 volume = MAX_INJECTOR_VOLUME * _options.volume; - audioPacketStream << volume; - - audioPacketStream << _options.ignorePenumbra; - - int audioDataOffset = audioPacket->pos(); - - QElapsedTimer timer; - timer.start(); - int nextFrame = 0; - - bool shouldLoop = _options.loop; - - // loop to send off our audio in NETWORK_BUFFER_LENGTH_SAMPLES_PER_CHANNEL byte chunks - quint16 outgoingInjectedAudioSequenceNumber = 0; - - while (_currentSendOffset < _audioData.size() && !_shouldStop) { - - int bytesToCopy = std::min((_options.stereo ? 2 : 1) * AudioConstants::NETWORK_FRAME_BYTES_PER_CHANNEL, - _audioData.size() - _currentSendOffset); - - // Measure the loudness of this frame - _loudness = 0.0f; - for (int i = 0; i < bytesToCopy; i += sizeof(int16_t)) { - _loudness += abs(*reinterpret_cast(_audioData.data() + _currentSendOffset + i)) / - (AudioConstants::MAX_SAMPLE_VALUE / 2.0f); - } - _loudness /= (float)(bytesToCopy / sizeof(int16_t)); + + // if we haven't setup the packet to send then do so now + static int positionOptionOffset = -1; + static int volumeOptionOffset = -1; + static int audioDataOffset = -1; + static quint16 outgoingInjectedAudioSequenceNumber = 0; + static int nextFrame = 0; + static QElapsedTimer frameTimer; + + if (!_currentPacket) { + if (_currentSendOffset < 0 || + _currentSendOffset >= _audioData.size()) { + _currentSendOffset = 0; + } + + // make sure we actually have samples downloaded to inject + if (_audioData.size()) { - audioPacket->seek(0); + nextFrame = 0; + frameTimer.restart(); - // pack the sequence number - audioPacket->writePrimitive(outgoingInjectedAudioSequenceNumber); + _currentPacket = NLPacket::create(PacketType::InjectAudio); - audioPacket->seek(positionOptionOffset); - audioPacket->writePrimitive(_options.position); - audioPacket->writePrimitive(_options.orientation); - - volume = MAX_INJECTOR_VOLUME * _options.volume; - audioPacket->seek(volumeOptionOffset); - audioPacket->writePrimitive(volume); - - audioPacket->seek(audioDataOffset); - - // copy the next NETWORK_BUFFER_LENGTH_BYTES_PER_CHANNEL bytes to the packet - audioPacket->write(_audioData.data() + _currentSendOffset, bytesToCopy); - - // set the correct size used for this packet - audioPacket->setPayloadSize(audioPacket->pos()); - - // grab our audio mixer from the NodeList, if it exists - SharedNodePointer audioMixer = nodeList->soloNodeOfType(NodeType::AudioMixer); + // setup the packet for injected audio + QDataStream audioPacketStream(_currentPacket.get()); - if (audioMixer) { - // send off this audio packet - nodeList->sendUnreliablePacket(*audioPacket, *audioMixer); - outgoingInjectedAudioSequenceNumber++; - } - - _currentSendOffset += bytesToCopy; - - // send two packets before the first sleep so the mixer can start playback right away - - if (_currentSendOffset != bytesToCopy && _currentSendOffset < _audioData.size()) { - - - - if (_shouldStop) { - break; - } - - // not the first packet and not done - // sleep for the appropriate time - int usecToSleep = (++nextFrame * AudioConstants::NETWORK_FRAME_USECS) - timer.nsecsElapsed() / 1000; - - qDebug() << "AudioInjector" << this << "will sleep on thread" << QThread::currentThread() << "for" << usecToSleep; - - if (usecToSleep > 0) { - usleep(usecToSleep); - } - } - - if (shouldLoop && _currentSendOffset >= _audioData.size()) { - _currentSendOffset = 0; - } + // pack some placeholder sequence number for now + audioPacketStream << (quint16) 0; + + // pack stream identifier (a generated UUID) + audioPacketStream << QUuid::createUuid(); + + // pack the stereo/mono type of the stream + audioPacketStream << _options.stereo; + + // pack the flag for loopback + uchar loopbackFlag = (uchar) true; + audioPacketStream << loopbackFlag; + + // pack the position for injected audio + positionOptionOffset = _currentPacket->pos(); + audioPacketStream.writeRawData(reinterpret_cast(&_options.position), + sizeof(_options.position)); + + // pack our orientation for injected audio + audioPacketStream.writeRawData(reinterpret_cast(&_options.orientation), + sizeof(_options.orientation)); + + // pack zero for radius + float radius = 0; + audioPacketStream << radius; + + // pack 255 for attenuation byte + volumeOptionOffset = _currentPacket->pos(); + quint8 volume = MAX_INJECTOR_VOLUME; + audioPacketStream << volume; + + audioPacketStream << _options.ignorePenumbra; + + audioDataOffset = _currentPacket->pos(); + + } else { + // no samples to inject, return immediately + qDebug() << "AudioInjector::injectNextFrame() called with no samples to inject. Returning."; + return NEXT_FRAME_DELTA_ERROR_OR_FINISHED; } } - - finish(); + + int bytesToCopy = std::min((_options.stereo ? 2 : 1) * AudioConstants::NETWORK_FRAME_BYTES_PER_CHANNEL, + _audioData.size() - _currentSendOffset); + + // Measure the loudness of this frame + _loudness = 0.0f; + for (int i = 0; i < bytesToCopy; i += sizeof(int16_t)) { + _loudness += abs(*reinterpret_cast(_audioData.data() + _currentSendOffset + i)) / + (AudioConstants::MAX_SAMPLE_VALUE / 2.0f); + } + _loudness /= (float)(bytesToCopy / sizeof(int16_t)); + + _currentPacket->seek(0); + + // pack the sequence number + _currentPacket->writePrimitive(outgoingInjectedAudioSequenceNumber); + + _currentPacket->seek(positionOptionOffset); + _currentPacket->writePrimitive(_options.position); + _currentPacket->writePrimitive(_options.orientation); + + quint8 volume = MAX_INJECTOR_VOLUME * _options.volume; + _currentPacket->seek(volumeOptionOffset); + _currentPacket->writePrimitive(volume); + + _currentPacket->seek(audioDataOffset); + + // copy the next NETWORK_BUFFER_LENGTH_BYTES_PER_CHANNEL bytes to the packet + _currentPacket->write(_audioData.data() + _currentSendOffset, bytesToCopy); + + // set the correct size used for this packet + _currentPacket->setPayloadSize(_currentPacket->pos()); + + // grab our audio mixer from the NodeList, if it exists + auto nodeList = DependencyManager::get(); + SharedNodePointer audioMixer = nodeList->soloNodeOfType(NodeType::AudioMixer); + + if (audioMixer) { + // send off this audio packet + nodeList->sendUnreliablePacket(*_currentPacket, *audioMixer); + outgoingInjectedAudioSequenceNumber++; + } + + _currentSendOffset += bytesToCopy; + + if (_currentSendOffset >= _audioData.size()) { + // we're at the end of the audio data to send + if (_options.loop) { + // we were asked to loop, set our send offset to 0 + _currentSendOffset = 0; + } else { + // we weren't to loop, say that we're done now + qDebug() << "AudioInjector::injectNextFrame has sent all data and was not asked to loop - calling finish()."; + finish(); + return NEXT_FRAME_DELTA_ERROR_OR_FINISHED; + } + } + + if (_currentSendOffset == bytesToCopy) { + // ask AudioInjectorManager to call us right away again to + // immediately send the first two frames so the mixer can start using the audio right away + return NEXT_FRAME_DELTA_IMMEDIATELY; + } else { + return (++nextFrame * AudioConstants::NETWORK_FRAME_USECS) - frameTimer.nsecsElapsed() / 1000; + } + } void AudioInjector::stop() { @@ -360,11 +369,11 @@ AudioInjector* AudioInjector::playSound(const QByteArray& buffer, const AudioInj // grab the AudioInjectorManager auto injectorManager = DependencyManager::get(); + // setup parameters required for injection + injector->setupInjection(); + // attempt to thread the new injector if (injectorManager->threadInjector(injector)) { - // call inject audio on the correct thread - QMetaObject::invokeMethod(injector, "injectAudio", Qt::QueuedConnection); - return injector; } else { // we failed to thread the new injector (we are at the max number of injector threads) diff --git a/libraries/audio/src/AudioInjector.h b/libraries/audio/src/AudioInjector.h index 25b814f0f6..7567a18388 100644 --- a/libraries/audio/src/AudioInjector.h +++ b/libraries/audio/src/AudioInjector.h @@ -26,6 +26,8 @@ #include "Sound.h" class AbstractAudioInterface; +class AudioInjectorManager; +class NLPacket; // In order to make scripting cleaner for the AudioInjector, the script now holds on to the AudioInjector object // until it dies. @@ -59,7 +61,6 @@ public: static AudioInjector* playSound(const QString& soundUrl, const float volume, const float stretchFactor, const glm::vec3 position); public slots: - void injectAudio(); void restart(); void stop(); @@ -74,9 +75,11 @@ public slots: signals: void finished(); + void restartedWhileFinished(); private: - void injectToMixer(); + void setupInjection(); + uint64_t injectNextFrame(); void injectLocally(); void finish(); @@ -88,8 +91,11 @@ private: bool _shouldStop = false; float _loudness = 0.0f; int _currentSendOffset = 0; - AbstractAudioInterface* _localAudioInterface = NULL; - AudioInjectorLocalBuffer* _localBuffer = NULL; + std::unique_ptr _currentPacket { nullptr }; + AbstractAudioInterface* _localAudioInterface { nullptr }; + AudioInjectorLocalBuffer* _localBuffer { nullptr }; + + friend class AudioInjectorManager; }; diff --git a/libraries/audio/src/AudioInjectorManager.cpp b/libraries/audio/src/AudioInjectorManager.cpp index 8eda670ddf..8435048032 100644 --- a/libraries/audio/src/AudioInjectorManager.cpp +++ b/libraries/audio/src/AudioInjectorManager.cpp @@ -13,17 +13,28 @@ #include +#include + +#include "AudioConstants.h" #include "AudioInjector.h" AudioInjectorManager::~AudioInjectorManager() { _shouldStop = true; + std::unique_lock lock(_injectorsMutex); + // make sure any still living injectors are stopped and deleted - for (auto injector : _injectors) { - injector->stopAndDeleteLater(); + while (!_injectors.empty()) { + // grab the injector at the front + auto& timePointerPair = _injectors.front(); + + // ask it to stop and be deleted + timePointerPair.second->stopAndDeleteLater(); + + _injectors.pop(); } - // quit and wait on the injector thread, if we ever created it + // quit and wait on the manager thread, if we ever created it if (_thread) { _thread->quit(); _thread->wait(); @@ -43,7 +54,46 @@ void AudioInjectorManager::createThread() { void AudioInjectorManager::run() { while (!_shouldStop) { - // process events in case we have been told to stop or our injectors have been told to stop + // wait until the next injector is ready, or until we get a new injector given to us + std::unique_lock lock(_injectorsMutex); + + if (_injectors.size() > 0) { + // when does the next injector need to send a frame? + // do we get to wait or should we just go for it now? + + auto timeInjectorPair = _injectors.front(); + + auto nextTimestamp = timeInjectorPair.first; + int64_t difference = int64_t(nextTimestamp - usecTimestampNow()); + + if (difference > 0) { + _injectorReady.wait_for(lock, std::chrono::microseconds(difference)); + } + + // loop through the injectors in the map and send whatever frames need to go out + auto front = _injectors.front(); + while (_injectors.size() > 0 && front.first <= usecTimestampNow()) { + // either way we're popping this injector off - get a copy first + auto injector = front.second; + _injectors.pop(); + + if (!injector.isNull()) { + // this is an injector that's ready to go, have it send a frame now + auto nextCallDelta = injector->injectNextFrame(); + + if (nextCallDelta > 0 && !injector->isFinished()) { + // re-enqueue the injector with the correct timing + _injectors.push({ usecTimestampNow() + nextCallDelta, injector }); + } + } + + front = _injectors.front(); + } + } else { + // we have no current injectors, wait until we get at least one before we do anything + _injectorReady.wait(lock); + } + QCoreApplication::processEvents(); } } @@ -54,23 +104,21 @@ bool AudioInjectorManager::threadInjector(AudioInjector* injector) { // guard the injectors vector with a mutex std::unique_lock lock(_injectorsMutex); - // check if we'll be able to thread this injector (do we have < MAX concurrent injectors) + // check if we'll be able to thread this injector (do we have < max concurrent injectors) if (_injectors.size() < MAX_INJECTORS_PER_THREAD) { if (!_thread) { createThread(); } - auto it = nextInjectorIterator(); - qDebug() << "Inserting injector at" << it - _injectors.begin(); - - // store a QPointer to this injector - _injectors.insert(it, QPointer(injector)); - - qDebug() << "Moving injector to thread" << _thread; - // move the injector to the QThread injector->moveToThread(_thread); + // handle a restart once the injector has finished + connect(injector, &AudioInjector::restartedWhileFinished, this, &AudioInjectorManager::restartFinishedInjector); + + // store a QPointer to this injector + addInjectorToQueue(injector); + return true; } else { // unable to thread this injector, at the max @@ -80,17 +128,15 @@ bool AudioInjectorManager::threadInjector(AudioInjector* injector) { } } -AudioInjectorVector::iterator AudioInjectorManager::nextInjectorIterator() { - // find the next usable iterator for an injector - auto it = _injectors.begin(); - - while (it != _injectors.end()) { - if (it->isNull()) { - return it; - } else { - ++it; - } - } - - return it; +void AudioInjectorManager::restartFinishedInjector() { + auto injector = qobject_cast(sender()); + addInjectorToQueue(injector); +} + +void AudioInjectorManager::addInjectorToQueue(AudioInjector* injector) { + // add the injector to the queue with a send timestamp of now + _injectors.emplace(usecTimestampNow(), InjectorQPointer { injector }); + + // notify our wait condition so we can inject two frames for this injector immediately + _injectorReady.notify_one(); } diff --git a/libraries/audio/src/AudioInjectorManager.h b/libraries/audio/src/AudioInjectorManager.h index 4e571d2d8d..2b21f0f0ec 100644 --- a/libraries/audio/src/AudioInjectorManager.h +++ b/libraries/audio/src/AudioInjectorManager.h @@ -14,7 +14,7 @@ #ifndef hifi_AudioInjectorManager_h #define hifi_AudioInjectorManager_h -#include +#include #include #include @@ -23,7 +23,9 @@ #include class AudioInjector; -using AudioInjectorVector = std::vector>; +using InjectorQPointer = QPointer; +using TimeInjectorPointerPair = std::pair; +using InjectorQueue = std::queue; class AudioInjectorManager : public QObject, public Dependency { Q_OBJECT @@ -34,18 +36,20 @@ private slots: void run(); private: bool threadInjector(AudioInjector* injector); + void restartFinishedInjector(); + void addInjectorToQueue(AudioInjector* injector); AudioInjectorManager() {}; AudioInjectorManager(const AudioInjectorManager&) = delete; AudioInjectorManager& operator=(const AudioInjectorManager&) = delete; void createThread(); - AudioInjectorVector::iterator nextInjectorIterator(); QThread* _thread { nullptr }; bool _shouldStop { false }; - AudioInjectorVector _injectors; + InjectorQueue _injectors; std::mutex _injectorsMutex; + std::condition_variable _injectorReady; friend class AudioInjector; }; From 4fa934dccf539e546e72cc0f44d030104eda5e4e Mon Sep 17 00:00:00 2001 From: Stephen Birarda Date: Mon, 16 Nov 2015 18:54:16 -0800 Subject: [PATCH 07/25] make sure AudioInjectorManager process is on AI thread --- libraries/audio/src/AudioInjectorManager.cpp | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/libraries/audio/src/AudioInjectorManager.cpp b/libraries/audio/src/AudioInjectorManager.cpp index 8435048032..3f83dcc5b5 100644 --- a/libraries/audio/src/AudioInjectorManager.cpp +++ b/libraries/audio/src/AudioInjectorManager.cpp @@ -46,7 +46,7 @@ void AudioInjectorManager::createThread() { _thread->setObjectName("Audio Injector Thread"); // when the thread is started, have it call our run to handle injection of audio - connect(_thread, &QThread::started, this, &AudioInjectorManager::run); + connect(_thread, &QThread::started, this, &AudioInjectorManager::run, Qt::DirectConnection); // start the thread _thread->start(); From 7da6ec46c430eaa27972ee83bfce0ea63a4521cc Mon Sep 17 00:00:00 2001 From: Stephen Birarda Date: Tue, 17 Nov 2015 14:20:59 -0800 Subject: [PATCH 08/25] more handling of various AudioInjector states --- libraries/audio/src/AudioInjector.cpp | 75 ++++++++++++-------- libraries/audio/src/AudioInjector.h | 4 +- libraries/audio/src/AudioInjectorManager.cpp | 15 ++-- libraries/audio/src/AudioInjectorManager.h | 1 - 4 files changed, 58 insertions(+), 37 deletions(-) diff --git a/libraries/audio/src/AudioInjector.cpp b/libraries/audio/src/AudioInjector.cpp index e88a858608..b0b0527e11 100644 --- a/libraries/audio/src/AudioInjector.cpp +++ b/libraries/audio/src/AudioInjector.cpp @@ -65,8 +65,8 @@ void AudioInjector::finish() { } void AudioInjector::setupInjection() { - if (!_hasStarted) { - _hasStarted = true; + if (!_hasSetup) { + _hasSetup = true; // check if we need to offset the sound by some number of seconds if (_options.secondOffset > 0.0f) { @@ -79,18 +79,14 @@ void AudioInjector::setupInjection() { } else { _currentSendOffset = 0; } - - if (_options.localOnly) { - injectLocally(); - } } else { - qCDebug(audio) << "AudioInjector::setupInjection called but already started."; + qCDebug(audio) << "AudioInjector::setupInjection called but already setup."; } } void AudioInjector::restart() { if (thread() != QThread::currentThread()) { - QMetaObject::invokeMethod(this, "restart"); + QMetaObject::invokeMethod(this, "restart"); return; } @@ -98,23 +94,28 @@ void AudioInjector::restart() { _currentSendOffset = 0; // check our state to decide if we need extra handling for the restart request - if (!isPlaying()) { + if (_state == State::Finished) { // we finished playing, need to reset state so we can get going again - _hasStarted = false; + _hasSetup = false; _shouldStop = false; _state = State::NotFinished; - qDebug() << "Calling inject audio again to restart an injector"; + qDebug() << "Emitting restartedWhileFinished to inject audio again to restart an injector"; // call inject audio to start injection over again setupInjection(); - // emit our restarted signal, this allows the AudioInjectorManager to start considering us again + // if we're a local injector call inject locally to start injecting again + if (_options.localOnly) { + injectLocally(); + } + + // emit our restarted signal, for network injectors this allows the AudioInjectorManager to start considering us again emit restartedWhileFinished(); } } -void AudioInjector::injectLocally() { +bool AudioInjector::injectLocally() { bool success = false; if (_localAudioInterface) { if (_audioData.size() > 0) { @@ -145,7 +146,8 @@ void AudioInjector::injectLocally() { // we never started so we are finished, call our stop method stop(); } - + + return success; } const uchar MAX_INJECTOR_VOLUME = 0xFF; @@ -294,18 +296,21 @@ uint64_t AudioInjector::injectNextFrame() { } void AudioInjector::stop() { - _shouldStop = true; - - if (_options.localOnly) { - // we're only a local injector, so we can say we are finished right away too - finish(); - } + // trigger a call on the injector's thread to change state to finished + QMetaObject::invokeMethod(this, "finish"); } void AudioInjector::triggerDeleteAfterFinish() { - auto expectedState = State::NotFinished; - if (!_state.compare_exchange_strong(expectedState, State::NotFinishedWithPendingDelete)) { + // make sure this fires on the AudioInjector thread + if (thread() != QThread::currentThread()) { + QMetaObject::invokeMethod(this, "triggerDeleteAfterFinish", Qt::QueuedConnection); + return; + } + + if (_state == State::Finished) { stopAndDeleteLater(); + } else { + _state = State::NotFinishedWithPendingDelete; } } @@ -357,7 +362,11 @@ AudioInjector* AudioInjector::playSound(const QString& soundUrl, const float vol AudioInjector* AudioInjector::playSoundAndDelete(const QByteArray& buffer, const AudioInjectorOptions options, AbstractAudioInterface* localInterface) { AudioInjector* sound = playSound(buffer, options, localInterface); - sound->_state = AudioInjector::State::NotFinishedWithPendingDelete; + + if (sound) { + sound->_state = AudioInjector::State::NotFinishedWithPendingDelete; + } + return sound; } @@ -372,11 +381,21 @@ AudioInjector* AudioInjector::playSound(const QByteArray& buffer, const AudioInj // setup parameters required for injection injector->setupInjection(); - // attempt to thread the new injector - if (injectorManager->threadInjector(injector)) { - return injector; + if (options.localOnly) { + if (injector->injectLocally()) { + // local injection succeeded, return the pointer to injector + return injector; + } else { + // unable to inject locally, return a nullptr + return nullptr; + } } else { - // we failed to thread the new injector (we are at the max number of injector threads) - return nullptr; + // attempt to thread the new injector + if (injectorManager->threadInjector(injector)) { + return injector; + } else { + // we failed to thread the new injector (we are at the max number of injector threads) + return nullptr; + } } } diff --git a/libraries/audio/src/AudioInjector.h b/libraries/audio/src/AudioInjector.h index 7567a18388..9bacb44fde 100644 --- a/libraries/audio/src/AudioInjector.h +++ b/libraries/audio/src/AudioInjector.h @@ -80,14 +80,14 @@ signals: private: void setupInjection(); uint64_t injectNextFrame(); - void injectLocally(); + bool injectLocally(); void finish(); QByteArray _audioData; AudioInjectorOptions _options; std::atomic _state { State::NotFinished }; - bool _hasStarted = false; + bool _hasSetup = false; bool _shouldStop = false; float _loudness = 0.0f; int _currentSendOffset = 0; diff --git a/libraries/audio/src/AudioInjectorManager.cpp b/libraries/audio/src/AudioInjectorManager.cpp index 3f83dcc5b5..eaf1cdd406 100644 --- a/libraries/audio/src/AudioInjectorManager.cpp +++ b/libraries/audio/src/AudioInjectorManager.cpp @@ -116,8 +116,11 @@ bool AudioInjectorManager::threadInjector(AudioInjector* injector) { // handle a restart once the injector has finished connect(injector, &AudioInjector::restartedWhileFinished, this, &AudioInjectorManager::restartFinishedInjector); - // store a QPointer to this injector - addInjectorToQueue(injector); + // add the injector to the queue with a send timestamp of now + _injectors.emplace(usecTimestampNow(), InjectorQPointer { injector }); + + // notify our wait condition so we can inject two frames for this injector immediately + _injectorReady.notify_one(); return true; } else { @@ -130,10 +133,10 @@ bool AudioInjectorManager::threadInjector(AudioInjector* injector) { void AudioInjectorManager::restartFinishedInjector() { auto injector = qobject_cast(sender()); - addInjectorToQueue(injector); -} - -void AudioInjectorManager::addInjectorToQueue(AudioInjector* injector) { + + // guard the injectors vector with a mutex + std::unique_lock lock(_injectorsMutex); + // add the injector to the queue with a send timestamp of now _injectors.emplace(usecTimestampNow(), InjectorQPointer { injector }); diff --git a/libraries/audio/src/AudioInjectorManager.h b/libraries/audio/src/AudioInjectorManager.h index 2b21f0f0ec..2b1506ffb3 100644 --- a/libraries/audio/src/AudioInjectorManager.h +++ b/libraries/audio/src/AudioInjectorManager.h @@ -37,7 +37,6 @@ private slots: private: bool threadInjector(AudioInjector* injector); void restartFinishedInjector(); - void addInjectorToQueue(AudioInjector* injector); AudioInjectorManager() {}; AudioInjectorManager(const AudioInjectorManager&) = delete; From d764ff2e87b35efda81c7a49fbca28e0ee205059 Mon Sep 17 00:00:00 2001 From: Stephen Birarda Date: Tue, 17 Nov 2015 15:12:04 -0800 Subject: [PATCH 09/25] remove static variables in AudioInjector.cpp --- libraries/audio/src/AudioInjector.cpp | 19 +++++++++++-------- libraries/audio/src/AudioInjector.h | 5 +++++ libraries/audio/src/AudioInjectorManager.cpp | 3 +++ 3 files changed, 19 insertions(+), 8 deletions(-) diff --git a/libraries/audio/src/AudioInjector.cpp b/libraries/audio/src/AudioInjector.cpp index b0b0527e11..f43325a846 100644 --- a/libraries/audio/src/AudioInjector.cpp +++ b/libraries/audio/src/AudioInjector.cpp @@ -165,9 +165,6 @@ uint64_t AudioInjector::injectNextFrame() { static int positionOptionOffset = -1; static int volumeOptionOffset = -1; static int audioDataOffset = -1; - static quint16 outgoingInjectedAudioSequenceNumber = 0; - static int nextFrame = 0; - static QElapsedTimer frameTimer; if (!_currentPacket) { if (_currentSendOffset < 0 || @@ -178,8 +175,14 @@ uint64_t AudioInjector::injectNextFrame() { // make sure we actually have samples downloaded to inject if (_audioData.size()) { - nextFrame = 0; - frameTimer.restart(); + _outgoingSequenceNumber = 0; + _nextFrame = 0; + + if (!_frameTimer) { + _frameTimer = std::unique_ptr(new QElapsedTimer); + } + + _frameTimer->restart(); _currentPacket = NLPacket::create(PacketType::InjectAudio); @@ -242,7 +245,7 @@ uint64_t AudioInjector::injectNextFrame() { _currentPacket->seek(0); // pack the sequence number - _currentPacket->writePrimitive(outgoingInjectedAudioSequenceNumber); + _currentPacket->writePrimitive(_outgoingSequenceNumber); _currentPacket->seek(positionOptionOffset); _currentPacket->writePrimitive(_options.position); @@ -267,7 +270,7 @@ uint64_t AudioInjector::injectNextFrame() { if (audioMixer) { // send off this audio packet nodeList->sendUnreliablePacket(*_currentPacket, *audioMixer); - outgoingInjectedAudioSequenceNumber++; + _outgoingSequenceNumber++; } _currentSendOffset += bytesToCopy; @@ -290,7 +293,7 @@ uint64_t AudioInjector::injectNextFrame() { // immediately send the first two frames so the mixer can start using the audio right away return NEXT_FRAME_DELTA_IMMEDIATELY; } else { - return (++nextFrame * AudioConstants::NETWORK_FRAME_USECS) - frameTimer.nsecsElapsed() / 1000; + return (++_nextFrame * AudioConstants::NETWORK_FRAME_USECS) - _frameTimer->nsecsElapsed() / 1000; } } diff --git a/libraries/audio/src/AudioInjector.h b/libraries/audio/src/AudioInjector.h index 9bacb44fde..a02a20c1e7 100644 --- a/libraries/audio/src/AudioInjector.h +++ b/libraries/audio/src/AudioInjector.h @@ -14,6 +14,7 @@ #include +#include #include #include #include @@ -95,6 +96,10 @@ private: AbstractAudioInterface* _localAudioInterface { nullptr }; AudioInjectorLocalBuffer* _localBuffer { nullptr }; + int _nextFrame { 0 }; + std::unique_ptr _frameTimer { nullptr }; + quint16 _outgoingSequenceNumber { 0 }; + friend class AudioInjectorManager; }; diff --git a/libraries/audio/src/AudioInjectorManager.cpp b/libraries/audio/src/AudioInjectorManager.cpp index eaf1cdd406..ee0a1af77b 100644 --- a/libraries/audio/src/AudioInjectorManager.cpp +++ b/libraries/audio/src/AudioInjectorManager.cpp @@ -94,6 +94,9 @@ void AudioInjectorManager::run() { _injectorReady.wait(lock); } + // unlock the lock in case something in process events needs to modify the queue + lock.unlock(); + QCoreApplication::processEvents(); } } From ab5c8e072faada61d0dfbaf2f8d1be279926754e Mon Sep 17 00:00:00 2001 From: Stephen Birarda Date: Tue, 17 Nov 2015 16:31:18 -0800 Subject: [PATCH 10/25] handle blocked AudioInjectorManager for restart --- examples/tests/injectorTest.js | 2 +- libraries/audio/src/AudioInjector.cpp | 19 +++++++++++++------ libraries/audio/src/AudioInjector.h | 9 +++++---- libraries/audio/src/AudioInjectorManager.cpp | 5 +---- libraries/audio/src/AudioInjectorManager.h | 3 ++- 5 files changed, 22 insertions(+), 16 deletions(-) diff --git a/examples/tests/injectorTest.js b/examples/tests/injectorTest.js index 0ad4aad84c..d383b7ef9a 100644 --- a/examples/tests/injectorTest.js +++ b/examples/tests/injectorTest.js @@ -9,7 +9,7 @@ // See the accompanying file LICENSE or http://www.apache.org/licenses/LICENSE-2.0.html // -var soundURL = "atp:44a83a788ccfd2924e35c902c34808b24dbd0309d000299ce01a355f91cf8115.wav"; +var soundURL = "http://hifi-public.s3.amazonaws.com/birarda/medium-crowd.wav"; var audioOptions = { position: MyAvatar.position, volume: 0.5 diff --git a/libraries/audio/src/AudioInjector.cpp b/libraries/audio/src/AudioInjector.cpp index f43325a846..b3435bb9b7 100644 --- a/libraries/audio/src/AudioInjector.cpp +++ b/libraries/audio/src/AudioInjector.cpp @@ -85,8 +85,17 @@ void AudioInjector::setupInjection() { } void AudioInjector::restart() { + // grab the AudioInjectorManager + auto injectorManager = DependencyManager::get(); + if (thread() != QThread::currentThread()) { - QMetaObject::invokeMethod(this, "restart"); + QMetaObject::invokeMethod(this, "restart"); + + if (!_options.localOnly) { + // notify the AudioInjectorManager to wake up in case it's waiting for new injectors + injectorManager->notifyInjectorReadyCondition(); + } + return; } @@ -100,18 +109,16 @@ void AudioInjector::restart() { _shouldStop = false; _state = State::NotFinished; - qDebug() << "Emitting restartedWhileFinished to inject audio again to restart an injector"; - // call inject audio to start injection over again setupInjection(); // if we're a local injector call inject locally to start injecting again if (_options.localOnly) { injectLocally(); + } else { + // wake the AudioInjectorManager back up if it's stuck waiting + injectorManager->restartFinishedInjector(this); } - - // emit our restarted signal, for network injectors this allows the AudioInjectorManager to start considering us again - emit restartedWhileFinished(); } } diff --git a/libraries/audio/src/AudioInjector.h b/libraries/audio/src/AudioInjector.h index a02a20c1e7..0c549eaabb 100644 --- a/libraries/audio/src/AudioInjector.h +++ b/libraries/audio/src/AudioInjector.h @@ -76,15 +76,16 @@ public slots: signals: void finished(); - void restartedWhileFinished(); - + void restarting(); + +private slots: + void finish(); + private: void setupInjection(); uint64_t injectNextFrame(); bool injectLocally(); - void finish(); - QByteArray _audioData; AudioInjectorOptions _options; std::atomic _state { State::NotFinished }; diff --git a/libraries/audio/src/AudioInjectorManager.cpp b/libraries/audio/src/AudioInjectorManager.cpp index ee0a1af77b..33df3e4e94 100644 --- a/libraries/audio/src/AudioInjectorManager.cpp +++ b/libraries/audio/src/AudioInjectorManager.cpp @@ -117,7 +117,6 @@ bool AudioInjectorManager::threadInjector(AudioInjector* injector) { injector->moveToThread(_thread); // handle a restart once the injector has finished - connect(injector, &AudioInjector::restartedWhileFinished, this, &AudioInjectorManager::restartFinishedInjector); // add the injector to the queue with a send timestamp of now _injectors.emplace(usecTimestampNow(), InjectorQPointer { injector }); @@ -134,9 +133,7 @@ bool AudioInjectorManager::threadInjector(AudioInjector* injector) { } } -void AudioInjectorManager::restartFinishedInjector() { - auto injector = qobject_cast(sender()); - +void AudioInjectorManager::restartFinishedInjector(AudioInjector* injector) { // guard the injectors vector with a mutex std::unique_lock lock(_injectorsMutex); diff --git a/libraries/audio/src/AudioInjectorManager.h b/libraries/audio/src/AudioInjectorManager.h index 2b1506ffb3..7518bb1df8 100644 --- a/libraries/audio/src/AudioInjectorManager.h +++ b/libraries/audio/src/AudioInjectorManager.h @@ -36,7 +36,8 @@ private slots: void run(); private: bool threadInjector(AudioInjector* injector); - void restartFinishedInjector(); + void restartFinishedInjector(AudioInjector* injector); + void notifyInjectorReadyCondition() { _injectorReady.notify_one(); } AudioInjectorManager() {}; AudioInjectorManager(const AudioInjectorManager&) = delete; From 935c0cc13710765520b29bad4d659eeeee4daced Mon Sep 17 00:00:00 2001 From: Stephen Birarda Date: Tue, 17 Nov 2015 16:39:23 -0800 Subject: [PATCH 11/25] handle AudioInjectorManager cleanup in Application --- interface/src/Application.cpp | 4 ++++ libraries/audio/src/AudioInjectorManager.cpp | 6 ++++++ 2 files changed, 10 insertions(+) diff --git a/interface/src/Application.cpp b/interface/src/Application.cpp index 9bc0bf0179..6f3116ef9b 100644 --- a/interface/src/Application.cpp +++ b/interface/src/Application.cpp @@ -863,6 +863,10 @@ void Application::cleanupBeforeQuit() { // destroy the AudioClient so it and its thread have a chance to go down safely DependencyManager::destroy(); + + // destroy the AudioInjectorManager so it and its thread have a chance to go down safely + // this will also stop any ongoing network injectors + DependencyManager::destroy(); // Destroy third party processes after scripts have finished using them. #ifdef HAVE_DDE diff --git a/libraries/audio/src/AudioInjectorManager.cpp b/libraries/audio/src/AudioInjectorManager.cpp index 33df3e4e94..a441f679d6 100644 --- a/libraries/audio/src/AudioInjectorManager.cpp +++ b/libraries/audio/src/AudioInjectorManager.cpp @@ -34,6 +34,12 @@ AudioInjectorManager::~AudioInjectorManager() { _injectors.pop(); } + // get rid of the lock now that we've stopped all living injectors + lock.unlock(); + + // in case the thread is waiting for injectors wake it up now + _injectorReady.notify_one(); + // quit and wait on the manager thread, if we ever created it if (_thread) { _thread->quit(); From 8e4d7c69cee082d4657d4d759ed27b6461049929 Mon Sep 17 00:00:00 2001 From: Stephen Birarda Date: Tue, 17 Nov 2015 17:01:11 -0800 Subject: [PATCH 12/25] handle AudioInjectorManager for scripted ACs --- assignment-client/src/Agent.cpp | 10 +++- examples/tests/injectorTest.js | 2 +- libraries/audio/src/AudioInjectorManager.cpp | 56 ++++++++++++-------- 3 files changed, 42 insertions(+), 26 deletions(-) diff --git a/assignment-client/src/Agent.cpp b/assignment-client/src/Agent.cpp index 7ee696693d..953da03d24 100644 --- a/assignment-client/src/Agent.cpp +++ b/assignment-client/src/Agent.cpp @@ -16,8 +16,9 @@ #include #include -#include #include +#include +#include #include #include #include @@ -43,7 +44,7 @@ Agent::Agent(NLPacket& packet) : DEFAULT_WINDOW_SECONDS_FOR_DESIRED_REDUCTION, false)) { DependencyManager::get()->setPacketSender(&_entityEditSender); - + auto assetClient = DependencyManager::set(); QThread* assetThread = new QThread; @@ -54,6 +55,8 @@ Agent::Agent(NLPacket& packet) : DependencyManager::set(); DependencyManager::set(); + + DependencyManager::set(); auto& packetReceiver = DependencyManager::get()->getPacketReceiver(); @@ -396,6 +399,9 @@ void Agent::aboutToFinish() { DependencyManager::destroy(); assetThread->quit(); assetThread->wait(); + + // cleanup the AudioInjectorManager (and any still running injectors) + DependencyManager::set(); } void Agent::sendPingRequests() { diff --git a/examples/tests/injectorTest.js b/examples/tests/injectorTest.js index d383b7ef9a..171186c91e 100644 --- a/examples/tests/injectorTest.js +++ b/examples/tests/injectorTest.js @@ -11,7 +11,7 @@ var soundURL = "http://hifi-public.s3.amazonaws.com/birarda/medium-crowd.wav"; var audioOptions = { - position: MyAvatar.position, + position: { x: 0.0, y: 0.0, z: 0.0 }, volume: 0.5 }; diff --git a/libraries/audio/src/AudioInjectorManager.cpp b/libraries/audio/src/AudioInjectorManager.cpp index a441f679d6..4770371e48 100644 --- a/libraries/audio/src/AudioInjectorManager.cpp +++ b/libraries/audio/src/AudioInjectorManager.cpp @@ -76,25 +76,28 @@ void AudioInjectorManager::run() { _injectorReady.wait_for(lock, std::chrono::microseconds(difference)); } - // loop through the injectors in the map and send whatever frames need to go out - auto front = _injectors.front(); - while (_injectors.size() > 0 && front.first <= usecTimestampNow()) { - // either way we're popping this injector off - get a copy first - auto injector = front.second; - _injectors.pop(); - - if (!injector.isNull()) { - // this is an injector that's ready to go, have it send a frame now - auto nextCallDelta = injector->injectNextFrame(); + if (_injectors.size() > 0) { + // loop through the injectors in the map and send whatever frames need to go out + auto front = _injectors.front(); + while (_injectors.size() > 0 && front.first <= usecTimestampNow()) { + // either way we're popping this injector off - get a copy first + auto injector = front.second; + _injectors.pop(); - if (nextCallDelta > 0 && !injector->isFinished()) { - // re-enqueue the injector with the correct timing - _injectors.push({ usecTimestampNow() + nextCallDelta, injector }); + if (!injector.isNull()) { + // this is an injector that's ready to go, have it send a frame now + auto nextCallDelta = injector->injectNextFrame(); + + if (nextCallDelta > 0 && !injector->isFinished()) { + // re-enqueue the injector with the correct timing + _injectors.push({ usecTimestampNow() + nextCallDelta, injector }); + } } + + front = _injectors.front(); } - - front = _injectors.front(); } + } else { // we have no current injectors, wait until we get at least one before we do anything _injectorReady.wait(lock); @@ -110,6 +113,11 @@ void AudioInjectorManager::run() { static const int MAX_INJECTORS_PER_THREAD = 50; // calculated based on AudioInjector while loop time, with sufficient padding bool AudioInjectorManager::threadInjector(AudioInjector* injector) { + if (_shouldStop) { + qDebug() << "AudioInjectorManager::threadInjector asked to thread injector but is shutting down."; + return false; + } + // guard the injectors vector with a mutex std::unique_lock lock(_injectorsMutex); @@ -140,12 +148,14 @@ bool AudioInjectorManager::threadInjector(AudioInjector* injector) { } void AudioInjectorManager::restartFinishedInjector(AudioInjector* injector) { - // guard the injectors vector with a mutex - std::unique_lock lock(_injectorsMutex); - - // add the injector to the queue with a send timestamp of now - _injectors.emplace(usecTimestampNow(), InjectorQPointer { injector }); - - // notify our wait condition so we can inject two frames for this injector immediately - _injectorReady.notify_one(); + if (!_shouldStop) { + // guard the injectors vector with a mutex + std::unique_lock lock(_injectorsMutex); + + // add the injector to the queue with a send timestamp of now + _injectors.emplace(usecTimestampNow(), InjectorQPointer { injector }); + + // notify our wait condition so we can inject two frames for this injector immediately + _injectorReady.notify_one(); + } } From 62b218632d1d441322f72d358ef3b4b09506bc80 Mon Sep 17 00:00:00 2001 From: Stephen Birarda Date: Tue, 17 Nov 2015 17:06:21 -0800 Subject: [PATCH 13/25] don't use unneeded atomic for AudioInjector State --- libraries/audio/src/AudioInjector.cpp | 4 ++-- libraries/audio/src/AudioInjector.h | 2 +- 2 files changed, 3 insertions(+), 3 deletions(-) diff --git a/libraries/audio/src/AudioInjector.cpp b/libraries/audio/src/AudioInjector.cpp index b3435bb9b7..fc7bcfd95d 100644 --- a/libraries/audio/src/AudioInjector.cpp +++ b/libraries/audio/src/AudioInjector.cpp @@ -47,8 +47,8 @@ AudioInjector::AudioInjector(const QByteArray& audioData, const AudioInjectorOpt } void AudioInjector::finish() { - State oldState = std::atomic_exchange(&_state, State::Finished); - bool shouldDelete = (oldState == State::NotFinishedWithPendingDelete); + bool shouldDelete = (_state == State::NotFinishedWithPendingDelete); + _state = State::Finished; emit finished(); diff --git a/libraries/audio/src/AudioInjector.h b/libraries/audio/src/AudioInjector.h index 0c549eaabb..85d01c8ce6 100644 --- a/libraries/audio/src/AudioInjector.h +++ b/libraries/audio/src/AudioInjector.h @@ -88,7 +88,7 @@ private: QByteArray _audioData; AudioInjectorOptions _options; - std::atomic _state { State::NotFinished }; + State _state { State::NotFinished }; bool _hasSetup = false; bool _shouldStop = false; float _loudness = 0.0f; From cf2ad9a8776cd5568e107636c4e6e0112ef290a3 Mon Sep 17 00:00:00 2001 From: Stephen Birarda Date: Tue, 17 Nov 2015 17:07:29 -0800 Subject: [PATCH 14/25] remove atomic include from AudioInjector --- libraries/audio/src/AudioInjector.h | 2 -- 1 file changed, 2 deletions(-) diff --git a/libraries/audio/src/AudioInjector.h b/libraries/audio/src/AudioInjector.h index 85d01c8ce6..d8b168d2e5 100644 --- a/libraries/audio/src/AudioInjector.h +++ b/libraries/audio/src/AudioInjector.h @@ -12,8 +12,6 @@ #ifndef hifi_AudioInjector_h #define hifi_AudioInjector_h -#include - #include #include #include From d20fd6d7e71fc914a3eb4f5c3fef4034cfa274d6 Mon Sep 17 00:00:00 2001 From: Stephen Birarda Date: Tue, 17 Nov 2015 17:08:48 -0800 Subject: [PATCH 15/25] cap max to 40, clarify comment --- libraries/audio/src/AudioInjectorManager.cpp | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/libraries/audio/src/AudioInjectorManager.cpp b/libraries/audio/src/AudioInjectorManager.cpp index 4770371e48..ee2f9ad5fe 100644 --- a/libraries/audio/src/AudioInjectorManager.cpp +++ b/libraries/audio/src/AudioInjectorManager.cpp @@ -110,7 +110,7 @@ void AudioInjectorManager::run() { } } -static const int MAX_INJECTORS_PER_THREAD = 50; // calculated based on AudioInjector while loop time, with sufficient padding +static const int MAX_INJECTORS_PER_THREAD = 40; // calculated based on AudioInjector time to send frame, with sufficient padding bool AudioInjectorManager::threadInjector(AudioInjector* injector) { if (_shouldStop) { From 1c485bacffef9b5ed7ff3105a0aaa3eea642fdaa Mon Sep 17 00:00:00 2001 From: Stephen Birarda Date: Tue, 17 Nov 2015 17:11:40 -0800 Subject: [PATCH 16/25] include the condition_variable header in AIM --- libraries/audio/src/AudioInjectorManager.h | 1 + 1 file changed, 1 insertion(+) diff --git a/libraries/audio/src/AudioInjectorManager.h b/libraries/audio/src/AudioInjectorManager.h index 7518bb1df8..5e88ade4db 100644 --- a/libraries/audio/src/AudioInjectorManager.h +++ b/libraries/audio/src/AudioInjectorManager.h @@ -14,6 +14,7 @@ #ifndef hifi_AudioInjectorManager_h #define hifi_AudioInjectorManager_h +#include #include #include From 35dd5fb564d9ebd94fc0fbe8d099fbfefe39d15b Mon Sep 17 00:00:00 2001 From: Stephen Birarda Date: Tue, 17 Nov 2015 17:26:07 -0800 Subject: [PATCH 17/25] include NLPacket in AudioInjector for unique_ptr --- libraries/audio/src/AudioInjector.h | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/libraries/audio/src/AudioInjector.h b/libraries/audio/src/AudioInjector.h index d8b168d2e5..ae9048db83 100644 --- a/libraries/audio/src/AudioInjector.h +++ b/libraries/audio/src/AudioInjector.h @@ -20,13 +20,14 @@ #include #include +#include + #include "AudioInjectorLocalBuffer.h" #include "AudioInjectorOptions.h" #include "Sound.h" class AbstractAudioInterface; class AudioInjectorManager; -class NLPacket; // In order to make scripting cleaner for the AudioInjector, the script now holds on to the AudioInjector object // until it dies. From 2aa56b538410fd5f8be79b7bd4fb4df754973fe1 Mon Sep 17 00:00:00 2001 From: Stephen Birarda Date: Tue, 17 Nov 2015 17:26:45 -0800 Subject: [PATCH 18/25] include memory header for unique_ptr --- libraries/audio/src/AudioInjector.h | 2 ++ 1 file changed, 2 insertions(+) diff --git a/libraries/audio/src/AudioInjector.h b/libraries/audio/src/AudioInjector.h index ae9048db83..f815b6fe3a 100644 --- a/libraries/audio/src/AudioInjector.h +++ b/libraries/audio/src/AudioInjector.h @@ -12,6 +12,8 @@ #ifndef hifi_AudioInjector_h #define hifi_AudioInjector_h +#include + #include #include #include From 8f5212acac9c41a29580cde80405741bfad2abc8 Mon Sep 17 00:00:00 2001 From: Stephen Birarda Date: Thu, 19 Nov 2015 17:26:10 -0800 Subject: [PATCH 19/25] use destroy instead of accidental set --- assignment-client/src/Agent.cpp | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/assignment-client/src/Agent.cpp b/assignment-client/src/Agent.cpp index 37d3c1097e..f9c8c966a8 100644 --- a/assignment-client/src/Agent.cpp +++ b/assignment-client/src/Agent.cpp @@ -410,5 +410,5 @@ void Agent::aboutToFinish() { assetThread->wait(); // cleanup the AudioInjectorManager (and any still running injectors) - DependencyManager::set(); + DependencyManager::destroy(); } From 3b56df6e9995bb8a913008f8bf95a4c2de004984 Mon Sep 17 00:00:00 2001 From: Stephen Birarda Date: Thu, 19 Nov 2015 17:27:18 -0800 Subject: [PATCH 20/25] use deleteLater immediately not that finish is right thread --- libraries/audio/src/AudioInjector.cpp | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/libraries/audio/src/AudioInjector.cpp b/libraries/audio/src/AudioInjector.cpp index fc7bcfd95d..a0d78da7f8 100644 --- a/libraries/audio/src/AudioInjector.cpp +++ b/libraries/audio/src/AudioInjector.cpp @@ -59,8 +59,8 @@ void AudioInjector::finish() { } if (shouldDelete) { - // we've been asked to delete after finishing, trigger a queued deleteLater here - QMetaObject::invokeMethod(this, "deleteLater", Qt::QueuedConnection); + // we've been asked to delete after finishing, trigger a deleteLater here + deleteLater(); } } From adf41fce19f8d02d6e9cd0847bbf85c2a174c6bf Mon Sep 17 00:00:00 2001 From: Stephen Birarda Date: Thu, 19 Nov 2015 17:29:20 -0800 Subject: [PATCH 21/25] use alias for unique_lock and mutex --- libraries/audio/src/AudioInjectorManager.cpp | 8 ++++---- libraries/audio/src/AudioInjectorManager.h | 11 +++++++---- 2 files changed, 11 insertions(+), 8 deletions(-) diff --git a/libraries/audio/src/AudioInjectorManager.cpp b/libraries/audio/src/AudioInjectorManager.cpp index ee2f9ad5fe..862136f22f 100644 --- a/libraries/audio/src/AudioInjectorManager.cpp +++ b/libraries/audio/src/AudioInjectorManager.cpp @@ -21,7 +21,7 @@ AudioInjectorManager::~AudioInjectorManager() { _shouldStop = true; - std::unique_lock lock(_injectorsMutex); + Lock lock(_injectorsMutex); // make sure any still living injectors are stopped and deleted while (!_injectors.empty()) { @@ -61,7 +61,7 @@ void AudioInjectorManager::createThread() { void AudioInjectorManager::run() { while (!_shouldStop) { // wait until the next injector is ready, or until we get a new injector given to us - std::unique_lock lock(_injectorsMutex); + Lock lock(_injectorsMutex); if (_injectors.size() > 0) { // when does the next injector need to send a frame? @@ -119,7 +119,7 @@ bool AudioInjectorManager::threadInjector(AudioInjector* injector) { } // guard the injectors vector with a mutex - std::unique_lock lock(_injectorsMutex); + Lock lock(_injectorsMutex); // check if we'll be able to thread this injector (do we have < max concurrent injectors) if (_injectors.size() < MAX_INJECTORS_PER_THREAD) { @@ -150,7 +150,7 @@ bool AudioInjectorManager::threadInjector(AudioInjector* injector) { void AudioInjectorManager::restartFinishedInjector(AudioInjector* injector) { if (!_shouldStop) { // guard the injectors vector with a mutex - std::unique_lock lock(_injectorsMutex); + Lock lock(_injectorsMutex); // add the injector to the queue with a send timestamp of now _injectors.emplace(usecTimestampNow(), InjectorQPointer { injector }); diff --git a/libraries/audio/src/AudioInjectorManager.h b/libraries/audio/src/AudioInjectorManager.h index 5e88ade4db..67fb1a7b6c 100644 --- a/libraries/audio/src/AudioInjectorManager.h +++ b/libraries/audio/src/AudioInjectorManager.h @@ -24,9 +24,6 @@ #include class AudioInjector; -using InjectorQPointer = QPointer; -using TimeInjectorPointerPair = std::pair; -using InjectorQueue = std::queue; class AudioInjectorManager : public QObject, public Dependency { Q_OBJECT @@ -36,6 +33,12 @@ public: private slots: void run(); private: + using InjectorQPointer = QPointer; + using TimeInjectorPointerPair = std::pair; + using InjectorQueue = std::queue; + using Mutex = std::mutex; + using Lock = std::unique_lock; + bool threadInjector(AudioInjector* injector); void restartFinishedInjector(AudioInjector* injector); void notifyInjectorReadyCondition() { _injectorReady.notify_one(); } @@ -49,7 +52,7 @@ private: QThread* _thread { nullptr }; bool _shouldStop { false }; InjectorQueue _injectors; - std::mutex _injectorsMutex; + Mutex _injectorsMutex; std::condition_variable _injectorReady; friend class AudioInjector; From 5f9c3eea767ae7f408208d7210efaa865a741962 Mon Sep 17 00:00:00 2001 From: Stephen Birarda Date: Thu, 19 Nov 2015 17:42:14 -0800 Subject: [PATCH 22/25] use a priority queue to ensure ordering of injectors --- libraries/audio/src/AudioInjectorManager.cpp | 10 +++++----- libraries/audio/src/AudioInjectorManager.h | 14 +++++++++++++- 2 files changed, 18 insertions(+), 6 deletions(-) diff --git a/libraries/audio/src/AudioInjectorManager.cpp b/libraries/audio/src/AudioInjectorManager.cpp index 862136f22f..942527caf1 100644 --- a/libraries/audio/src/AudioInjectorManager.cpp +++ b/libraries/audio/src/AudioInjectorManager.cpp @@ -26,7 +26,7 @@ AudioInjectorManager::~AudioInjectorManager() { // make sure any still living injectors are stopped and deleted while (!_injectors.empty()) { // grab the injector at the front - auto& timePointerPair = _injectors.front(); + auto& timePointerPair = _injectors.top(); // ask it to stop and be deleted timePointerPair.second->stopAndDeleteLater(); @@ -67,7 +67,7 @@ void AudioInjectorManager::run() { // when does the next injector need to send a frame? // do we get to wait or should we just go for it now? - auto timeInjectorPair = _injectors.front(); + auto timeInjectorPair = _injectors.top(); auto nextTimestamp = timeInjectorPair.first; int64_t difference = int64_t(nextTimestamp - usecTimestampNow()); @@ -78,7 +78,7 @@ void AudioInjectorManager::run() { if (_injectors.size() > 0) { // loop through the injectors in the map and send whatever frames need to go out - auto front = _injectors.front(); + auto front = _injectors.top(); while (_injectors.size() > 0 && front.first <= usecTimestampNow()) { // either way we're popping this injector off - get a copy first auto injector = front.second; @@ -90,11 +90,11 @@ void AudioInjectorManager::run() { if (nextCallDelta > 0 && !injector->isFinished()) { // re-enqueue the injector with the correct timing - _injectors.push({ usecTimestampNow() + nextCallDelta, injector }); + _injectors.emplace(usecTimestampNow() + nextCallDelta, injector); } } - front = _injectors.front(); + front = _injectors.top(); } } diff --git a/libraries/audio/src/AudioInjectorManager.h b/libraries/audio/src/AudioInjectorManager.h index 67fb1a7b6c..91648fff39 100644 --- a/libraries/audio/src/AudioInjectorManager.h +++ b/libraries/audio/src/AudioInjectorManager.h @@ -33,9 +33,19 @@ public: private slots: void run(); private: + using InjectorQPointer = QPointer; using TimeInjectorPointerPair = std::pair; - using InjectorQueue = std::queue; + + struct greaterTime { + bool operator() (const TimeInjectorPointerPair& x, const TimeInjectorPointerPair& y) const { + return x.first > y.first; + } + }; + + using InjectorQueue = std::priority_queue, + greaterTime>; using Mutex = std::mutex; using Lock = std::unique_lock; @@ -58,4 +68,6 @@ private: friend class AudioInjector; }; + + #endif // hifi_AudioInjectorManager_h From c8b5f6f7376cd8403ed6717e07be8d2ca2b04d6e Mon Sep 17 00:00:00 2001 From: Stephen Birarda Date: Fri, 20 Nov 2015 16:03:18 -0800 Subject: [PATCH 23/25] fix for top on deque when empty --- libraries/audio/src/AudioInjectorManager.cpp | 7 ++++++- 1 file changed, 6 insertions(+), 1 deletion(-) diff --git a/libraries/audio/src/AudioInjectorManager.cpp b/libraries/audio/src/AudioInjectorManager.cpp index 942527caf1..f504b31907 100644 --- a/libraries/audio/src/AudioInjectorManager.cpp +++ b/libraries/audio/src/AudioInjectorManager.cpp @@ -94,7 +94,12 @@ void AudioInjectorManager::run() { } } - front = _injectors.top(); + if (_injectors.size() > 0) { + front = _injectors.top(); + } else { + // no more injectors to look at, break + break; + } } } From b24b095c90ba102cb32d8354c2202dacb62b1df0 Mon Sep 17 00:00:00 2001 From: Stephen Birarda Date: Fri, 20 Nov 2015 16:09:41 -0800 Subject: [PATCH 24/25] add a missing space in Agent --- assignment-client/src/Agent.cpp | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/assignment-client/src/Agent.cpp b/assignment-client/src/Agent.cpp index 410e75e833..3406816588 100644 --- a/assignment-client/src/Agent.cpp +++ b/assignment-client/src/Agent.cpp @@ -416,7 +416,7 @@ void Agent::processAgentAvatarAndAudio(float deltaTime) { glm::quat headOrientation = scriptedAvatar->getHeadOrientation(); audioPacket->writePrimitive(headOrientation); - }else if (nextSoundOutput) { + } else if (nextSoundOutput) { // assume scripted avatar audio is mono and set channel flag to zero audioPacket->writePrimitive((quint8)0); From 7f7d05fc42dce4f442eb9a7fbd81c69e91bbda9d Mon Sep 17 00:00:00 2001 From: Stephen Birarda Date: Tue, 24 Nov 2015 10:46:35 -0600 Subject: [PATCH 25/25] fix double initialization of AssetClient in Agent --- assignment-client/src/Agent.cpp | 8 -------- 1 file changed, 8 deletions(-) diff --git a/assignment-client/src/Agent.cpp b/assignment-client/src/Agent.cpp index a16e28df99..13904e6b75 100644 --- a/assignment-client/src/Agent.cpp +++ b/assignment-client/src/Agent.cpp @@ -52,14 +52,6 @@ Agent::Agent(NLPacket& packet) : DEFAULT_WINDOW_SECONDS_FOR_DESIRED_REDUCTION, false)) { DependencyManager::get()->setPacketSender(&_entityEditSender); - - auto assetClient = DependencyManager::set(); - - QThread* assetThread = new QThread; - assetThread->setObjectName("Asset Thread"); - assetClient->moveToThread(assetThread); - connect(assetThread, &QThread::started, assetClient.data(), &AssetClient::init); - assetThread->start(); auto assetClient = DependencyManager::set();