try to fix audio injector threading issues

This commit is contained in:
SamGondelman 2019-03-12 18:41:43 -07:00
parent c0240d2431
commit 609c4ab52e
25 changed files with 403 additions and 334 deletions

View file

@ -232,6 +232,10 @@ Item {
text: "Audio Codec: " + root.audioCodec + " Noise Gate: " + text: "Audio Codec: " + root.audioCodec + " Noise Gate: " +
root.audioNoiseGate; root.audioNoiseGate;
} }
StatText {
visible: root.expanded;
text: "Injectors (Local/NonLocal): " + root.audioInjectors.x + "/" + root.audioInjectors.y;
}
StatText { StatText {
visible: root.expanded; visible: root.expanded;
text: "Entity Servers In: " + root.entityPacketsInKbps + " kbps"; text: "Entity Servers In: " + root.entityPacketsInKbps + " kbps";

View file

@ -2695,9 +2695,7 @@ void Application::cleanupBeforeQuit() {
DependencyManager::destroy<OffscreenQmlSurfaceCache>(); DependencyManager::destroy<OffscreenQmlSurfaceCache>();
if (_snapshotSoundInjector != nullptr) { _snapshotSoundInjector = nullptr;
_snapshotSoundInjector->stop();
}
// destroy Audio so it and its threads have a chance to go down safely // destroy Audio so it and its threads have a chance to go down safely
// this must happen after QML, as there are unexplained audio crashes originating in qtwebengine // this must happen after QML, as there are unexplained audio crashes originating in qtwebengine
@ -4216,10 +4214,9 @@ void Application::keyPressEvent(QKeyEvent* event) {
Setting::Handle<bool> notificationSoundSnapshot{ MenuOption::NotificationSoundsSnapshot, true }; Setting::Handle<bool> notificationSoundSnapshot{ MenuOption::NotificationSoundsSnapshot, true };
if (notificationSounds.get() && notificationSoundSnapshot.get()) { if (notificationSounds.get() && notificationSoundSnapshot.get()) {
if (_snapshotSoundInjector) { if (_snapshotSoundInjector) {
_snapshotSoundInjector->setOptions(options); DependencyManager::get<AudioInjectorManager>()->setOptionsAndRestart(_snapshotSoundInjector, options);
_snapshotSoundInjector->restart();
} else { } else {
_snapshotSoundInjector = AudioInjector::playSound(_snapshotSound, options); _snapshotSoundInjector = DependencyManager::get<AudioInjectorManager>()->playSound(_snapshotSound, options);
} }
} }
takeSnapshot(true); takeSnapshot(true);

View file

@ -629,8 +629,7 @@ void AvatarManager::handleCollisionEvents(const CollisionEvents& collisionEvents
// but most avatars are roughly the same size, so let's not be so fancy yet. // but most avatars are roughly the same size, so let's not be so fancy yet.
const float AVATAR_STRETCH_FACTOR = 1.0f; const float AVATAR_STRETCH_FACTOR = 1.0f;
_collisionInjectors.remove_if( _collisionInjectors.remove_if([](const AudioInjectorPointer& injector) { return !injector; });
[](const AudioInjectorPointer& injector) { return !injector || injector->isFinished(); });
static const int MAX_INJECTOR_COUNT = 3; static const int MAX_INJECTOR_COUNT = 3;
if (_collisionInjectors.size() < MAX_INJECTOR_COUNT) { if (_collisionInjectors.size() < MAX_INJECTOR_COUNT) {
@ -640,7 +639,7 @@ void AvatarManager::handleCollisionEvents(const CollisionEvents& collisionEvents
options.volume = energyFactorOfFull; options.volume = energyFactorOfFull;
options.pitch = 1.0f / AVATAR_STRETCH_FACTOR; options.pitch = 1.0f / AVATAR_STRETCH_FACTOR;
auto injector = AudioInjector::playSoundAndDelete(collisionSound, options); auto injector = DependencyManager::get<AudioInjectorManager>()->playSound(collisionSound, options, true);
_collisionInjectors.emplace_back(injector); _collisionInjectors.emplace_back(injector);
} }
} }

View file

@ -24,7 +24,7 @@
#include <SimpleMovingAverage.h> #include <SimpleMovingAverage.h>
#include <shared/RateCounter.h> #include <shared/RateCounter.h>
#include <avatars-renderer/ScriptAvatar.h> #include <avatars-renderer/ScriptAvatar.h>
#include <AudioInjector.h> #include <AudioInjectorManager.h>
#include <workload/Space.h> #include <workload/Space.h>
#include <EntitySimulation.h> // for SetOfEntities #include <EntitySimulation.h> // for SetOfEntities
@ -239,7 +239,7 @@ private:
std::shared_ptr<MyAvatar> _myAvatar; std::shared_ptr<MyAvatar> _myAvatar;
quint64 _lastSendAvatarDataTime = 0; // Controls MyAvatar send data rate. quint64 _lastSendAvatarDataTime = 0; // Controls MyAvatar send data rate.
std::list<AudioInjectorPointer> _collisionInjectors; std::list<QWeakPointer<AudioInjector>> _collisionInjectors;
RateCounter<> _myAvatarSendRate; RateCounter<> _myAvatarSendRate;
int _numAvatarsUpdated { 0 }; int _numAvatarsUpdated { 0 };

View file

@ -66,7 +66,7 @@ void TTSScriptingInterface::updateLastSoundAudioInjector() {
if (_lastSoundAudioInjector) { if (_lastSoundAudioInjector) {
AudioInjectorOptions options; AudioInjectorOptions options;
options.position = DependencyManager::get<AvatarManager>()->getMyAvatarPosition(); options.position = DependencyManager::get<AvatarManager>()->getMyAvatarPosition();
_lastSoundAudioInjector->setOptions(options); DependencyManager::get<AudioInjectorManager>()->setOptions(_lastSoundAudioInjector, options);
_lastSoundAudioInjectorUpdateTimer.start(INJECTOR_INTERVAL_MS); _lastSoundAudioInjectorUpdateTimer.start(INJECTOR_INTERVAL_MS);
} }
} }
@ -143,7 +143,7 @@ void TTSScriptingInterface::speakText(const QString& textToSpeak) {
options.position = DependencyManager::get<AvatarManager>()->getMyAvatarPosition(); options.position = DependencyManager::get<AvatarManager>()->getMyAvatarPosition();
if (_lastSoundAudioInjector) { if (_lastSoundAudioInjector) {
_lastSoundAudioInjector->stop(); DependencyManager::get<AudioInjectorManager>()->stop(_lastSoundAudioInjector);
_lastSoundAudioInjectorUpdateTimer.stop(); _lastSoundAudioInjectorUpdateTimer.stop();
} }
@ -151,7 +151,7 @@ void TTSScriptingInterface::speakText(const QString& textToSpeak) {
uint32_t numSamples = (uint32_t)_lastSoundByteArray.size() / sizeof(AudioData::AudioSample); uint32_t numSamples = (uint32_t)_lastSoundByteArray.size() / sizeof(AudioData::AudioSample);
auto samples = reinterpret_cast<AudioData::AudioSample*>(_lastSoundByteArray.data()); auto samples = reinterpret_cast<AudioData::AudioSample*>(_lastSoundByteArray.data());
auto newAudioData = AudioData::make(numSamples, numChannels, samples); auto newAudioData = AudioData::make(numSamples, numChannels, samples);
_lastSoundAudioInjector = AudioInjector::playSoundAndDelete(newAudioData, options); _lastSoundAudioInjector = DependencyManager::get<AudioInjectorManager>()->playSound(newAudioData, options, true);
_lastSoundAudioInjectorUpdateTimer.start(INJECTOR_INTERVAL_MS); _lastSoundAudioInjectorUpdateTimer.start(INJECTOR_INTERVAL_MS);
#else #else
@ -161,7 +161,7 @@ void TTSScriptingInterface::speakText(const QString& textToSpeak) {
void TTSScriptingInterface::stopLastSpeech() { void TTSScriptingInterface::stopLastSpeech() {
if (_lastSoundAudioInjector) { if (_lastSoundAudioInjector) {
_lastSoundAudioInjector->stop(); DependencyManager::get<AudioInjectorManager>()->stop(_lastSoundAudioInjector);
_lastSoundAudioInjector = NULL; _lastSoundAudioInjector = nullptr;
} }
} }

View file

@ -29,7 +29,7 @@
#include <PathUtils.h> #include <PathUtils.h>
#include <ResourceManager.h> #include <ResourceManager.h>
#include <SoundCache.h> #include <SoundCache.h>
#include <AudioInjector.h> #include <AudioInjectorManager.h>
#include <RegisteredMetaTypes.h> #include <RegisteredMetaTypes.h>
#include <ui/TabletScriptingInterface.h> #include <ui/TabletScriptingInterface.h>
@ -537,7 +537,7 @@ void Keyboard::handleTriggerBegin(const QUuid& id, const PointerEvent& event) {
audioOptions.position = keyWorldPosition; audioOptions.position = keyWorldPosition;
audioOptions.volume = 0.05f; audioOptions.volume = 0.05f;
AudioInjector::playSoundAndDelete(_keySound, audioOptions); DependencyManager::get<AudioInjectorManager>()->playSound(_keySound, audioOptions, true);
int scanCode = key.getScanCode(_capsEnabled); int scanCode = key.getScanCode(_capsEnabled);
QString keyString = key.getKeyString(_capsEnabled); QString keyString = key.getKeyString(_capsEnabled);

View file

@ -19,9 +19,9 @@
#include <QtCore/QObject> #include <QtCore/QObject>
#include <QTimer> #include <QTimer>
#include <QHash> #include <QHash>
#include <QUuid>
#include <DependencyManager.h> #include <DependencyManager.h>
#include <Sound.h> #include <Sound.h>
#include <AudioInjector.h>
#include <shared/ReadWriteLockable.h> #include <shared/ReadWriteLockable.h>
#include <SettingHandle.h> #include <SettingHandle.h>

View file

@ -266,6 +266,11 @@ void Stats::updateStats(bool force) {
} }
STAT_UPDATE(audioCodec, audioClient->getSelectedAudioFormat()); STAT_UPDATE(audioCodec, audioClient->getSelectedAudioFormat());
STAT_UPDATE(audioNoiseGate, audioClient->getNoiseGateOpen() ? "Open" : "Closed"); STAT_UPDATE(audioNoiseGate, audioClient->getNoiseGateOpen() ? "Open" : "Closed");
{
int localInjectors = audioClient->getNumLocalInjectors();
int nonLocalInjectors = DependencyManager::get<AudioInjectorManager>()->getNumInjectors();
STAT_UPDATE(audioInjectors, QVector2D(localInjectors, nonLocalInjectors));
}
STAT_UPDATE(entityPacketsInKbps, octreeServerCount ? totalEntityKbps / octreeServerCount : -1); STAT_UPDATE(entityPacketsInKbps, octreeServerCount ? totalEntityKbps / octreeServerCount : -1);

View file

@ -87,6 +87,7 @@ private: \
* @property {number} audioPacketLoss - <em>Read-only.</em> * @property {number} audioPacketLoss - <em>Read-only.</em>
* @property {string} audioCodec - <em>Read-only.</em> * @property {string} audioCodec - <em>Read-only.</em>
* @property {string} audioNoiseGate - <em>Read-only.</em> * @property {string} audioNoiseGate - <em>Read-only.</em>
* @property {Vec2} audioInjectors - <em>Read-only.</em>
* @property {number} entityPacketsInKbps - <em>Read-only.</em> * @property {number} entityPacketsInKbps - <em>Read-only.</em>
* *
* @property {number} downloads - <em>Read-only.</em> * @property {number} downloads - <em>Read-only.</em>
@ -243,6 +244,7 @@ class Stats : public QQuickItem {
STATS_PROPERTY(int, audioPacketLoss, 0) STATS_PROPERTY(int, audioPacketLoss, 0)
STATS_PROPERTY(QString, audioCodec, QString()) STATS_PROPERTY(QString, audioCodec, QString())
STATS_PROPERTY(QString, audioNoiseGate, QString()) STATS_PROPERTY(QString, audioNoiseGate, QString())
STATS_PROPERTY(QVector2D, audioInjectors, QVector2D());
STATS_PROPERTY(int, entityPacketsInKbps, 0) STATS_PROPERTY(int, entityPacketsInKbps, 0)
STATS_PROPERTY(int, downloads, 0) STATS_PROPERTY(int, downloads, 0)
@ -692,6 +694,13 @@ signals:
*/ */
void audioNoiseGateChanged(); void audioNoiseGateChanged();
/**jsdoc
* Triggered when the value of the <code>audioInjectors</code> property changes.
* @function Stats.audioInjectorsChanged
* @returns {Signal}
*/
void audioInjectorsChanged();
/**jsdoc /**jsdoc
* Triggered when the value of the <code>entityPacketsInKbps</code> property changes. * Triggered when the value of the <code>entityPacketsInKbps</code> property changes.
* @function Stats.entityPacketsInKbpsChanged * @function Stats.entityPacketsInKbpsChanged

View file

@ -1354,26 +1354,28 @@ bool AudioClient::mixLocalAudioInjectors(float* mixBuffer) {
for (const AudioInjectorPointer& injector : _activeLocalAudioInjectors) { for (const AudioInjectorPointer& injector : _activeLocalAudioInjectors) {
// the lock guarantees that injectorBuffer, if found, is invariant // the lock guarantees that injectorBuffer, if found, is invariant
AudioInjectorLocalBuffer* injectorBuffer = injector->getLocalBuffer(); auto injectorBuffer = injector->getLocalBuffer();
if (injectorBuffer) { if (injectorBuffer) {
auto options = injector->getOptions();
static const int HRTF_DATASET_INDEX = 1; static const int HRTF_DATASET_INDEX = 1;
int numChannels = injector->isAmbisonic() ? AudioConstants::AMBISONIC : (injector->isStereo() ? AudioConstants::STEREO : AudioConstants::MONO); int numChannels = options.ambisonic ? AudioConstants::AMBISONIC : (options.stereo ? AudioConstants::STEREO : AudioConstants::MONO);
size_t bytesToRead = numChannels * AudioConstants::NETWORK_FRAME_BYTES_PER_CHANNEL; size_t bytesToRead = numChannels * AudioConstants::NETWORK_FRAME_BYTES_PER_CHANNEL;
// get one frame from the injector // get one frame from the injector
memset(_localScratchBuffer, 0, bytesToRead); memset(_localScratchBuffer, 0, bytesToRead);
if (0 < injectorBuffer->readData((char*)_localScratchBuffer, bytesToRead)) { if (0 < injectorBuffer->readData((char*)_localScratchBuffer, bytesToRead)) {
float gain = injector->getVolume(); float gain = options.volume;
if (injector->isAmbisonic()) { if (options.ambisonic) {
if (injector->isPositionSet()) { if (options.positionSet) {
// distance attenuation // distance attenuation
glm::vec3 relativePosition = injector->getPosition() - _positionGetter(); glm::vec3 relativePosition = options.position - _positionGetter();
float distance = glm::max(glm::length(relativePosition), EPSILON); float distance = glm::max(glm::length(relativePosition), EPSILON);
gain = gainForSource(distance, gain); gain = gainForSource(distance, gain);
} }
@ -1382,7 +1384,7 @@ bool AudioClient::mixLocalAudioInjectors(float* mixBuffer) {
// Calculate the soundfield orientation relative to the listener. // Calculate the soundfield orientation relative to the listener.
// Injector orientation can be used to align a recording to our world coordinates. // Injector orientation can be used to align a recording to our world coordinates.
// //
glm::quat relativeOrientation = injector->getOrientation() * glm::inverse(_orientationGetter()); glm::quat relativeOrientation = options.orientation * glm::inverse(_orientationGetter());
// convert from Y-up (OpenGL) to Z-up (Ambisonic) coordinate system // convert from Y-up (OpenGL) to Z-up (Ambisonic) coordinate system
float qw = relativeOrientation.w; float qw = relativeOrientation.w;
@ -1394,12 +1396,12 @@ bool AudioClient::mixLocalAudioInjectors(float* mixBuffer) {
injector->getLocalFOA().render(_localScratchBuffer, mixBuffer, HRTF_DATASET_INDEX, injector->getLocalFOA().render(_localScratchBuffer, mixBuffer, HRTF_DATASET_INDEX,
qw, qx, qy, qz, gain, AudioConstants::NETWORK_FRAME_SAMPLES_PER_CHANNEL); qw, qx, qy, qz, gain, AudioConstants::NETWORK_FRAME_SAMPLES_PER_CHANNEL);
} else if (injector->isStereo()) { } else if (options.stereo) {
if (injector->isPositionSet()) { if (options.positionSet) {
// distance attenuation // distance attenuation
glm::vec3 relativePosition = injector->getPosition() - _positionGetter(); glm::vec3 relativePosition = options.position - _positionGetter();
float distance = glm::max(glm::length(relativePosition), EPSILON); float distance = glm::max(glm::length(relativePosition), EPSILON);
gain = gainForSource(distance, gain); gain = gainForSource(distance, gain);
} }
@ -1412,10 +1414,10 @@ bool AudioClient::mixLocalAudioInjectors(float* mixBuffer) {
} else { // injector is mono } else { // injector is mono
if (injector->isPositionSet()) { if (options.positionSet) {
// distance attenuation // distance attenuation
glm::vec3 relativePosition = injector->getPosition() - _positionGetter(); glm::vec3 relativePosition = options.position - _positionGetter();
float distance = glm::max(glm::length(relativePosition), EPSILON); float distance = glm::max(glm::length(relativePosition), EPSILON);
gain = gainForSource(distance, gain); gain = gainForSource(distance, gain);
@ -1437,21 +1439,21 @@ bool AudioClient::mixLocalAudioInjectors(float* mixBuffer) {
} else { } else {
qCDebug(audioclient) << "injector has no more data, marking finished for removal"; //qCDebug(audioclient) << "injector has no more data, marking finished for removal";
injector->finishLocalInjection(); injector->finishLocalInjection();
injectorsToRemove.append(injector); injectorsToRemove.append(injector);
} }
} else { } else {
qCDebug(audioclient) << "injector has no local buffer, marking as finished for removal"; //qCDebug(audioclient) << "injector has no local buffer, marking as finished for removal";
injector->finishLocalInjection(); injector->finishLocalInjection();
injectorsToRemove.append(injector); injectorsToRemove.append(injector);
} }
} }
for (const AudioInjectorPointer& injector : injectorsToRemove) { for (const AudioInjectorPointer& injector : injectorsToRemove) {
qCDebug(audioclient) << "removing injector"; //qCDebug(audioclient) << "removing injector";
_activeLocalAudioInjectors.removeOne(injector); _activeLocalAudioInjectors.removeOne(injector);
} }
@ -1562,15 +1564,13 @@ bool AudioClient::setIsStereoInput(bool isStereoInput) {
} }
bool AudioClient::outputLocalInjector(const AudioInjectorPointer& injector) { bool AudioClient::outputLocalInjector(const AudioInjectorPointer& injector) {
AudioInjectorLocalBuffer* injectorBuffer = injector->getLocalBuffer(); auto injectorBuffer = injector->getLocalBuffer();
if (injectorBuffer) { if (injectorBuffer) {
// local injectors are on the AudioInjectorsThread, so we must guard access // local injectors are on the AudioInjectorsThread, so we must guard access
Lock lock(_injectorsMutex); Lock lock(_injectorsMutex);
if (!_activeLocalAudioInjectors.contains(injector)) { if (!_activeLocalAudioInjectors.contains(injector)) {
qCDebug(audioclient) << "adding new injector"; //qCDebug(audioclient) << "adding new injector";
_activeLocalAudioInjectors.append(injector); _activeLocalAudioInjectors.append(injector);
// move local buffer to the LocalAudioThread to avoid dataraces with AudioInjector (like stop())
injectorBuffer->setParent(nullptr);
// update the flag // update the flag
_localInjectorsAvailable.exchange(true, std::memory_order_release); _localInjectorsAvailable.exchange(true, std::memory_order_release);
@ -1586,6 +1586,11 @@ bool AudioClient::outputLocalInjector(const AudioInjectorPointer& injector) {
} }
} }
int AudioClient::getNumLocalInjectors() {
Lock lock(_injectorsMutex);
return _activeLocalAudioInjectors.size();
}
void AudioClient::outputFormatChanged() { void AudioClient::outputFormatChanged() {
_outputFrameSize = (AudioConstants::NETWORK_FRAME_SAMPLES_PER_CHANNEL * OUTPUT_CHANNEL_COUNT * _outputFormat.sampleRate()) / _outputFrameSize = (AudioConstants::NETWORK_FRAME_SAMPLES_PER_CHANNEL * OUTPUT_CHANNEL_COUNT * _outputFormat.sampleRate()) /
_desiredOutputFormat.sampleRate(); _desiredOutputFormat.sampleRate();

View file

@ -181,6 +181,8 @@ public:
bool isHeadsetPluggedIn() { return _isHeadsetPluggedIn; } bool isHeadsetPluggedIn() { return _isHeadsetPluggedIn; }
#endif #endif
int getNumLocalInjectors();
public slots: public slots:
void start(); void start();
void stop(); void stop();

View file

@ -24,9 +24,10 @@
#include "AudioRingBuffer.h" #include "AudioRingBuffer.h"
#include "AudioLogging.h" #include "AudioLogging.h"
#include "SoundCache.h" #include "SoundCache.h"
#include "AudioSRC.h"
#include "AudioHelpers.h" #include "AudioHelpers.h"
int metaType = qRegisterMetaType<AudioInjectorPointer>("AudioInjectorPointer");
AbstractAudioInterface* AudioInjector::_localAudioInterface{ nullptr }; AbstractAudioInterface* AudioInjector::_localAudioInterface{ nullptr };
AudioInjectorState operator& (AudioInjectorState lhs, AudioInjectorState rhs) { AudioInjectorState operator& (AudioInjectorState lhs, AudioInjectorState rhs) {
@ -51,26 +52,30 @@ AudioInjector::AudioInjector(AudioDataPointer audioData, const AudioInjectorOpti
{ {
} }
AudioInjector::~AudioInjector() { AudioInjector::~AudioInjector() {}
deleteLocalBuffer();
}
bool AudioInjector::stateHas(AudioInjectorState state) const { bool AudioInjector::stateHas(AudioInjectorState state) const {
return (_state & state) == state; return resultWithReadLock<bool>([&] {
return (_state & state) == state;
});
} }
void AudioInjector::setOptions(const AudioInjectorOptions& options) { void AudioInjector::setOptions(const AudioInjectorOptions& options) {
// since options.stereo is computed from the audio stream, // since options.stereo is computed from the audio stream,
// we need to copy it from existing options just in case. // we need to copy it from existing options just in case.
bool currentlyStereo = _options.stereo; withWriteLock([&] {
bool currentlyAmbisonic = _options.ambisonic; bool currentlyStereo = _options.stereo;
_options = options; bool currentlyAmbisonic = _options.ambisonic;
_options.stereo = currentlyStereo; _options = options;
_options.ambisonic = currentlyAmbisonic; _options.stereo = currentlyStereo;
_options.ambisonic = currentlyAmbisonic;
});
} }
void AudioInjector::finishNetworkInjection() { void AudioInjector::finishNetworkInjection() {
_state |= AudioInjectorState::NetworkInjectionFinished; withWriteLock([&] {
_state |= AudioInjectorState::NetworkInjectionFinished;
});
// if we are already finished with local // if we are already finished with local
// injection, then we are finished // injection, then we are finished
@ -80,35 +85,31 @@ void AudioInjector::finishNetworkInjection() {
} }
void AudioInjector::finishLocalInjection() { void AudioInjector::finishLocalInjection() {
_state |= AudioInjectorState::LocalInjectionFinished; if (QThread::currentThread() != thread()) {
if(_options.localOnly || stateHas(AudioInjectorState::NetworkInjectionFinished)) { QMetaObject::invokeMethod(this, "finishLocalInjection");
return;
}
bool localOnly = false;
withWriteLock([&] {
_state |= AudioInjectorState::LocalInjectionFinished;
localOnly = _options.localOnly;
});
if(localOnly || stateHas(AudioInjectorState::NetworkInjectionFinished)) {
finish(); finish();
} }
} }
void AudioInjector::finish() { void AudioInjector::finish() {
_state |= AudioInjectorState::Finished; withWriteLock([&] {
_state |= AudioInjectorState::Finished;
});
emit finished(); emit finished();
_localBuffer = nullptr;
deleteLocalBuffer();
} }
void AudioInjector::restart() { void AudioInjector::restart() {
// grab the AudioInjectorManager
auto injectorManager = DependencyManager::get<AudioInjectorManager>();
if (thread() != QThread::currentThread()) {
QMetaObject::invokeMethod(this, "restart");
if (!_options.localOnly) {
// notify the AudioInjectorManager to wake up in case it's waiting for new injectors
injectorManager->notifyInjectorReadyCondition();
}
return;
}
// reset the current send offset to zero // reset the current send offset to zero
_currentSendOffset = 0; _currentSendOffset = 0;
@ -121,19 +122,23 @@ void AudioInjector::restart() {
// check our state to decide if we need extra handling for the restart request // check our state to decide if we need extra handling for the restart request
if (stateHas(AudioInjectorState::Finished)) { if (stateHas(AudioInjectorState::Finished)) {
if (!inject(&AudioInjectorManager::restartFinishedInjector)) { if (!inject(&AudioInjectorManager::threadInjector)) {
qWarning() << "AudioInjector::restart failed to thread injector"; qWarning() << "AudioInjector::restart failed to thread injector";
} }
} }
} }
bool AudioInjector::inject(bool(AudioInjectorManager::*injection)(const AudioInjectorPointer&)) { bool AudioInjector::inject(bool(AudioInjectorManager::*injection)(const AudioInjectorPointer&)) {
_state = AudioInjectorState::NotFinished; AudioInjectorOptions options;
withWriteLock([&] {
_state = AudioInjectorState::NotFinished;
options = _options;
});
int byteOffset = 0; int byteOffset = 0;
if (_options.secondOffset > 0.0f) { if (options.secondOffset > 0.0f) {
int numChannels = _options.ambisonic ? 4 : (_options.stereo ? 2 : 1); int numChannels = options.ambisonic ? 4 : (options.stereo ? 2 : 1);
byteOffset = (int)(AudioConstants::SAMPLE_RATE * _options.secondOffset * numChannels); byteOffset = (int)(AudioConstants::SAMPLE_RATE * options.secondOffset * numChannels);
byteOffset *= AudioConstants::SAMPLE_SIZE; byteOffset *= AudioConstants::SAMPLE_SIZE;
} }
_currentSendOffset = byteOffset; _currentSendOffset = byteOffset;
@ -143,7 +148,7 @@ bool AudioInjector::inject(bool(AudioInjectorManager::*injection)(const AudioInj
} }
bool success = true; bool success = true;
if (!_options.localOnly) { if (!options.localOnly) {
auto injectorManager = DependencyManager::get<AudioInjectorManager>(); auto injectorManager = DependencyManager::get<AudioInjectorManager>();
if (!(*injectorManager.*injection)(sharedFromThis())) { if (!(*injectorManager.*injection)(sharedFromThis())) {
success = false; success = false;
@ -158,7 +163,8 @@ bool AudioInjector::injectLocally() {
if (_localAudioInterface) { if (_localAudioInterface) {
if (_audioData->getNumBytes() > 0) { if (_audioData->getNumBytes() > 0) {
_localBuffer = new AudioInjectorLocalBuffer(_audioData); _localBuffer = QSharedPointer<AudioInjectorLocalBuffer>(new AudioInjectorLocalBuffer(_audioData), &AudioInjectorLocalBuffer::deleteLater);
_localBuffer->moveToThread(thread());
_localBuffer->open(QIODevice::ReadOnly); _localBuffer->open(QIODevice::ReadOnly);
_localBuffer->setShouldLoop(_options.loop); _localBuffer->setShouldLoop(_options.loop);
@ -181,14 +187,6 @@ bool AudioInjector::injectLocally() {
return success; return success;
} }
void AudioInjector::deleteLocalBuffer() {
if (_localBuffer) {
_localBuffer->stop();
_localBuffer->deleteLater();
_localBuffer = nullptr;
}
}
const uchar MAX_INJECTOR_VOLUME = packFloatGainToByte(1.0f); const uchar MAX_INJECTOR_VOLUME = packFloatGainToByte(1.0f);
static const int64_t NEXT_FRAME_DELTA_ERROR_OR_FINISHED = -1; static const int64_t NEXT_FRAME_DELTA_ERROR_OR_FINISHED = -1;
static const int64_t NEXT_FRAME_DELTA_IMMEDIATELY = 0; static const int64_t NEXT_FRAME_DELTA_IMMEDIATELY = 0;
@ -220,6 +218,10 @@ int64_t AudioInjector::injectNextFrame() {
static int volumeOptionOffset = -1; static int volumeOptionOffset = -1;
static int audioDataOffset = -1; static int audioDataOffset = -1;
AudioInjectorOptions options = resultWithReadLock<AudioInjectorOptions>([&] {
return _options;
});
if (!_currentPacket) { if (!_currentPacket) {
if (_currentSendOffset < 0 || if (_currentSendOffset < 0 ||
_currentSendOffset >= (int)_audioData->getNumBytes()) { _currentSendOffset >= (int)_audioData->getNumBytes()) {
@ -253,7 +255,7 @@ int64_t AudioInjector::injectNextFrame() {
audioPacketStream << QUuid::createUuid(); audioPacketStream << QUuid::createUuid();
// pack the stereo/mono type of the stream // pack the stereo/mono type of the stream
audioPacketStream << _options.stereo; audioPacketStream << options.stereo;
// pack the flag for loopback, if requested // pack the flag for loopback, if requested
loopbackOptionOffset = _currentPacket->pos(); loopbackOptionOffset = _currentPacket->pos();
@ -262,15 +264,16 @@ int64_t AudioInjector::injectNextFrame() {
// pack the position for injected audio // pack the position for injected audio
positionOptionOffset = _currentPacket->pos(); positionOptionOffset = _currentPacket->pos();
audioPacketStream.writeRawData(reinterpret_cast<const char*>(&_options.position), audioPacketStream.writeRawData(reinterpret_cast<const char*>(&options.position),
sizeof(_options.position)); sizeof(options.position));
// pack our orientation for injected audio // pack our orientation for injected audio
audioPacketStream.writeRawData(reinterpret_cast<const char*>(&_options.orientation), audioPacketStream.writeRawData(reinterpret_cast<const char*>(&options.orientation),
sizeof(_options.orientation)); sizeof(options.orientation));
audioPacketStream.writeRawData(reinterpret_cast<const char*>(&options.position),
sizeof(options.position));
audioPacketStream.writeRawData(reinterpret_cast<const char*>(&_options.position),
sizeof(_options.position));
glm::vec3 boxCorner = glm::vec3(0); glm::vec3 boxCorner = glm::vec3(0);
audioPacketStream.writeRawData(reinterpret_cast<const char*>(&boxCorner), audioPacketStream.writeRawData(reinterpret_cast<const char*>(&boxCorner),
sizeof(glm::vec3)); sizeof(glm::vec3));
@ -283,7 +286,7 @@ int64_t AudioInjector::injectNextFrame() {
volumeOptionOffset = _currentPacket->pos(); volumeOptionOffset = _currentPacket->pos();
quint8 volume = MAX_INJECTOR_VOLUME; quint8 volume = MAX_INJECTOR_VOLUME;
audioPacketStream << volume; audioPacketStream << volume;
audioPacketStream << _options.ignorePenumbra; audioPacketStream << options.ignorePenumbra;
audioDataOffset = _currentPacket->pos(); audioDataOffset = _currentPacket->pos();
@ -313,10 +316,10 @@ int64_t AudioInjector::injectNextFrame() {
_currentPacket->writePrimitive((uchar)(_localAudioInterface && _localAudioInterface->shouldLoopbackInjectors())); _currentPacket->writePrimitive((uchar)(_localAudioInterface && _localAudioInterface->shouldLoopbackInjectors()));
_currentPacket->seek(positionOptionOffset); _currentPacket->seek(positionOptionOffset);
_currentPacket->writePrimitive(_options.position); _currentPacket->writePrimitive(options.position);
_currentPacket->writePrimitive(_options.orientation); _currentPacket->writePrimitive(options.orientation);
quint8 volume = packFloatGainToByte(_options.volume); quint8 volume = packFloatGainToByte(options.volume);
_currentPacket->seek(volumeOptionOffset); _currentPacket->seek(volumeOptionOffset);
_currentPacket->writePrimitive(volume); _currentPacket->writePrimitive(volume);
@ -326,8 +329,8 @@ int64_t AudioInjector::injectNextFrame() {
// Might be a reasonable place to do the encode step here. // Might be a reasonable place to do the encode step here.
QByteArray decodedAudio; QByteArray decodedAudio;
int totalBytesLeftToCopy = (_options.stereo ? 2 : 1) * AudioConstants::NETWORK_FRAME_BYTES_PER_CHANNEL; int totalBytesLeftToCopy = (options.stereo ? 2 : 1) * AudioConstants::NETWORK_FRAME_BYTES_PER_CHANNEL;
if (!_options.loop) { if (!options.loop) {
// If we aren't looping, let's make sure we don't read past the end // If we aren't looping, let's make sure we don't read past the end
int bytesLeftToRead = _audioData->getNumBytes() - _currentSendOffset; int bytesLeftToRead = _audioData->getNumBytes() - _currentSendOffset;
totalBytesLeftToCopy = std::min(totalBytesLeftToCopy, bytesLeftToRead); totalBytesLeftToCopy = std::min(totalBytesLeftToCopy, bytesLeftToRead);
@ -342,14 +345,16 @@ int64_t AudioInjector::injectNextFrame() {
auto samplesOut = reinterpret_cast<AudioSample*>(decodedAudio.data()); auto samplesOut = reinterpret_cast<AudioSample*>(decodedAudio.data());
// Copy and Measure the loudness of this frame // Copy and Measure the loudness of this frame
_loudness = 0.0f; withWriteLock([&] {
for (int i = 0; i < samplesLeftToCopy; ++i) { _loudness = 0.0f;
auto index = (currentSample + i) % _audioData->getNumSamples(); for (int i = 0; i < samplesLeftToCopy; ++i) {
auto sample = samples[index]; auto index = (currentSample + i) % _audioData->getNumSamples();
samplesOut[i] = sample; auto sample = samples[index];
_loudness += abs(sample) / (AudioConstants::MAX_SAMPLE_VALUE / 2.0f); samplesOut[i] = sample;
} _loudness += abs(sample) / (AudioConstants::MAX_SAMPLE_VALUE / 2.0f);
_loudness /= (float)samplesLeftToCopy; }
_loudness /= (float)samplesLeftToCopy;
});
_currentSendOffset = (_currentSendOffset + totalBytesLeftToCopy) % _currentSendOffset = (_currentSendOffset + totalBytesLeftToCopy) %
_audioData->getNumBytes(); _audioData->getNumBytes();
@ -371,7 +376,7 @@ int64_t AudioInjector::injectNextFrame() {
_outgoingSequenceNumber++; _outgoingSequenceNumber++;
} }
if (_currentSendOffset == 0 && !_options.loop) { if (_currentSendOffset == 0 && !options.loop) {
finishNetworkInjection(); finishNetworkInjection();
return NEXT_FRAME_DELTA_ERROR_OR_FINISHED; return NEXT_FRAME_DELTA_ERROR_OR_FINISHED;
} }
@ -391,134 +396,10 @@ int64_t AudioInjector::injectNextFrame() {
// If we are falling behind by more frames than our threshold, let's skip the frames ahead // If we are falling behind by more frames than our threshold, let's skip the frames ahead
qCDebug(audio) << this << "injectNextFrame() skipping ahead, fell behind by " << (currentFrameBasedOnElapsedTime - _nextFrame) << " frames"; qCDebug(audio) << this << "injectNextFrame() skipping ahead, fell behind by " << (currentFrameBasedOnElapsedTime - _nextFrame) << " frames";
_nextFrame = currentFrameBasedOnElapsedTime; _nextFrame = currentFrameBasedOnElapsedTime;
_currentSendOffset = _nextFrame * AudioConstants::NETWORK_FRAME_BYTES_PER_CHANNEL * (_options.stereo ? 2 : 1) % _audioData->getNumBytes(); _currentSendOffset = _nextFrame * AudioConstants::NETWORK_FRAME_BYTES_PER_CHANNEL * (options.stereo ? 2 : 1) % _audioData->getNumBytes();
} }
int64_t playNextFrameAt = ++_nextFrame * AudioConstants::NETWORK_FRAME_USECS; int64_t playNextFrameAt = ++_nextFrame * AudioConstants::NETWORK_FRAME_USECS;
return std::max(INT64_C(0), playNextFrameAt - currentTime); return std::max(INT64_C(0), playNextFrameAt - currentTime);
}
void AudioInjector::stop() {
// trigger a call on the injector's thread to change state to finished
QMetaObject::invokeMethod(this, "finish");
}
void AudioInjector::triggerDeleteAfterFinish() {
// make sure this fires on the AudioInjector thread
if (thread() != QThread::currentThread()) {
QMetaObject::invokeMethod(this, "triggerDeleteAfterFinish", Qt::QueuedConnection);
return;
}
if (stateHas(AudioInjectorState::Finished)) {
stop();
} else {
_state |= AudioInjectorState::PendingDelete;
}
}
AudioInjectorPointer AudioInjector::playSoundAndDelete(SharedSoundPointer sound, const AudioInjectorOptions& options) {
AudioInjectorPointer injector = playSound(sound, options);
if (injector) {
injector->_state |= AudioInjectorState::PendingDelete;
}
return injector;
}
AudioInjectorPointer AudioInjector::playSound(SharedSoundPointer sound, const AudioInjectorOptions& options) {
if (!sound || !sound->isReady()) {
return AudioInjectorPointer();
}
if (options.pitch == 1.0f) {
AudioInjectorPointer injector = AudioInjectorPointer::create(sound, options);
if (!injector->inject(&AudioInjectorManager::threadInjector)) {
qWarning() << "AudioInjector::playSound failed to thread injector";
}
return injector;
} else {
using AudioConstants::AudioSample;
using AudioConstants::SAMPLE_RATE;
const int standardRate = SAMPLE_RATE;
// limit pitch to 4 octaves
const float pitch = glm::clamp(options.pitch, 1 / 16.0f, 16.0f);
const int resampledRate = glm::round(SAMPLE_RATE / pitch);
auto audioData = sound->getAudioData();
auto numChannels = audioData->getNumChannels();
auto numFrames = audioData->getNumFrames();
AudioSRC resampler(standardRate, resampledRate, numChannels);
// create a resampled buffer that is guaranteed to be large enough
const int maxOutputFrames = resampler.getMaxOutput(numFrames);
const int maxOutputSize = maxOutputFrames * numChannels * sizeof(AudioSample);
QByteArray resampledBuffer(maxOutputSize, '\0');
auto bufferPtr = reinterpret_cast<AudioSample*>(resampledBuffer.data());
resampler.render(audioData->data(), bufferPtr, numFrames);
int numSamples = maxOutputFrames * numChannels;
auto newAudioData = AudioData::make(numSamples, numChannels, bufferPtr);
AudioInjectorPointer injector = AudioInjectorPointer::create(newAudioData, options);
if (!injector->inject(&AudioInjectorManager::threadInjector)) {
qWarning() << "AudioInjector::playSound failed to thread pitch-shifted injector";
}
return injector;
}
}
AudioInjectorPointer AudioInjector::playSoundAndDelete(AudioDataPointer audioData, const AudioInjectorOptions& options) {
AudioInjectorPointer injector = playSound(audioData, options);
if (injector) {
injector->_state |= AudioInjectorState::PendingDelete;
}
return injector;
}
AudioInjectorPointer AudioInjector::playSound(AudioDataPointer audioData, const AudioInjectorOptions& options) {
if (options.pitch == 1.0f) {
AudioInjectorPointer injector = AudioInjectorPointer::create(audioData, options);
if (!injector->inject(&AudioInjectorManager::threadInjector)) {
qWarning() << "AudioInjector::playSound failed to thread pitch-shifted injector";
}
return injector;
} else {
using AudioConstants::AudioSample;
using AudioConstants::SAMPLE_RATE;
const int standardRate = SAMPLE_RATE;
// limit pitch to 4 octaves
const float pitch = glm::clamp(options.pitch, 1 / 16.0f, 16.0f);
const int resampledRate = glm::round(SAMPLE_RATE / pitch);
auto numChannels = audioData->getNumChannels();
auto numFrames = audioData->getNumFrames();
AudioSRC resampler(standardRate, resampledRate, numChannels);
// create a resampled buffer that is guaranteed to be large enough
const int maxOutputFrames = resampler.getMaxOutput(numFrames);
const int maxOutputSize = maxOutputFrames * numChannels * sizeof(AudioSample);
QByteArray resampledBuffer(maxOutputSize, '\0');
auto bufferPtr = reinterpret_cast<AudioSample*>(resampledBuffer.data());
resampler.render(audioData->data(), bufferPtr, numFrames);
int numSamples = maxOutputFrames * numChannels;
auto newAudioData = AudioData::make(numSamples, numChannels, bufferPtr);
return AudioInjector::playSound(newAudioData, options);
}
} }

View file

@ -19,6 +19,8 @@
#include <QtCore/QSharedPointer> #include <QtCore/QSharedPointer>
#include <QtCore/QThread> #include <QtCore/QThread>
#include <shared/ReadWriteLockable.h>
#include <glm/glm.hpp> #include <glm/glm.hpp>
#include <glm/gtx/quaternion.hpp> #include <glm/gtx/quaternion.hpp>
@ -49,7 +51,7 @@ AudioInjectorState& operator|= (AudioInjectorState& lhs, AudioInjectorState rhs)
// In order to make scripting cleaner for the AudioInjector, the script now holds on to the AudioInjector object // In order to make scripting cleaner for the AudioInjector, the script now holds on to the AudioInjector object
// until it dies. // until it dies.
class AudioInjector : public QObject, public QEnableSharedFromThis<AudioInjector> { class AudioInjector : public QObject, public QEnableSharedFromThis<AudioInjector>, public ReadWriteLockable {
Q_OBJECT Q_OBJECT
public: public:
AudioInjector(SharedSoundPointer sound, const AudioInjectorOptions& injectorOptions); AudioInjector(SharedSoundPointer sound, const AudioInjectorOptions& injectorOptions);
@ -61,38 +63,30 @@ public:
int getCurrentSendOffset() const { return _currentSendOffset; } int getCurrentSendOffset() const { return _currentSendOffset; }
void setCurrentSendOffset(int currentSendOffset) { _currentSendOffset = currentSendOffset; } void setCurrentSendOffset(int currentSendOffset) { _currentSendOffset = currentSendOffset; }
AudioInjectorLocalBuffer* getLocalBuffer() const { return _localBuffer; } QSharedPointer<AudioInjectorLocalBuffer> getLocalBuffer() const { return _localBuffer; }
AudioHRTF& getLocalHRTF() { return _localHRTF; } AudioHRTF& getLocalHRTF() { return _localHRTF; }
AudioFOA& getLocalFOA() { return _localFOA; } AudioFOA& getLocalFOA() { return _localFOA; }
bool isLocalOnly() const { return _options.localOnly; } float getLoudness() const { return resultWithReadLock<float>([&] { return _loudness; }); }
float getVolume() const { return _options.volume; } bool isPlaying() const { return !stateHas(AudioInjectorState::Finished); }
bool isPositionSet() const { return _options.positionSet; }
glm::vec3 getPosition() const { return _options.position; } bool isLocalOnly() const { return resultWithReadLock<bool>([&] { return _options.localOnly; }); }
glm::quat getOrientation() const { return _options.orientation; } float getVolume() const { return resultWithReadLock<float>([&] { return _options.volume; }); }
bool isStereo() const { return _options.stereo; } bool isPositionSet() const { return resultWithReadLock<bool>([&] { return _options.positionSet; }); }
bool isAmbisonic() const { return _options.ambisonic; } glm::vec3 getPosition() const { return resultWithReadLock<glm::vec3>([&] { return _options.position; }); }
glm::quat getOrientation() const { return resultWithReadLock<glm::quat>([&] { return _options.orientation; }); }
bool isStereo() const { return resultWithReadLock<bool>([&] { return _options.stereo; }); }
bool isAmbisonic() const { return resultWithReadLock<bool>([&] { return _options.ambisonic; }); }
const AudioInjectorOptions& getOptions() const { return resultWithReadLock<AudioInjectorOptions>([&] { return _options; }); }
void setOptions(const AudioInjectorOptions& options);
bool stateHas(AudioInjectorState state) const ; bool stateHas(AudioInjectorState state) const ;
static void setLocalAudioInterface(AbstractAudioInterface* audioInterface) { _localAudioInterface = audioInterface; } static void setLocalAudioInterface(AbstractAudioInterface* audioInterface) { _localAudioInterface = audioInterface; }
static AudioInjectorPointer playSoundAndDelete(SharedSoundPointer sound, const AudioInjectorOptions& options);
static AudioInjectorPointer playSound(SharedSoundPointer sound, const AudioInjectorOptions& options);
static AudioInjectorPointer playSoundAndDelete(AudioDataPointer audioData, const AudioInjectorOptions& options);
static AudioInjectorPointer playSound(AudioDataPointer audioData, const AudioInjectorOptions& options);
public slots:
void restart(); void restart();
void stop();
void triggerDeleteAfterFinish();
const AudioInjectorOptions& getOptions() const { return _options; }
void setOptions(const AudioInjectorOptions& options);
float getLoudness() const { return _loudness; }
bool isPlaying() const { return !stateHas(AudioInjectorState::Finished); }
void finish(); void finish();
void finishLocalInjection(); void finishLocalInjection();
void finishNetworkInjection(); void finishNetworkInjection();
@ -104,7 +98,6 @@ private:
int64_t injectNextFrame(); int64_t injectNextFrame();
bool inject(bool(AudioInjectorManager::*injection)(const AudioInjectorPointer&)); bool inject(bool(AudioInjectorManager::*injection)(const AudioInjectorPointer&));
bool injectLocally(); bool injectLocally();
void deleteLocalBuffer();
static AbstractAudioInterface* _localAudioInterface; static AbstractAudioInterface* _localAudioInterface;
@ -116,7 +109,7 @@ private:
float _loudness { 0.0f }; float _loudness { 0.0f };
int _currentSendOffset { 0 }; int _currentSendOffset { 0 };
std::unique_ptr<NLPacket> _currentPacket { nullptr }; std::unique_ptr<NLPacket> _currentPacket { nullptr };
AudioInjectorLocalBuffer* _localBuffer { nullptr }; QSharedPointer<AudioInjectorLocalBuffer> _localBuffer { nullptr };
int64_t _nextFrame { 0 }; int64_t _nextFrame { 0 };
std::unique_ptr<QElapsedTimer> _frameTimer { nullptr }; std::unique_ptr<QElapsedTimer> _frameTimer { nullptr };
@ -128,4 +121,6 @@ private:
friend class AudioInjectorManager; friend class AudioInjectorManager;
}; };
Q_DECLARE_METATYPE(AudioInjectorPointer)
#endif // hifi_AudioInjector_h #endif // hifi_AudioInjector_h

View file

@ -16,6 +16,10 @@ AudioInjectorLocalBuffer::AudioInjectorLocalBuffer(AudioDataPointer audioData) :
{ {
} }
AudioInjectorLocalBuffer::~AudioInjectorLocalBuffer() {
stop();
}
void AudioInjectorLocalBuffer::stop() { void AudioInjectorLocalBuffer::stop() {
_isStopped = true; _isStopped = true;
@ -30,9 +34,8 @@ bool AudioInjectorLocalBuffer::seek(qint64 pos) {
} }
} }
qint64 AudioInjectorLocalBuffer::readData(char* data, qint64 maxSize) { qint64 AudioInjectorLocalBuffer::readData(char* data, qint64 maxSize) {
if (!_isStopped) { if (!_isStopped && _audioData) {
// first copy to the end of the raw audio // first copy to the end of the raw audio
int bytesToEnd = (int)_audioData->getNumBytes() - _currentOffset; int bytesToEnd = (int)_audioData->getNumBytes() - _currentOffset;

View file

@ -22,6 +22,7 @@ class AudioInjectorLocalBuffer : public QIODevice {
Q_OBJECT Q_OBJECT
public: public:
AudioInjectorLocalBuffer(AudioDataPointer audioData); AudioInjectorLocalBuffer(AudioDataPointer audioData);
~AudioInjectorLocalBuffer();
void stop(); void stop();

View file

@ -14,11 +14,14 @@
#include <QtCore/QCoreApplication> #include <QtCore/QCoreApplication>
#include <SharedUtil.h> #include <SharedUtil.h>
#include <shared/QtHelpers.h>
#include "AudioConstants.h" #include "AudioConstants.h"
#include "AudioInjector.h" #include "AudioInjector.h"
#include "AudioLogging.h" #include "AudioLogging.h"
#include "AudioSRC.h"
AudioInjectorManager::~AudioInjectorManager() { AudioInjectorManager::~AudioInjectorManager() {
_shouldStop = true; _shouldStop = true;
@ -30,7 +33,7 @@ AudioInjectorManager::~AudioInjectorManager() {
auto& timePointerPair = _injectors.top(); auto& timePointerPair = _injectors.top();
// ask it to stop and be deleted // ask it to stop and be deleted
timePointerPair.second->stop(); timePointerPair.second->finish();
_injectors.pop(); _injectors.pop();
} }
@ -46,6 +49,8 @@ AudioInjectorManager::~AudioInjectorManager() {
_thread->quit(); _thread->quit();
_thread->wait(); _thread->wait();
} }
moveToThread(qApp->thread());
} }
void AudioInjectorManager::createThread() { void AudioInjectorManager::createThread() {
@ -55,6 +60,8 @@ void AudioInjectorManager::createThread() {
// when the thread is started, have it call our run to handle injection of audio // when the thread is started, have it call our run to handle injection of audio
connect(_thread, &QThread::started, this, &AudioInjectorManager::run, Qt::DirectConnection); connect(_thread, &QThread::started, this, &AudioInjectorManager::run, Qt::DirectConnection);
moveToThread(_thread);
// start the thread // start the thread
_thread->start(); _thread->start();
} }
@ -141,36 +148,7 @@ bool AudioInjectorManager::wouldExceedLimits() { // Should be called inside of a
bool AudioInjectorManager::threadInjector(const AudioInjectorPointer& injector) { bool AudioInjectorManager::threadInjector(const AudioInjectorPointer& injector) {
if (_shouldStop) { if (_shouldStop) {
qCDebug(audio) << "AudioInjectorManager::threadInjector asked to thread injector but is shutting down."; qCDebug(audio) << "AudioInjectorManager::threadInjector asked to thread injector but is shutting down.";
return false;
}
// guard the injectors vector with a mutex
Lock lock(_injectorsMutex);
if (wouldExceedLimits()) {
return false;
} else {
if (!_thread) {
createThread();
}
// move the injector to the QThread
injector->moveToThread(_thread);
// add the injector to the queue with a send timestamp of now
_injectors.emplace(usecTimestampNow(), injector);
// notify our wait condition so we can inject two frames for this injector immediately
_injectorReady.notify_one();
return true;
}
}
bool AudioInjectorManager::restartFinishedInjector(const AudioInjectorPointer& injector) {
if (_shouldStop) {
qCDebug(audio) << "AudioInjectorManager::threadInjector asked to thread injector but is shutting down.";
return false; return false;
} }
@ -188,3 +166,192 @@ bool AudioInjectorManager::restartFinishedInjector(const AudioInjectorPointer& i
} }
return true; return true;
} }
AudioInjectorPointer AudioInjectorManager::playSound(const SharedSoundPointer& sound, const AudioInjectorOptions& options, bool setPendingDelete) {
if (_shouldStop) {
qCDebug(audio) << "AudioInjectorManager::threadInjector asked to thread injector but is shutting down.";
return nullptr;
}
AudioInjectorPointer injector = nullptr;
if (sound && sound->isReady()) {
if (options.pitch == 1.0f) {
injector = QSharedPointer<AudioInjector>(new AudioInjector(sound, options), &AudioInjector::deleteLater);
} else {
using AudioConstants::AudioSample;
using AudioConstants::SAMPLE_RATE;
const int standardRate = SAMPLE_RATE;
// limit pitch to 4 octaves
const float pitch = glm::clamp(options.pitch, 1 / 16.0f, 16.0f);
const int resampledRate = glm::round(SAMPLE_RATE / pitch);
auto audioData = sound->getAudioData();
auto numChannels = audioData->getNumChannels();
auto numFrames = audioData->getNumFrames();
AudioSRC resampler(standardRate, resampledRate, numChannels);
// create a resampled buffer that is guaranteed to be large enough
const int maxOutputFrames = resampler.getMaxOutput(numFrames);
const int maxOutputSize = maxOutputFrames * numChannels * sizeof(AudioSample);
QByteArray resampledBuffer(maxOutputSize, '\0');
auto bufferPtr = reinterpret_cast<AudioSample*>(resampledBuffer.data());
resampler.render(audioData->data(), bufferPtr, numFrames);
int numSamples = maxOutputFrames * numChannels;
auto newAudioData = AudioData::make(numSamples, numChannels, bufferPtr);
injector = QSharedPointer<AudioInjector>(new AudioInjector(newAudioData, options), &AudioInjector::deleteLater);
}
}
if (!injector) {
return nullptr;
}
if (setPendingDelete) {
injector->_state |= AudioInjectorState::PendingDelete;
}
injector->moveToThread(_thread);
injector->inject(&AudioInjectorManager::threadInjector);
return injector;
}
AudioInjectorPointer AudioInjectorManager::playSound(const AudioDataPointer& audioData, const AudioInjectorOptions& options, bool setPendingDelete) {
if (_shouldStop) {
qCDebug(audio) << "AudioInjectorManager::threadInjector asked to thread injector but is shutting down.";
return nullptr;
}
AudioInjectorPointer injector = nullptr;
if (options.pitch == 1.0f) {
injector = QSharedPointer<AudioInjector>(new AudioInjector(audioData, options), &AudioInjector::deleteLater);
} else {
using AudioConstants::AudioSample;
using AudioConstants::SAMPLE_RATE;
const int standardRate = SAMPLE_RATE;
// limit pitch to 4 octaves
const float pitch = glm::clamp(options.pitch, 1 / 16.0f, 16.0f);
const int resampledRate = glm::round(SAMPLE_RATE / pitch);
auto numChannels = audioData->getNumChannels();
auto numFrames = audioData->getNumFrames();
AudioSRC resampler(standardRate, resampledRate, numChannels);
// create a resampled buffer that is guaranteed to be large enough
const int maxOutputFrames = resampler.getMaxOutput(numFrames);
const int maxOutputSize = maxOutputFrames * numChannels * sizeof(AudioSample);
QByteArray resampledBuffer(maxOutputSize, '\0');
auto bufferPtr = reinterpret_cast<AudioSample*>(resampledBuffer.data());
resampler.render(audioData->data(), bufferPtr, numFrames);
int numSamples = maxOutputFrames * numChannels;
auto newAudioData = AudioData::make(numSamples, numChannels, bufferPtr);
injector = QSharedPointer<AudioInjector>(new AudioInjector(newAudioData, options), &AudioInjector::deleteLater);
}
if (!injector) {
return nullptr;
}
if (setPendingDelete) {
injector->_state |= AudioInjectorState::PendingDelete;
}
injector->moveToThread(_thread);
injector->inject(&AudioInjectorManager::threadInjector);
return injector;
}
void AudioInjectorManager::setOptionsAndRestart(const AudioInjectorPointer& injector, const AudioInjectorOptions& options) {
if (!injector) {
return;
}
if (QThread::currentThread() != _thread) {
QMetaObject::invokeMethod(this, "setOptionsAndRestart", Q_ARG(const AudioInjectorPointer&, injector), Q_ARG(const AudioInjectorOptions&, options));
_injectorReady.notify_one();
return;
}
injector->setOptions(options);
injector->restart();
}
void AudioInjectorManager::restart(const AudioInjectorPointer& injector) {
if (!injector) {
return;
}
if (QThread::currentThread() != _thread) {
QMetaObject::invokeMethod(this, "restart", Q_ARG(const AudioInjectorPointer&, injector));
_injectorReady.notify_one();
return;
}
injector->restart();
}
void AudioInjectorManager::setOptions(const AudioInjectorPointer& injector, const AudioInjectorOptions& options) {
if (!injector) {
return;
}
if (QThread::currentThread() != _thread) {
QMetaObject::invokeMethod(this, "setOptions", Q_ARG(const AudioInjectorPointer&, injector), Q_ARG(const AudioInjectorOptions&, options));
_injectorReady.notify_one();
return;
}
injector->setOptions(options);
}
AudioInjectorOptions AudioInjectorManager::getOptions(const AudioInjectorPointer& injector) {
if (!injector) {
return AudioInjectorOptions();
}
return injector->getOptions();
}
float AudioInjectorManager::getLoudness(const AudioInjectorPointer& injector) {
if (!injector) {
return 0.0f;
}
return injector->getLoudness();
}
bool AudioInjectorManager::isPlaying(const AudioInjectorPointer& injector) {
if (!injector) {
return false;
}
return injector->isPlaying();
}
void AudioInjectorManager::stop(const AudioInjectorPointer& injector) {
if (!injector) {
return;
}
if (QThread::currentThread() != _thread) {
QMetaObject::invokeMethod(this, "stop", Q_ARG(const AudioInjectorPointer&, injector));
_injectorReady.notify_one();
return;
}
injector->finish();
}
size_t AudioInjectorManager::getNumInjectors() {
Lock lock(_injectorsMutex);
return _injectors.size();
}

View file

@ -30,8 +30,27 @@ class AudioInjectorManager : public QObject, public Dependency {
SINGLETON_DEPENDENCY SINGLETON_DEPENDENCY
public: public:
~AudioInjectorManager(); ~AudioInjectorManager();
AudioInjectorPointer playSound(const SharedSoundPointer& sound, const AudioInjectorOptions& options, bool setPendingDelete = false);
AudioInjectorPointer playSound(const AudioDataPointer& audioData, const AudioInjectorOptions& options, bool setPendingDelete = false);
size_t getNumInjectors();
public slots:
void setOptionsAndRestart(const AudioInjectorPointer& injector, const AudioInjectorOptions& options);
void restart(const AudioInjectorPointer& injector);
void setOptions(const AudioInjectorPointer& injector, const AudioInjectorOptions& options);
AudioInjectorOptions getOptions(const AudioInjectorPointer& injector);
float getLoudness(const AudioInjectorPointer& injector);
bool isPlaying(const AudioInjectorPointer& injector);
void stop(const AudioInjectorPointer& injector);
private slots: private slots:
void run(); void run();
private: private:
using TimeInjectorPointerPair = std::pair<uint64_t, AudioInjectorPointer>; using TimeInjectorPointerPair = std::pair<uint64_t, AudioInjectorPointer>;
@ -49,11 +68,10 @@ private:
using Lock = std::unique_lock<Mutex>; using Lock = std::unique_lock<Mutex>;
bool threadInjector(const AudioInjectorPointer& injector); bool threadInjector(const AudioInjectorPointer& injector);
bool restartFinishedInjector(const AudioInjectorPointer& injector);
void notifyInjectorReadyCondition() { _injectorReady.notify_one(); } void notifyInjectorReadyCondition() { _injectorReady.notify_one(); }
bool wouldExceedLimits(); bool wouldExceedLimits();
AudioInjectorManager() {}; AudioInjectorManager() { createThread(); }
AudioInjectorManager(const AudioInjectorManager&) = delete; AudioInjectorManager(const AudioInjectorManager&) = delete;
AudioInjectorManager& operator=(const AudioInjectorManager&) = delete; AudioInjectorManager& operator=(const AudioInjectorManager&) = delete;

View file

@ -1105,7 +1105,7 @@ void EntityTreeRenderer::playEntityCollisionSound(const EntityItemPointer& entit
options.volume = volume; options.volume = volume;
options.pitch = 1.0f / stretchFactor; options.pitch = 1.0f / stretchFactor;
AudioInjector::playSoundAndDelete(collisionSound, options); DependencyManager::get<AudioInjectorManager>()->playSound(collisionSound, options, true);
} }
void EntityTreeRenderer::entityCollisionWithEntity(const EntityItemID& idA, const EntityItemID& idB, void EntityTreeRenderer::entityCollisionWithEntity(const EntityItemID& idA, const EntityItemID& idB,

View file

@ -16,7 +16,7 @@
#include <QtCore/QStack> #include <QtCore/QStack>
#include <QtGui/QMouseEvent> #include <QtGui/QMouseEvent>
#include <AbstractAudioInterface.h> #include <AudioInjectorManager.h>
#include <EntityScriptingInterface.h> // for RayToEntityIntersectionResult #include <EntityScriptingInterface.h> // for RayToEntityIntersectionResult
#include <EntityTree.h> #include <EntityTree.h>
#include <PointerEvent.h> #include <PointerEvent.h>

View file

@ -63,7 +63,7 @@ ScriptAudioInjector* AudioScriptingInterface::playSound(SharedSoundPointer sound
optionsCopy.ambisonic = sound->isAmbisonic(); optionsCopy.ambisonic = sound->isAmbisonic();
optionsCopy.localOnly = optionsCopy.localOnly || sound->isAmbisonic(); // force localOnly when Ambisonic optionsCopy.localOnly = optionsCopy.localOnly || sound->isAmbisonic(); // force localOnly when Ambisonic
auto injector = AudioInjector::playSound(sound, optionsCopy); auto injector = DependencyManager::get<AudioInjectorManager>()->playSound(sound, optionsCopy);
if (!injector) { if (!injector) {
return nullptr; return nullptr;
} }

View file

@ -20,8 +20,11 @@ QScriptValue injectorToScriptValue(QScriptEngine* engine, ScriptAudioInjector* c
} }
// when the script goes down we want to cleanup the injector // when the script goes down we want to cleanup the injector
QObject::connect(engine, &QScriptEngine::destroyed, in, &ScriptAudioInjector::stopInjectorImmediately, QObject::connect(engine, &QScriptEngine::destroyed, DependencyManager::get<AudioInjectorManager>().data(), [&] {
Qt::DirectConnection); qCDebug(scriptengine) << "Script was shutdown, stopping an injector";
// FIXME: this doesn't work and leaves the injectors lying around
//DependencyManager::get<AudioInjectorManager>()->stop(in->_injector);
});
return engine->newQObject(in, QScriptEngine::ScriptOwnership); return engine->newQObject(in, QScriptEngine::ScriptOwnership);
} }
@ -37,13 +40,5 @@ ScriptAudioInjector::ScriptAudioInjector(const AudioInjectorPointer& injector) :
} }
ScriptAudioInjector::~ScriptAudioInjector() { ScriptAudioInjector::~ScriptAudioInjector() {
if (!_injector.isNull()) { DependencyManager::get<AudioInjectorManager>()->stop(_injector);
// we've been asked to delete after finishing, trigger a queued deleteLater here }
_injector->triggerDeleteAfterFinish();
}
}
void ScriptAudioInjector::stopInjectorImmediately() {
qCDebug(scriptengine) << "ScriptAudioInjector::stopInjectorImmediately called to stop audio injector immediately.";
_injector->stop();
}

View file

@ -14,7 +14,7 @@
#include <QtCore/QObject> #include <QtCore/QObject>
#include <AudioInjector.h> #include <AudioInjectorManager.h>
/**jsdoc /**jsdoc
* Plays &mdash; "injects" &mdash; the content of an audio file. Used in the {@link Audio} API. * Plays &mdash; "injects" &mdash; the content of an audio file. Used in the {@link Audio} API.
@ -48,7 +48,7 @@ public slots:
* Stop current playback, if any, and start playing from the beginning. * Stop current playback, if any, and start playing from the beginning.
* @function AudioInjector.restart * @function AudioInjector.restart
*/ */
void restart() { _injector->restart(); } void restart() { DependencyManager::get<AudioInjectorManager>()->restart(_injector); }
/**jsdoc /**jsdoc
* Stop audio playback. * Stop audio playback.
@ -68,28 +68,28 @@ public slots:
* injector.stop(); * injector.stop();
* }, 2000); * }, 2000);
*/ */
void stop() { _injector->stop(); } void stop() { DependencyManager::get<AudioInjectorManager>()->stop(_injector); }
/**jsdoc /**jsdoc
* Get the current configuration of the audio injector. * Get the current configuration of the audio injector.
* @function AudioInjector.getOptions * @function AudioInjector.getOptions
* @returns {AudioInjector.AudioInjectorOptions} Configuration of how the injector plays the audio. * @returns {AudioInjector.AudioInjectorOptions} Configuration of how the injector plays the audio.
*/ */
const AudioInjectorOptions& getOptions() const { return _injector->getOptions(); } AudioInjectorOptions getOptions() const { return DependencyManager::get<AudioInjectorManager>()->getOptions(_injector); }
/**jsdoc /**jsdoc
* Configure how the injector plays the audio. * Configure how the injector plays the audio.
* @function AudioInjector.setOptions * @function AudioInjector.setOptions
* @param {AudioInjector.AudioInjectorOptions} options - Configuration of how the injector plays the audio. * @param {AudioInjector.AudioInjectorOptions} options - Configuration of how the injector plays the audio.
*/ */
void setOptions(const AudioInjectorOptions& options) { _injector->setOptions(options); } void setOptions(const AudioInjectorOptions& options) { DependencyManager::get<AudioInjectorManager>()->setOptions(_injector, options); }
/**jsdoc /**jsdoc
* Get the loudness of the most recent frame of audio played. * Get the loudness of the most recent frame of audio played.
* @function AudioInjector.getLoudness * @function AudioInjector.getLoudness
* @returns {number} The loudness of the most recent frame of audio played, range <code>0.0</code> &ndash; <code>1.0</code>. * @returns {number} The loudness of the most recent frame of audio played, range <code>0.0</code> &ndash; <code>1.0</code>.
*/ */
float getLoudness() const { return _injector->getLoudness(); } float getLoudness() const { return DependencyManager::get<AudioInjectorManager>()->getLoudness(_injector); }
/**jsdoc /**jsdoc
* Get whether or not the audio is currently playing. * Get whether or not the audio is currently playing.
@ -110,7 +110,7 @@ public slots:
* print("Sound is playing: " + injector.isPlaying()); * print("Sound is playing: " + injector.isPlaying());
* }, 2000); * }, 2000);
*/ */
bool isPlaying() const { return _injector->isPlaying(); } bool isPlaying() const { return DependencyManager::get<AudioInjectorManager>()->isPlaying(_injector); }
signals: signals:
@ -134,13 +134,6 @@ signals:
*/ */
void finished(); void finished();
protected slots:
/**jsdoc
* Stop audio playback. (Synonym of {@link AudioInjector.stop|stop}.)
* @function AudioInjector.stopInjectorImmediately
*/
void stopInjectorImmediately();
private: private:
AudioInjectorPointer _injector; AudioInjectorPointer _injector;

View file

@ -23,7 +23,7 @@
#include "ToolbarScriptingInterface.h" #include "ToolbarScriptingInterface.h"
#include "Logging.h" #include "Logging.h"
#include <AudioInjector.h> #include <AudioInjectorManager.h>
#include "SettingHandle.h" #include "SettingHandle.h"
@ -212,7 +212,7 @@ void TabletScriptingInterface::playSound(TabletAudioEvents aEvent) {
options.localOnly = true; options.localOnly = true;
options.positionSet = false; // system sound options.positionSet = false; // system sound
AudioInjectorPointer injector = AudioInjector::playSoundAndDelete(sound, options); DependencyManager::get<AudioInjectorManager>()->playSound(sound, options, true);
} }
} }

View file

@ -2,12 +2,10 @@
#include "SoundEffect.h" #include "SoundEffect.h"
#include <RegisteredMetaTypes.h> #include <RegisteredMetaTypes.h>
#include <AudioInjector.h>
SoundEffect::~SoundEffect() { SoundEffect::~SoundEffect() {
if (_injector) { if (_injector) {
// stop will cause the AudioInjector to delete itself. DependencyManager::get<AudioInjectorManager>()->stop(_injector);
_injector->stop();
} }
} }
@ -28,15 +26,14 @@ void SoundEffect::setVolume(float volume) {
_volume = volume; _volume = volume;
} }
void SoundEffect::play(QVariant position) { void SoundEffect::play(const QVariant& position) {
AudioInjectorOptions options; AudioInjectorOptions options;
options.position = vec3FromVariant(position); options.position = vec3FromVariant(position);
options.localOnly = true; options.localOnly = true;
options.volume = _volume; options.volume = _volume;
if (_injector) { if (_injector) {
_injector->setOptions(options); DependencyManager::get<AudioInjectorManager>()->setOptionsAndRestart(_injector, options);
_injector->restart();
} else { } else {
_injector = AudioInjector::playSound(_sound, options); _injector = DependencyManager::get<AudioInjectorManager>()->playSound(_sound, options);
} }
} }

View file

@ -13,9 +13,7 @@
#include <QQuickItem> #include <QQuickItem>
#include <SoundCache.h> #include <SoundCache.h>
#include <AudioInjectorManager.h>
class AudioInjector;
using AudioInjectorPointer = QSharedPointer<AudioInjector>;
// SoundEffect object, exposed to qml only, not interface JavaScript. // SoundEffect object, exposed to qml only, not interface JavaScript.
// This is used to play spatial sound effects on tablets/web entities from within QML. // This is used to play spatial sound effects on tablets/web entities from within QML.
@ -34,7 +32,7 @@ public:
float getVolume() const; float getVolume() const;
void setVolume(float volume); void setVolume(float volume);
Q_INVOKABLE void play(QVariant position); Q_INVOKABLE void play(const QVariant& position);
protected: protected:
QUrl _url; QUrl _url;
float _volume { 1.0f }; float _volume { 1.0f };