From 3e16dabb75c2796b900ac983469c85d082d1272f Mon Sep 17 00:00:00 2001 From: Zach Pomerantz Date: Tue, 25 Oct 2016 17:42:57 -0700 Subject: [PATCH 1/6] fix audio mixer throttling --- assignment-client/src/audio/AudioMixer.cpp | 45 +++++++++++----------- 1 file changed, 23 insertions(+), 22 deletions(-) diff --git a/assignment-client/src/audio/AudioMixer.cpp b/assignment-client/src/audio/AudioMixer.cpp index 0252c037bf..0f3318bf24 100644 --- a/assignment-client/src/audio/AudioMixer.cpp +++ b/assignment-client/src/audio/AudioMixer.cpp @@ -730,30 +730,25 @@ void AudioMixer::domainSettingsRequestComplete() { } void AudioMixer::broadcastMixes() { + const int TRAILING_AVERAGE_FRAMES = 100; + const float CURRENT_FRAME_RATIO = 1.0f / TRAILING_AVERAGE_FRAMES; + const float PREVIOUS_FRAMES_RATIO = 1.0f - CURRENT_FRAME_RATIO; + + const float STRUGGLE_TRIGGER_SLEEP_PERCENTAGE_THRESHOLD = 0.10f; + const float BACK_OFF_TRIGGER_SLEEP_PERCENTAGE_THRESHOLD = 0.20f; + + const float RATIO_BACK_OFF = 0.02f; + auto nodeList = DependencyManager::get(); auto nextFrameTimestamp = p_high_resolution_clock::now(); auto timeToSleep = std::chrono::microseconds(0); - const int TRAILING_AVERAGE_FRAMES = 100; + int currentFrame = 1; + int numFramesPerSecond = (int) ceil(AudioConstants::NETWORK_FRAMES_PER_SEC); int framesSinceCutoffEvent = TRAILING_AVERAGE_FRAMES; - int currentFrame { 1 }; - int numFramesPerSecond { (int) ceil(AudioConstants::NETWORK_FRAMES_PER_SEC) }; - while (!_isFinished) { - const float STRUGGLE_TRIGGER_SLEEP_PERCENTAGE_THRESHOLD = 0.10f; - const float BACK_OFF_TRIGGER_SLEEP_PERCENTAGE_THRESHOLD = 0.20f; - - const float RATIO_BACK_OFF = 0.02f; - - const float CURRENT_FRAME_RATIO = 1.0f / TRAILING_AVERAGE_FRAMES; - const float PREVIOUS_FRAMES_RATIO = 1.0f - CURRENT_FRAME_RATIO; - - if (timeToSleep.count() < 0) { - timeToSleep = std::chrono::microseconds(0); - } - _trailingSleepRatio = (PREVIOUS_FRAMES_RATIO * _trailingSleepRatio) + (timeToSleep.count() * CURRENT_FRAME_RATIO / (float) AudioConstants::NETWORK_FRAME_USECS); @@ -886,14 +881,20 @@ void AudioMixer::broadcastMixes() { break; } - // push the next frame timestamp to when we should send the next - nextFrameTimestamp += std::chrono::microseconds(AudioConstants::NETWORK_FRAME_USECS); + // sleep until the next frame, if necessary + { + nextFrameTimestamp += std::chrono::microseconds(AudioConstants::NETWORK_FRAME_USECS); - // sleep as long as we need until next frame, if we can - auto now = p_high_resolution_clock::now(); - timeToSleep = std::chrono::duration_cast(nextFrameTimestamp - now); + auto now = p_high_resolution_clock::now(); + timeToSleep = std::chrono::duration_cast(nextFrameTimestamp - now); - std::this_thread::sleep_for(timeToSleep); + if (timeToSleep.count() < 0) { + nextFrameTimestamp = now; + timeToSleep = std::chrono::microseconds(0); + } + + std::this_thread::sleep_for(timeToSleep); + } } } From 04e5d8e302af110ff3b2c558ce8c729ac98704c2 Mon Sep 17 00:00:00 2001 From: Zach Pomerantz Date: Wed, 26 Oct 2016 12:18:15 -0700 Subject: [PATCH 2/6] initialize audio client data --- assignment-client/src/audio/AudioMixer.cpp | 34 ++++++++++------------ assignment-client/src/audio/AudioMixer.h | 1 + 2 files changed, 17 insertions(+), 18 deletions(-) diff --git a/assignment-client/src/audio/AudioMixer.cpp b/assignment-client/src/audio/AudioMixer.cpp index 0f3318bf24..771ad77aa6 100644 --- a/assignment-client/src/audio/AudioMixer.cpp +++ b/assignment-client/src/audio/AudioMixer.cpp @@ -90,8 +90,8 @@ AudioMixer::AudioMixer(ReceivedMessage& message) : PacketType::InjectAudio, PacketType::SilentAudioFrame, PacketType::AudioStreamStats }, this, "handleNodeAudioPacket"); - packetReceiver.registerListener(PacketType::MuteEnvironment, this, "handleMuteEnvironmentPacket"); packetReceiver.registerListener(PacketType::NegotiateAudioFormat, this, "handleNegotiateAudioFormat"); + packetReceiver.registerListener(PacketType::MuteEnvironment, this, "handleMuteEnvironmentPacket"); packetReceiver.registerListener(PacketType::NodeIgnoreRequest, this, "handleNodeIgnoreRequestPacket"); connect(nodeList.data(), &NodeList::nodeKilled, this, &AudioMixer::handleNodeKilled); @@ -481,6 +481,7 @@ void AudioMixer::sendAudioEnvironmentPacket(SharedNodePointer node) { } void AudioMixer::handleNodeAudioPacket(QSharedPointer message, SharedNodePointer sendingNode) { + getOrCreateClientData(sendingNode.data()); DependencyManager::get()->updateNodeWithDataFromPacket(message, sendingNode); } @@ -579,18 +580,8 @@ void AudioMixer::handleNegotiateAudioFormat(QSharedPointer mess } } - auto clientData = dynamic_cast(sendingNode->getLinkedData()); - - // FIXME - why would we not have client data at this point?? - if (!clientData) { - qDebug() << "UNEXPECTED -- didn't have node linked data in " << __FUNCTION__; - sendingNode->setLinkedData(std::unique_ptr { new AudioMixerClientData(sendingNode->getUUID()) }); - clientData = dynamic_cast(sendingNode->getLinkedData()); - connect(clientData, &AudioMixerClientData::injectorStreamFinished, this, &AudioMixer::removeHRTFsForFinishedInjector); - } - + auto clientData = getOrCreateClientData(sendingNode.data()); clientData->setupCodec(selectedCodec, selectedCodecName); - qDebug() << "selectedCodecName:" << selectedCodecName; clientData->sendSelectAudioFormat(sendingNode, selectedCodecName); } @@ -707,17 +698,24 @@ void AudioMixer::run() { ThreadedAssignment::commonInit(AUDIO_MIXER_LOGGING_TARGET_NAME, NodeType::AudioMixer); } +AudioMixerClientData* AudioMixer::getOrCreateClientData(Node* node) { + auto clientData = dynamic_cast(node->getLinkedData()); + + if (!clientData) { + node->setLinkedData(std::unique_ptr { new AudioMixerClientData(node->getUUID()) }); + clientData = dynamic_cast(node->getLinkedData()); + connect(clientData, &AudioMixerClientData::injectorStreamFinished, this, &AudioMixer::removeHRTFsForFinishedInjector); + } + + return clientData; +} + void AudioMixer::domainSettingsRequestComplete() { auto nodeList = DependencyManager::get(); nodeList->addNodeTypeToInterestSet(NodeType::Agent); - nodeList->linkedDataCreateCallback = [&](Node* node) { - node->setLinkedData(std::unique_ptr { new AudioMixerClientData(node->getUUID()) }); - auto clientData = dynamic_cast(node->getLinkedData()); - - connect(clientData, &AudioMixerClientData::injectorStreamFinished, this, &AudioMixer::removeHRTFsForFinishedInjector); - }; + nodeList->linkedDataCreateCallback = [&](Node* node) { getOrCreateClientData(node); }; DomainHandler& domainHandler = nodeList->getDomainHandler(); const QJsonObject& settingsObject = domainHandler.getSettingsObject(); diff --git a/assignment-client/src/audio/AudioMixer.h b/assignment-client/src/audio/AudioMixer.h index bccac529c1..9caa39736b 100644 --- a/assignment-client/src/audio/AudioMixer.h +++ b/assignment-client/src/audio/AudioMixer.h @@ -52,6 +52,7 @@ private slots: void removeHRTFsForFinishedInjector(const QUuid& streamID); private: + AudioMixerClientData* getOrCreateClientData(Node* node); void domainSettingsRequestComplete(); /// adds one stream to the mix for a listening node From 9e9f8666cec5d5e8ad8131d78d7abd93923e8a9a Mon Sep 17 00:00:00 2001 From: Zach Pomerantz Date: Thu, 27 Oct 2016 13:36:14 -0700 Subject: [PATCH 3/6] clean up handleMismatchAudioFormat comment --- assignment-client/src/audio/AudioMixerClientData.cpp | 5 ++++- 1 file changed, 4 insertions(+), 1 deletion(-) diff --git a/assignment-client/src/audio/AudioMixerClientData.cpp b/assignment-client/src/audio/AudioMixerClientData.cpp index 1eb36cd8a7..d4866d2fa6 100644 --- a/assignment-client/src/audio/AudioMixerClientData.cpp +++ b/assignment-client/src/audio/AudioMixerClientData.cpp @@ -355,7 +355,10 @@ QJsonObject AudioMixerClientData::getAudioStreamStats() { } void AudioMixerClientData::handleMismatchAudioFormat(SharedNodePointer node, const QString& currentCodec, const QString& recievedCodec) { - qDebug() << __FUNCTION__ << "sendingNode:" << *node << "currentCodec:" << currentCodec << "recievedCodec:" << recievedCodec; + qDebug() << __FUNCTION__ << + "sendingNode:" << *node << + "currentCodec:" << currentCodec << + "receivedCodec:" << recievedCodec; sendSelectAudioFormat(node, currentCodec); } From 20bac5aeef00a7c8aab71626c59f5e1274087daa Mon Sep 17 00:00:00 2001 From: Zach Pomerantz Date: Thu, 27 Oct 2016 13:38:16 -0700 Subject: [PATCH 4/6] clean up broadcastMixes --- assignment-client/src/audio/AudioMixer.cpp | 78 +++++++++++----------- 1 file changed, 39 insertions(+), 39 deletions(-) diff --git a/assignment-client/src/audio/AudioMixer.cpp b/assignment-client/src/audio/AudioMixer.cpp index 771ad77aa6..b4db08ada0 100644 --- a/assignment-client/src/audio/AudioMixer.cpp +++ b/assignment-client/src/audio/AudioMixer.cpp @@ -747,48 +747,46 @@ void AudioMixer::broadcastMixes() { int framesSinceCutoffEvent = TRAILING_AVERAGE_FRAMES; while (!_isFinished) { - _trailingSleepRatio = (PREVIOUS_FRAMES_RATIO * _trailingSleepRatio) - + (timeToSleep.count() * CURRENT_FRAME_RATIO / (float) AudioConstants::NETWORK_FRAME_USECS); + // manage mixer load + { + _trailingSleepRatio = (PREVIOUS_FRAMES_RATIO * _trailingSleepRatio) + + // ratio of frame spent sleeping / total frame time + ((CURRENT_FRAME_RATIO * timeToSleep.count()) / (float) AudioConstants::NETWORK_FRAME_USECS); - float lastCutoffRatio = _performanceThrottlingRatio; - bool hasRatioChanged = false; + float lastCutoffRatio = _performanceThrottlingRatio; + bool hasRatioChanged = false; - if (framesSinceCutoffEvent >= TRAILING_AVERAGE_FRAMES) { - if (_trailingSleepRatio <= STRUGGLE_TRIGGER_SLEEP_PERCENTAGE_THRESHOLD) { - // we're struggling - change our min required loudness to reduce some load - _performanceThrottlingRatio = _performanceThrottlingRatio + (0.5f * (1.0f - _performanceThrottlingRatio)); - - qDebug() << "Mixer is struggling, sleeping" << _trailingSleepRatio * 100 << "% of frame time. Old cutoff was" - << lastCutoffRatio << "and is now" << _performanceThrottlingRatio; - hasRatioChanged = true; - } else if (_trailingSleepRatio >= BACK_OFF_TRIGGER_SLEEP_PERCENTAGE_THRESHOLD && _performanceThrottlingRatio != 0) { - // we've recovered and can back off the required loudness - _performanceThrottlingRatio = _performanceThrottlingRatio - RATIO_BACK_OFF; - - if (_performanceThrottlingRatio < 0) { - _performanceThrottlingRatio = 0; + if (framesSinceCutoffEvent >= TRAILING_AVERAGE_FRAMES) { + if (_trailingSleepRatio <= STRUGGLE_TRIGGER_SLEEP_PERCENTAGE_THRESHOLD) { + qDebug() << "Mixer is struggling"; + // change our min required loudness to reduce some load + _performanceThrottlingRatio = _performanceThrottlingRatio + (0.5f * (1.0f - _performanceThrottlingRatio)); + hasRatioChanged = true; + } else if (_trailingSleepRatio >= BACK_OFF_TRIGGER_SLEEP_PERCENTAGE_THRESHOLD && _performanceThrottlingRatio != 0) { + qDebug() << "Mixer is recovering"; + // back off the required loudness + _performanceThrottlingRatio = std::max(0.0f, _performanceThrottlingRatio - RATIO_BACK_OFF); + hasRatioChanged = true; } - qDebug() << "Mixer is recovering, sleeping" << _trailingSleepRatio * 100 << "% of frame time. Old cutoff was" - << lastCutoffRatio << "and is now" << _performanceThrottlingRatio; - hasRatioChanged = true; + if (hasRatioChanged) { + // set out min audability threshold from the new ratio + _minAudibilityThreshold = LOUDNESS_TO_DISTANCE_RATIO / (2.0f * (1.0f - _performanceThrottlingRatio)); + framesSinceCutoffEvent = 0; + + qDebug() << "Sleeping" << _trailingSleepRatio << "of frame"; + qDebug() << "Cutoff is" << _performanceThrottlingRatio; + qDebug() << "Minimum audibility to be mixed is" << _minAudibilityThreshold; + } } - if (hasRatioChanged) { - // set out min audability threshold from the new ratio - _minAudibilityThreshold = LOUDNESS_TO_DISTANCE_RATIO / (2.0f * (1.0f - _performanceThrottlingRatio)); - qDebug() << "Minimum audability required to be mixed is now" << _minAudibilityThreshold; - - framesSinceCutoffEvent = 0; + if (!hasRatioChanged) { + ++framesSinceCutoffEvent; } } - if (!hasRatioChanged) { - ++framesSinceCutoffEvent; - } - + // mix nodeList->eachNode([&](const SharedNodePointer& node) { - if (node->getLinkedData()) { AudioMixerClientData* nodeData = (AudioMixerClientData*)node->getLinkedData(); @@ -869,14 +867,16 @@ void AudioMixer::broadcastMixes() { ++_numStatFrames; - // since we're a while loop we need to help Qt's event processing - QCoreApplication::processEvents(); + // play nice with qt event-looping + { + // since we're a while loop we need to help qt's event processing + QCoreApplication::processEvents(); - if (_isFinished) { - // at this point the audio-mixer is done - // check if we have a deferred delete event to process (which we should once finished) - QCoreApplication::sendPostedEvents(this, QEvent::DeferredDelete); - break; + if (_isFinished) { + // alert qt that this is finished + QCoreApplication::sendPostedEvents(this, QEvent::DeferredDelete); + break; + } } // sleep until the next frame, if necessary From a7d54b8e0ada88e481df912c0a92a617f8581816 Mon Sep 17 00:00:00 2001 From: Zach Pomerantz Date: Thu, 27 Oct 2016 14:38:17 -0700 Subject: [PATCH 5/6] rm unused lastCutoffRatio --- assignment-client/src/audio/AudioMixer.cpp | 1 - 1 file changed, 1 deletion(-) diff --git a/assignment-client/src/audio/AudioMixer.cpp b/assignment-client/src/audio/AudioMixer.cpp index b4db08ada0..8463c282a0 100644 --- a/assignment-client/src/audio/AudioMixer.cpp +++ b/assignment-client/src/audio/AudioMixer.cpp @@ -753,7 +753,6 @@ void AudioMixer::broadcastMixes() { // ratio of frame spent sleeping / total frame time ((CURRENT_FRAME_RATIO * timeToSleep.count()) / (float) AudioConstants::NETWORK_FRAME_USECS); - float lastCutoffRatio = _performanceThrottlingRatio; bool hasRatioChanged = false; if (framesSinceCutoffEvent >= TRAILING_AVERAGE_FRAMES) { From 428bb8e518774d65a884daf5cc267c5d5b7631d0 Mon Sep 17 00:00:00 2001 From: Zach Pomerantz Date: Thu, 27 Oct 2016 14:40:51 -0700 Subject: [PATCH 6/6] rm unused OffscreenQmlSurface const --- libraries/gl/src/gl/OffscreenQmlSurface.cpp | 1 - 1 file changed, 1 deletion(-) diff --git a/libraries/gl/src/gl/OffscreenQmlSurface.cpp b/libraries/gl/src/gl/OffscreenQmlSurface.cpp index c82c2b4a32..06f755c1dd 100644 --- a/libraries/gl/src/gl/OffscreenQmlSurface.cpp +++ b/libraries/gl/src/gl/OffscreenQmlSurface.cpp @@ -344,7 +344,6 @@ bool OffscreenQmlSurface::allowNewFrame(uint8_t fps) { OffscreenQmlSurface::OffscreenQmlSurface() { } -static const uint64_t MAX_SHUTDOWN_WAIT_SECS = 2; OffscreenQmlSurface::~OffscreenQmlSurface() { QObject::disconnect(&_updateTimer); QObject::disconnect(qApp);