throttle audio streams by count using PI controller

This commit is contained in:
Zach Pomerantz 2017-01-23 21:30:41 -05:00
parent 3c9c78ae14
commit 39acba5455
8 changed files with 125 additions and 97 deletions

View file

@ -47,9 +47,6 @@ static const QString AUDIO_THREADING_GROUP_KEY = "audio_threading";
int AudioMixer::_numStaticJitterFrames{ -1 };
float AudioMixer::_noiseMutingThreshold{ DEFAULT_NOISE_MUTING_THRESHOLD };
float AudioMixer::_attenuationPerDoublingInDistance{ DEFAULT_ATTENUATION_PER_DOUBLING_IN_DISTANCE };
float AudioMixer::_trailingSleepRatio{ 1.0f };
float AudioMixer::_performanceThrottlingRatio{ 0.0f };
float AudioMixer::_minAudibilityThreshold{ LOUDNESS_TO_DISTANCE_RATIO / 2.0f };
QHash<QString, AABox> AudioMixer::_audioZones;
QVector<AudioMixer::ZoneSettings> AudioMixer::_zoneSettings;
QVector<AudioMixer::ReverbSettings> AudioMixer::_zoneReverbSettings;
@ -297,8 +294,7 @@ void AudioMixer::sendStatsPacket() {
statsObject["threads"] = _slavePool.numThreads();
statsObject["trailing_sleep_percentage"] = _trailingSleepRatio * 100.0f;
statsObject["performance_throttling_ratio"] = _performanceThrottlingRatio;
statsObject["throttling_ratio"] = _throttlingRatio;
statsObject["avg_streams_per_frame"] = (float)_stats.sumStreams / (float)_numStatFrames;
statsObject["avg_listeners_per_frame"] = (float)_stats.sumListeners / (float)_numStatFrames;
@ -335,7 +331,7 @@ void AudioMixer::sendStatsPacket() {
mixStats["%_hrtf_mixes"] = percentageForMixStats(_stats.hrtfRenders);
mixStats["%_hrtf_silent_mixes"] = percentageForMixStats(_stats.hrtfSilentRenders);
mixStats["%_hrtf_struggle_mixes"] = percentageForMixStats(_stats.hrtfStruggleRenders);
mixStats["%_hrtf_throttle_mixes"] = percentageForMixStats(_stats.hrtfThrottleRenders);
mixStats["%_manual_stereo_mixes"] = percentageForMixStats(_stats.manualStereoMixes);
mixStats["%_manual_echo_mixes"] = percentageForMixStats(_stats.manualEchoMixes);
@ -411,17 +407,15 @@ void AudioMixer::start() {
parseSettingsObject(settingsObject);
}
// manageLoad state
auto frameTimestamp = p_high_resolution_clock::time_point::min();
unsigned int framesSinceManagement = std::numeric_limits<int>::max();
// mix state
unsigned int frame = 1;
auto frameTimestamp = p_high_resolution_clock::time_point::min();
while (!_isFinished) {
{
auto timer = _sleepTiming.timer();
manageLoad(frameTimestamp, framesSinceManagement);
auto frameDuration = timeFrame(frameTimestamp);
throttle(frameDuration);
}
auto timer = _frameTiming.timer();
@ -438,7 +432,7 @@ void AudioMixer::start() {
// mix across slave threads
{
auto timer = _mixTiming.timer();
_slavePool.mix(cbegin, cend, frame);
_slavePool.mix(cbegin, cend, frame, _throttlingRatio);
}
});
@ -467,68 +461,53 @@ void AudioMixer::start() {
}
}
void AudioMixer::manageLoad(p_high_resolution_clock::time_point& frameTimestamp, unsigned int& framesSinceCutoffEvent) {
auto timeToSleep = std::chrono::microseconds(0);
std::chrono::microseconds AudioMixer::timeFrame(p_high_resolution_clock::time_point& timestamp) {
// advance the next frame
auto nextTimestamp = timestamp + std::chrono::microseconds(AudioConstants::NETWORK_FRAME_USECS);
auto now = p_high_resolution_clock::now();
// sleep until the next frame, if necessary
{
// advance the next frame
frameTimestamp += std::chrono::microseconds(AudioConstants::NETWORK_FRAME_USECS);
auto now = p_high_resolution_clock::now();
// compute how long the last frame took
auto duration = std::chrono::duration_cast<std::chrono::microseconds>(now - timestamp);
// calculate sleep
if (frameTimestamp < now) {
frameTimestamp = now;
// sleep until the next frame should start
if (nextTimestamp > now) {
auto timeToSleep = std::chrono::duration_cast<std::chrono::microseconds>(nextTimestamp - now);
std::this_thread::sleep_for(timeToSleep);
}
// set the new frame timestamp
timestamp = p_high_resolution_clock::now();
return duration;
}
void AudioMixer::throttle(std::chrono::microseconds duration) {
// throttle using a basic PI controller, keeping duration under 9000 us
const int TARGET = 9000;
const float PROPORTIONAL_TERM = -0.0f;
const float INTEGRAL_TERM = -0.0f;
// error term is the fraction of a frame away from the target duration
float error = (TARGET - duration.count()) / 10000.0f;
if (_throttlingRatio == 0.0f && error > 0) {
// if we are not throttling nor struggling, reset the controller and continue
if (error > 0) {
_throttlingIntegral = 0.0f;
return;
} else {
timeToSleep = std::chrono::duration_cast<std::chrono::microseconds>(frameTimestamp - now);
std::this_thread::sleep_for(timeToSleep);
qDebug() << "audio-mixer is struggling - throttling";
}
}
// manage mixer load
{
const int TRAILING_AVERAGE_FRAMES = 100;
const float CURRENT_FRAME_RATIO = 1.0f / TRAILING_AVERAGE_FRAMES;
const float PREVIOUS_FRAMES_RATIO = 1.0f - CURRENT_FRAME_RATIO;
_throttlingIntegral += error;
_throttlingIntegral = INTEGRAL_TERM == 0.0f ? 1.0f : std::max(_throttlingIntegral, 1 / INTEGRAL_TERM);
_throttlingRatio = PROPORTIONAL_TERM * error + INTEGRAL_TERM * _throttlingIntegral;
const float STRUGGLE_TRIGGER_SLEEP_PERCENTAGE_THRESHOLD = 0.10f;
const float BACK_OFF_TRIGGER_SLEEP_PERCENTAGE_THRESHOLD = 0.20f;
_throttlingRatio = glm::clamp(_throttlingRatio, 0.0f, 1.0f);
const float RATIO_BACK_OFF = 0.02f;
_trailingSleepRatio = (PREVIOUS_FRAMES_RATIO * _trailingSleepRatio) +
// ratio of frame spent sleeping / total frame time
((CURRENT_FRAME_RATIO * timeToSleep.count()) / (float) AudioConstants::NETWORK_FRAME_USECS);
bool hasRatioChanged = false;
if (framesSinceCutoffEvent >= TRAILING_AVERAGE_FRAMES) {
if (_trailingSleepRatio <= STRUGGLE_TRIGGER_SLEEP_PERCENTAGE_THRESHOLD) {
qDebug() << "Mixer is struggling";
// change our min required loudness to reduce some load
_performanceThrottlingRatio = _performanceThrottlingRatio + (0.5f * (1.0f - _performanceThrottlingRatio));
hasRatioChanged = true;
} else if (_trailingSleepRatio >= BACK_OFF_TRIGGER_SLEEP_PERCENTAGE_THRESHOLD && _performanceThrottlingRatio != 0) {
qDebug() << "Mixer is recovering";
// back off the required loudness
_performanceThrottlingRatio = std::max(0.0f, _performanceThrottlingRatio - RATIO_BACK_OFF);
hasRatioChanged = true;
}
if (hasRatioChanged) {
// set out min audability threshold from the new ratio
_minAudibilityThreshold = LOUDNESS_TO_DISTANCE_RATIO / (2.0f * (1.0f - _performanceThrottlingRatio));
framesSinceCutoffEvent = 0;
qDebug() << "Sleeping" << _trailingSleepRatio << "of frame";
qDebug() << "Cutoff is" << _performanceThrottlingRatio;
qDebug() << "Minimum audibility to be mixed is" << _minAudibilityThreshold;
}
}
if (!hasRatioChanged) {
++framesSinceCutoffEvent;
}
if (_throttlingRatio == 0.0f) {
qDebug() << "audio-mixer is recovered - no longer throttling";
}
}

View file

@ -46,7 +46,6 @@ public:
static int getStaticJitterFrames() { return _numStaticJitterFrames; }
static bool shouldMute(float quietestFrame) { return quietestFrame > _noiseMutingThreshold; }
static float getAttenuationPerDoublingInDistance() { return _attenuationPerDoublingInDistance; }
static float getMinimumAudibilityThreshold() { return _performanceThrottlingRatio > 0.0f ? _minAudibilityThreshold : 0.0f; }
static const QHash<QString, AABox>& getAudioZones() { return _audioZones; }
static const QVector<ZoneSettings>& getZoneSettings() { return _zoneSettings; }
static const QVector<ReverbSettings>& getReverbSettings() { return _zoneReverbSettings; }
@ -73,8 +72,8 @@ private slots:
private:
// mixing helpers
// check and maybe throttle mixer load by changing audibility threshold
void manageLoad(p_high_resolution_clock::time_point& frameTimestamp, unsigned int& framesSinceManagement);
std::chrono::microseconds timeFrame(p_high_resolution_clock::time_point& timestamp);
void throttle(std::chrono::microseconds frameDuration);
// pop a frame from any streams on the node
// returns the number of available streams
int prepareFrame(const SharedNodePointer& node, unsigned int frame);
@ -85,6 +84,9 @@ private:
void parseSettingsObject(const QJsonObject& settingsObject);
float _throttlingIntegral { 0.0f };
float _throttlingRatio { 0.0f };
int _numStatFrames { 0 };
AudioMixerStats _stats;
@ -122,9 +124,6 @@ private:
static int _numStaticJitterFrames; // -1 denotes dynamic jitter buffering
static float _noiseMutingThreshold;
static float _attenuationPerDoublingInDistance;
static float _trailingSleepRatio;
static float _performanceThrottlingRatio;
static float _minAudibilityThreshold;
static QHash<QString, AABox> _audioZones;
static QVector<ZoneSettings> _zoneSettings;
static QVector<ReverbSettings> _zoneReverbSettings;

View file

@ -36,6 +36,8 @@
#include "AudioMixerSlave.h"
using AudioStreamMap = AudioMixerClientData::AudioStreamMap;
std::unique_ptr<NLPacket> createAudioPacket(PacketType type, int size, quint16 sequence, QString codec) {
auto audioPacket = NLPacket::create(type, size);
audioPacket->writePrimitive(sequence);
@ -134,10 +136,11 @@ void sendEnvironmentPacket(const SharedNodePointer& node, AudioMixerClientData&
}
}
void AudioMixerSlave::configure(ConstIter begin, ConstIter end, unsigned int frame) {
void AudioMixerSlave::configure(ConstIter begin, ConstIter end, unsigned int frame, float throttlingRatio) {
_begin = begin;
_end = end;
_frame = frame;
_throttlingRatio = throttlingRatio;
}
void AudioMixerSlave::mix(const SharedNodePointer& node) {
@ -196,6 +199,41 @@ void AudioMixerSlave::mix(const SharedNodePointer& node) {
}
}
AudioStreamMap splitQuietestStreams(const AvatarAudioStream& listener, AudioStreamMap& streams, int numToRetain) {
using AudioStreamKey = AudioStreamMap::key_type;
std::vector<std::pair<float, AudioStreamKey>> streamVolumes;
// satisfy the push_heap precondition that streamVolumes be non-empty
streamVolumes.push_back(std::make_pair(0.0f, QUuid()));
// put streams into a heap by their volume
for (const auto& streamPair : streams) {
const auto& streamKey = streamPair.first;
const auto& stream = streamPair.second;
// calculate the volume
float distance = glm::length(stream->getPosition() - listener.getPosition());
float volume = stream->getLastPopOutputTrailingLoudness() / distance;
streamVolumes.push_back(std::make_pair(volume, streamKey));
std::push_heap(streamVolumes.begin(), streamVolumes.end());
}
// pop the loudest numToRetain streams off the heap
AudioStreamMap poppedStreams;
for (int i = 0; i < numToRetain; i++) {
std::pop_heap(streamVolumes.begin(), streamVolumes.end());
AudioStreamKey streamKey = streamVolumes.back().second;
poppedStreams[streamKey] = streams[streamKey];
streams.erase(streamKey);
streamVolumes.pop_back();
}
streams.swap(poppedStreams);
return poppedStreams;
}
bool AudioMixerSlave::prepareMix(const SharedNodePointer& node) {
AvatarAudioStream* nodeAudioStream = static_cast<AudioMixerClientData*>(node->getLinkedData())->getAvatarAudioStream();
AudioMixerClientData* nodeData = static_cast<AudioMixerClientData*>(node->getLinkedData());
@ -250,16 +288,27 @@ bool AudioMixerSlave::prepareMix(const SharedNodePointer& node) {
}
}
// Enumerate the audio streams attached to the otherNode
auto streamsCopy = otherData->getAudioStreams();
for (auto& streamPair : streamsCopy) {
auto otherNodeStream = streamPair.second;
bool isSelfWithEcho = ((*otherNode == *node) && (otherNodeStream->shouldLoopbackForNode()));
// Add all audio streams that should be added to the mix
if (isSelfWithEcho || (!isSelfWithEcho && !insideIgnoreRadius)) {
addStreamToMix(*nodeData, otherNode->getUUID(), *nodeAudioStream, *otherNodeStream);
auto addStreams = [&](AudioStreamMap& streams, bool throttle) {
for (auto& streamPair : streams) {
auto otherNodeStream = streamPair.second;
bool isSelfWithEcho = ((*otherNode == *node) && (otherNodeStream->shouldLoopbackForNode()));
// Add all audio streams that should be added to the mix
if (isSelfWithEcho || (!isSelfWithEcho && !insideIgnoreRadius)) {
addStreamToMix(*nodeData, otherNode->getUUID(), *nodeAudioStream, *otherNodeStream, throttle);
}
}
};
auto streams = otherData->getAudioStreams();
// if throttling, we need to sort by loudness and omit the quietest streams
if (_throttlingRatio > 0.0f) {
int numStreams = (int)(streams.size() * (1 - _throttlingRatio));
auto quietStreams = splitQuietestStreams(*nodeAudioStream, streams, numStreams);
addStreams(quietStreams, true);
}
addStreams(streams, false);
}
});
@ -278,7 +327,8 @@ bool AudioMixerSlave::prepareMix(const SharedNodePointer& node) {
}
void AudioMixerSlave::addStreamToMix(AudioMixerClientData& listenerNodeData, const QUuid& sourceNodeID,
const AvatarAudioStream& listeningNodeStream, const PositionalAudioStream& streamToAdd) {
const AvatarAudioStream& listeningNodeStream, const PositionalAudioStream& streamToAdd,
bool throttle) {
// to reduce artifacts we calculate the gain and azimuth for every source for this listener
// even if we are not going to end up mixing in this source
@ -391,16 +441,12 @@ void AudioMixerSlave::addStreamToMix(AudioMixerClientData& listenerNodeData, con
return;
}
float audibilityThreshold = AudioMixer::getMinimumAudibilityThreshold();
if (audibilityThreshold > 0.0f &&
streamToAdd.getLastPopOutputTrailingLoudness() / glm::length(relativePosition) <= audibilityThreshold) {
// the mixer is struggling so we're going to drop off some streams
if (throttle) {
// we call renderSilent via the HRTF with the actual frame data and a gain of 0.0
hrtf.renderSilent(_bufferSamples, _mixSamples, HRTF_DATASET_INDEX, azimuth, distance, 0.0f,
AudioConstants::NETWORK_FRAME_SAMPLES_PER_CHANNEL);
++stats.hrtfStruggleRenders;
++stats.hrtfThrottleRenders;
return;
}

View file

@ -30,7 +30,7 @@ class AudioMixerSlave {
public:
using ConstIter = NodeList::const_iterator;
void configure(ConstIter begin, ConstIter end, unsigned int frame);
void configure(ConstIter begin, ConstIter end, unsigned int frame, float throttlingRatio);
// mix and broadcast non-ignored streams to the node
// returns true if a mixed packet was sent to the node
@ -43,7 +43,8 @@ private:
bool prepareMix(const SharedNodePointer& node);
// add a stream to the mix
void addStreamToMix(AudioMixerClientData& listenerData, const QUuid& streamerID,
const AvatarAudioStream& listenerStream, const PositionalAudioStream& streamer);
const AvatarAudioStream& listenerStream, const PositionalAudioStream& streamer,
bool throttle);
float gainForSource(const AvatarAudioStream& listener, const PositionalAudioStream& streamer,
const glm::vec3& relativePosition, bool isEcho);
@ -58,6 +59,7 @@ private:
ConstIter _begin;
ConstIter _end;
unsigned int _frame { 0 };
float _throttlingRatio { 0.0f };
};
#endif // hifi_AudioMixerSlave_h

View file

@ -41,7 +41,7 @@ void AudioMixerSlaveThread::wait() {
});
++_pool._numStarted;
}
configure(_pool._begin, _pool._end, _pool._frame);
configure(_pool._begin, _pool._end, _pool._frame, _pool._throttlingRatio);
}
void AudioMixerSlaveThread::notify(bool stopping) {
@ -64,13 +64,14 @@ bool AudioMixerSlaveThread::try_pop(SharedNodePointer& node) {
static AudioMixerSlave slave;
#endif
void AudioMixerSlavePool::mix(ConstIter begin, ConstIter end, unsigned int frame) {
void AudioMixerSlavePool::mix(ConstIter begin, ConstIter end, unsigned int frame, float throttlingRatio) {
_begin = begin;
_end = end;
_frame = frame;
_throttlingRatio = throttlingRatio;
#ifdef AUDIO_SINGLE_THREADED
slave.configure(_begin, _end, frame);
slave.configure(_begin, _end, frame, throttlingRatio);
std::for_each(begin, end, [&](const SharedNodePointer& node) {
slave.mix(node);
});

View file

@ -61,7 +61,7 @@ public:
~AudioMixerSlavePool() { resize(0); }
// mix on slave threads
void mix(ConstIter begin, ConstIter end, unsigned int frame);
void mix(ConstIter begin, ConstIter end, unsigned int frame, float throttlingRatio);
// iterate over all slaves
void each(std::function<void(AudioMixerSlave& slave)> functor);
@ -90,6 +90,7 @@ private:
// frame state
Queue _queue;
unsigned int _frame { 0 };
float _throttlingRatio { 0.0f };
ConstIter _begin;
ConstIter _end;
};

View file

@ -17,7 +17,7 @@ void AudioMixerStats::reset() {
totalMixes = 0;
hrtfRenders = 0;
hrtfSilentRenders = 0;
hrtfStruggleRenders = 0;
hrtfThrottleRenders = 0;
manualStereoMixes = 0;
manualEchoMixes = 0;
}
@ -28,7 +28,7 @@ void AudioMixerStats::accumulate(const AudioMixerStats& otherStats) {
totalMixes += otherStats.totalMixes;
hrtfRenders += otherStats.hrtfRenders;
hrtfSilentRenders += otherStats.hrtfSilentRenders;
hrtfStruggleRenders += otherStats.hrtfStruggleRenders;
hrtfThrottleRenders += otherStats.hrtfThrottleRenders;
manualStereoMixes += otherStats.manualStereoMixes;
manualEchoMixes += otherStats.manualEchoMixes;
}

View file

@ -20,7 +20,7 @@ struct AudioMixerStats {
int hrtfRenders { 0 };
int hrtfSilentRenders { 0 };
int hrtfStruggleRenders { 0 };
int hrtfThrottleRenders { 0 };
int manualStereoMixes { 0 };
int manualEchoMixes { 0 };