Mirrors/overte-lubosz
3
0
Fork 0
mirror of https://github.com/lubosz/overte.git synced 2025-04-23 09:33:45 +02:00
Code Issues Projects Releases Packages Wiki Activity Actions

Merge pull request #9502 from zzmp/audio/throttle

Browse source 
Throttle audio with PI controller, by node
This commit is contained in:
Chris Collins 2017-01-26 14:25:44 -08:00 committed by GitHub
commit c0ee091946
8 changed files with 428 additions and 372 deletions
Show stats Download patch file Download diff file Expand all files Collapse all files

163
assignment-client/src/audio/AudioMixer.cpp
View file

@ -36,7 +36,6 @@
#include "AudioMixer.h"
static const float LOUDNESS_TO_DISTANCE_RATIO = 0.00001f;
static const float DEFAULT_ATTENUATION_PER_DOUBLING_IN_DISTANCE = 0.5f; // attenuation = -6dB * log2(distance)
static const float DEFAULT_NOISE_MUTING_THRESHOLD = 1.0f;
static const QString AUDIO_MIXER_LOGGING_TARGET_NAME = "audio-mixer";
@ -47,9 +46,6 @@ static const QString AUDIO_THREADING_GROUP_KEY = "audio_threading";
int AudioMixer::_numStaticJitterFrames{ -1 };
float AudioMixer::_noiseMutingThreshold{ DEFAULT_NOISE_MUTING_THRESHOLD };
float AudioMixer::_attenuationPerDoublingInDistance{ DEFAULT_ATTENUATION_PER_DOUBLING_IN_DISTANCE };
float AudioMixer::_trailingSleepRatio{ 1.0f };
float AudioMixer::_performanceThrottlingRatio{ 0.0f };
float AudioMixer::_minAudibilityThreshold{ LOUDNESS_TO_DISTANCE_RATIO / 2.0f };
QHash<QString, AABox> AudioMixer::_audioZones;
QVector<AudioMixer::ZoneSettings> AudioMixer::_zoneSettings;
QVector<AudioMixer::ReverbSettings> AudioMixer::_zoneReverbSettings;
@ -294,35 +290,31 @@ void AudioMixer::sendStatsPacket() {
// general stats
statsObject["useDynamicJitterBuffers"] = _numStaticJitterFrames == -1;
statsObject["trailing_sleep_percentage"] = _trailingSleepRatio * 100.0f;
statsObject["performance_throttling_ratio"] = _performanceThrottlingRatio;
statsObject["threads"] = _slavePool.numThreads();
statsObject["trailing_mix_ratio"] = _trailingMixRatio;
statsObject["throttling_ratio"] = _throttlingRatio;
statsObject["avg_streams_per_frame"] = (float)_stats.sumStreams / (float)_numStatFrames;
statsObject["avg_listeners_per_frame"] = (float)_stats.sumListeners / (float)_numStatFrames;
// timing stats
QJsonObject timingStats;
uint64_t timing, trailing;
_sleepTiming.get(timing, trailing);
timingStats["us_per_sleep"] = (qint64)(timing / _numStatFrames);
timingStats["us_per_sleep_trailing"] = (qint64)(trailing / _numStatFrames);
auto addTiming = [&](Timer& timer, std::string name) {
uint64_t timing, trailing;
timer.get(timing, trailing);
timingStats[("us_per_" + name).c_str()] = (qint64)(timing / _numStatFrames);
timingStats[("us_per_" + name + "_trailing").c_str()] = (qint64)(trailing / _numStatFrames);
};
_frameTiming.get(timing, trailing);
timingStats["us_per_frame"] = (qint64)(timing / _numStatFrames);
timingStats["us_per_frame_trailing"] = (qint64)(trailing / _numStatFrames);
_prepareTiming.get(timing, trailing);
timingStats["us_per_prepare"] = (qint64)(timing / _numStatFrames);
timingStats["us_per_prepare_trailing"] = (qint64)(trailing / _numStatFrames);
_mixTiming.get(timing, trailing);
timingStats["us_per_mix"] = (qint64)(timing / _numStatFrames);
timingStats["us_per_mix_trailing"] = (qint64)(trailing / _numStatFrames);
_eventsTiming.get(timing, trailing);
timingStats["us_per_events"] = (qint64)(timing / _numStatFrames);
timingStats["us_per_events_trailing"] = (qint64)(trailing / _numStatFrames);
addTiming(_ticTiming, "tic");
addTiming(_sleepTiming, "sleep");
addTiming(_frameTiming, "frame");
addTiming(_prepareTiming, "prepare");
addTiming(_mixTiming, "mix");
addTiming(_eventsTiming, "events");
// call it "avg_..." to keep it higher in the display, sorted alphabetically
statsObject["avg_timing_stats"] = timingStats;
@ -332,7 +324,7 @@ void AudioMixer::sendStatsPacket() {
mixStats["%_hrtf_mixes"] = percentageForMixStats(_stats.hrtfRenders);
mixStats["%_hrtf_silent_mixes"] = percentageForMixStats(_stats.hrtfSilentRenders);
mixStats["%_hrtf_struggle_mixes"] = percentageForMixStats(_stats.hrtfStruggleRenders);
mixStats["%_hrtf_throttle_mixes"] = percentageForMixStats(_stats.hrtfThrottleRenders);
mixStats["%_manual_stereo_mixes"] = percentageForMixStats(_stats.manualStereoMixes);
mixStats["%_manual_echo_mixes"] = percentageForMixStats(_stats.manualEchoMixes);
@ -408,25 +400,25 @@ void AudioMixer::start() {
parseSettingsObject(settingsObject);
}
// manageLoad state
auto frameTimestamp = p_high_resolution_clock::time_point::min();
unsigned int framesSinceManagement = std::numeric_limits<int>::max();
// mix state
unsigned int frame = 1;
auto frameTimestamp = p_high_resolution_clock::now();
while (!_isFinished) {
auto ticTimer = _ticTiming.timer();
{
auto timer = _sleepTiming.timer();
manageLoad(frameTimestamp, framesSinceManagement);
auto frameDuration = timeFrame(frameTimestamp);
throttle(frameDuration, frame);
}
auto timer = _frameTiming.timer();
auto frameTimer = _frameTiming.timer();
nodeList->nestedEach([&](NodeList::const_iterator cbegin, NodeList::const_iterator cend) {
// prepare frames; pop off any new audio from their streams
{
auto timer = _prepareTiming.timer();
auto prepareTimer = _prepareTiming.timer();
std::for_each(cbegin, cend, [&](const SharedNodePointer& node) {
_stats.sumStreams += prepareFrame(node, frame);
});
@ -434,8 +426,8 @@ void AudioMixer::start() {
// mix across slave threads
{
auto timer = _mixTiming.timer();
_slavePool.mix(cbegin, cend, frame);
auto mixTimer = _mixTiming.timer();
_slavePool.mix(cbegin, cend, frame, _throttlingRatio);
}
});
@ -450,7 +442,7 @@ void AudioMixer::start() {
// play nice with qt event-looping
{
auto timer = _eventsTiming.timer();
auto eventsTimer = _eventsTiming.timer();
// since we're a while loop we need to yield to qt's event processing
QCoreApplication::processEvents();
@ -464,67 +456,66 @@ void AudioMixer::start() {
}
}
void AudioMixer::manageLoad(p_high_resolution_clock::time_point& frameTimestamp, unsigned int& framesSinceCutoffEvent) {
auto timeToSleep = std::chrono::microseconds(0);
std::chrono::microseconds AudioMixer::timeFrame(p_high_resolution_clock::time_point& timestamp) {
// advance the next frame
auto nextTimestamp = timestamp + std::chrono::microseconds(AudioConstants::NETWORK_FRAME_USECS);
auto now = p_high_resolution_clock::now();
// sleep until the next frame, if necessary
{
// advance the next frame
frameTimestamp += std::chrono::microseconds(AudioConstants::NETWORK_FRAME_USECS);
auto now = p_high_resolution_clock::now();
// compute how long the last frame took
auto duration = std::chrono::duration_cast<std::chrono::microseconds>(now - timestamp);
// calculate sleep
if (frameTimestamp < now) {
frameTimestamp = now;
} else {
timeToSleep = std::chrono::duration_cast<std::chrono::microseconds>(frameTimestamp - now);
std::this_thread::sleep_for(timeToSleep);
}
}
// set the new frame timestamp
timestamp = std::max(now, nextTimestamp);
// manage mixer load
{
const int TRAILING_AVERAGE_FRAMES = 100;
const float CURRENT_FRAME_RATIO = 1.0f / TRAILING_AVERAGE_FRAMES;
const float PREVIOUS_FRAMES_RATIO = 1.0f - CURRENT_FRAME_RATIO;
// sleep until the next frame should start
std::this_thread::sleep_until(timestamp);
const float STRUGGLE_TRIGGER_SLEEP_PERCENTAGE_THRESHOLD = 0.10f;
const float BACK_OFF_TRIGGER_SLEEP_PERCENTAGE_THRESHOLD = 0.20f;
return duration;
}
const float RATIO_BACK_OFF = 0.02f;
void AudioMixer::throttle(std::chrono::microseconds duration, int frame) {
// throttle using a modified proportional-integral controller
const float FRAME_TIME = 10000.0f;
float mixRatio = duration.count() / FRAME_TIME;
_trailingSleepRatio = (PREVIOUS_FRAMES_RATIO * _trailingSleepRatio) +
// ratio of frame spent sleeping / total frame time
((CURRENT_FRAME_RATIO * timeToSleep.count()) / (float) AudioConstants::NETWORK_FRAME_USECS);
// constants are determined based on a "regular" 16-CPU EC2 server
bool hasRatioChanged = false;
// target different mix and backoff ratios (they also have different backoff rates)
// this is to prevent oscillation, and encourage throttling to find a steady state
const float TARGET = 0.9f;
// on a "regular" machine with 100 avatars, this is the largest value where
// - overthrottling can be recovered
// - oscillations will not occur after the recovery
const float BACKOFF_TARGET = 0.44f;
if (framesSinceCutoffEvent >= TRAILING_AVERAGE_FRAMES) {
if (_trailingSleepRatio <= STRUGGLE_TRIGGER_SLEEP_PERCENTAGE_THRESHOLD) {
qDebug() << "Mixer is struggling";
// change our min required loudness to reduce some load
_performanceThrottlingRatio = _performanceThrottlingRatio + (0.5f * (1.0f - _performanceThrottlingRatio));
hasRatioChanged = true;
} else if (_trailingSleepRatio >= BACK_OFF_TRIGGER_SLEEP_PERCENTAGE_THRESHOLD && _performanceThrottlingRatio != 0) {
qDebug() << "Mixer is recovering";
// back off the required loudness
_performanceThrottlingRatio = std::max(0.0f, _performanceThrottlingRatio - RATIO_BACK_OFF);
hasRatioChanged = true;
}
// the mixer is known to struggle at about 80 on a "regular" machine
// so throttle 2/80 the streams to ensure smooth audio (throttling is linear)
const float THROTTLE_RATE = 2 / 80.0f;
const float BACKOFF_RATE = THROTTLE_RATE / 4;
if (hasRatioChanged) {
// set out min audability threshold from the new ratio
_minAudibilityThreshold = LOUDNESS_TO_DISTANCE_RATIO / (2.0f * (1.0f - _performanceThrottlingRatio));
framesSinceCutoffEvent = 0;
// recovery should be bounded so that large changes in user count is a tolerable experience
// throttling is linear, so most cases will not need a full recovery
const int RECOVERY_TIME = 180;
qDebug() << "Sleeping" << _trailingSleepRatio << "of frame";
qDebug() << "Cutoff is" << _performanceThrottlingRatio;
qDebug() << "Minimum audibility to be mixed is" << _minAudibilityThreshold;
}
}
// weight more recent frames to determine if throttling is necessary,
const int TRAILING_FRAMES = (int)(100 * RECOVERY_TIME * BACKOFF_RATE);
const float CURRENT_FRAME_RATIO = 1.0f / TRAILING_FRAMES;
const float PREVIOUS_FRAMES_RATIO = 1.0f - CURRENT_FRAME_RATIO;
_trailingMixRatio = PREVIOUS_FRAMES_RATIO * _trailingMixRatio + CURRENT_FRAME_RATIO * mixRatio;
if (!hasRatioChanged) {
++framesSinceCutoffEvent;
if (frame % TRAILING_FRAMES == 0) {
if (_trailingMixRatio > TARGET) {
int proportionalTerm = 1 + (_trailingMixRatio - TARGET) / 0.1f;
_throttlingRatio += THROTTLE_RATE * proportionalTerm;
_throttlingRatio = std::min(_throttlingRatio, 1.0f);
qDebug("audio-mixer is struggling (%f mix/sleep) - throttling %f of streams",
(double)_trailingMixRatio, (double)_throttlingRatio);
} else if (_throttlingRatio > 0.0f && _trailingMixRatio <= BACKOFF_TARGET) {
int proportionalTerm = 1 + (TARGET - _trailingMixRatio) / 0.2f;
_throttlingRatio -= BACKOFF_RATE * proportionalTerm;
_throttlingRatio = std::max(_throttlingRatio, 0.0f);
qDebug("audio-mixer is recovering (%f mix/sleep) - throttling %f of streams",
(double)_trailingMixRatio, (double)_throttlingRatio);
}
}
}

12
assignment-client/src/audio/AudioMixer.h
View file

@ -46,7 +46,6 @@ public:
static int getStaticJitterFrames() { return _numStaticJitterFrames; }
static bool shouldMute(float quietestFrame) { return quietestFrame > _noiseMutingThreshold; }
static float getAttenuationPerDoublingInDistance() { return _attenuationPerDoublingInDistance; }
static float getMinimumAudibilityThreshold() { return _performanceThrottlingRatio > 0.0f ? _minAudibilityThreshold : 0.0f; }
static const QHash<QString, AABox>& getAudioZones() { return _audioZones; }
static const QVector<ZoneSettings>& getZoneSettings() { return _zoneSettings; }
static const QVector<ReverbSettings>& getReverbSettings() { return _zoneReverbSettings; }
@ -73,8 +72,8 @@ private slots:
private:
// mixing helpers
// check and maybe throttle mixer load by changing audibility threshold
void manageLoad(p_high_resolution_clock::time_point& frameTimestamp, unsigned int& framesSinceManagement);
std::chrono::microseconds timeFrame(p_high_resolution_clock::time_point& timestamp);
void throttle(std::chrono::microseconds frameDuration, int frame);
// pop a frame from any streams on the node
// returns the number of available streams
int prepareFrame(const SharedNodePointer& node, unsigned int frame);
@ -85,6 +84,9 @@ private:
void parseSettingsObject(const QJsonObject& settingsObject);
float _trailingMixRatio { 0.0f };
float _throttlingRatio { 0.0f };
int _numStatFrames { 0 };
AudioMixerStats _stats;
@ -113,6 +115,7 @@ private:
uint64_t _history[TIMER_TRAILING_SECONDS] {};
int _index { 0 };
};
Timer _ticTiming;
Timer _sleepTiming;
Timer _frameTiming;
Timer _prepareTiming;
@ -122,9 +125,6 @@ private:
static int _numStaticJitterFrames; // -1 denotes dynamic jitter buffering
static float _noiseMutingThreshold;
static float _attenuationPerDoublingInDistance;
static float _trailingSleepRatio;
static float _performanceThrottlingRatio;
static float _minAudibilityThreshold;
static QHash<QString, AABox> _audioZones;
static QVector<ZoneSettings> _zoneSettings;
static QVector<ReverbSettings> _zoneReverbSettings;

591
assignment-client/src/audio/AudioMixerSlave.cpp
View file

@ -36,6 +36,292 @@
#include "AudioMixerSlave.h"
using AudioStreamMap = AudioMixerClientData::AudioStreamMap;
// packet helpers
std::unique_ptr<NLPacket> createAudioPacket(PacketType type, int size, quint16 sequence, QString codec);
void sendMixPacket(const SharedNodePointer& node, AudioMixerClientData& data, QByteArray& buffer);
void sendSilentPacket(const SharedNodePointer& node, AudioMixerClientData& data);
void sendMutePacket(const SharedNodePointer& node, AudioMixerClientData&);
void sendEnvironmentPacket(const SharedNodePointer& node, AudioMixerClientData& data);
// mix helpers
bool shouldIgnoreNode(const SharedNodePointer& listener, const SharedNodePointer& node);
float gainForSource(const AvatarAudioStream& listeningNodeStream, const PositionalAudioStream& streamToAdd,
const glm::vec3& relativePosition, bool isEcho);
float azimuthForSource(const AvatarAudioStream& listeningNodeStream, const PositionalAudioStream& streamToAdd,
const glm::vec3& relativePosition);
void AudioMixerSlave::configure(ConstIter begin, ConstIter end, unsigned int frame, float throttlingRatio) {
_begin = begin;
_end = end;
_frame = frame;
_throttlingRatio = throttlingRatio;
}
void AudioMixerSlave::mix(const SharedNodePointer& node) {
// check that the node is valid
AudioMixerClientData* data = (AudioMixerClientData*)node->getLinkedData();
if (data == nullptr) {
return;
}
// check that the stream is valid
auto avatarStream = data->getAvatarAudioStream();
if (avatarStream == nullptr) {
return;
}
// send mute packet, if necessary
if (AudioMixer::shouldMute(avatarStream->getQuietestFrameLoudness()) || data->shouldMuteClient()) {
sendMutePacket(node, *data);
}
// send audio packets, if necessary
if (node->getType() == NodeType::Agent && node->getActiveSocket()) {
++stats.sumListeners;
// mix the audio
bool mixHasAudio = prepareMix(node);
// send audio packet
if (mixHasAudio || data->shouldFlushEncoder()) {
QByteArray encodedBuffer;
if (mixHasAudio) {
// encode the audio
QByteArray decodedBuffer(reinterpret_cast<char*>(_bufferSamples), AudioConstants::NETWORK_FRAME_BYTES_STEREO);
data->encode(decodedBuffer, encodedBuffer);
} else {
// time to flush (resets shouldFlush until the next encode)
data->encodeFrameOfZeros(encodedBuffer);
}
sendMixPacket(node, *data, encodedBuffer);
} else {
sendSilentPacket(node, *data);
}
// send environment packet
sendEnvironmentPacket(node, *data);
// send stats packet (about every second)
const unsigned int NUM_FRAMES_PER_SEC = (int)ceil(AudioConstants::NETWORK_FRAMES_PER_SEC);
if (data->shouldSendStats(_frame % NUM_FRAMES_PER_SEC)) {
data->sendAudioStreamStatsPackets(node);
}
}
}
bool AudioMixerSlave::prepareMix(const SharedNodePointer& listener) {
AvatarAudioStream* listenerAudioStream = static_cast<AudioMixerClientData*>(listener->getLinkedData())->getAvatarAudioStream();
AudioMixerClientData* listenerData = static_cast<AudioMixerClientData*>(listener->getLinkedData());
// zero out the mix for this listener
memset(_mixSamples, 0, sizeof(_mixSamples));
bool isThrottling = _throttlingRatio > 0.0f;
std::vector<std::pair<float, SharedNodePointer>> throttledNodes;
typedef void (AudioMixerSlave::*MixFunctor)(
AudioMixerClientData&, const QUuid&, const AvatarAudioStream&, const PositionalAudioStream&);
auto allStreams = [&](const SharedNodePointer& node, MixFunctor mixFunctor) {
AudioMixerClientData* nodeData = static_cast<AudioMixerClientData*>(node->getLinkedData());
for (auto& streamPair : nodeData->getAudioStreams()) {
auto nodeStream = streamPair.second;
(this->*mixFunctor)(*listenerData, node->getUUID(), *listenerAudioStream, *nodeStream);
}
};
std::for_each(_begin, _end, [&](const SharedNodePointer& node) {
if (*node == *listener) {
AudioMixerClientData* nodeData = static_cast<AudioMixerClientData*>(node->getLinkedData());
// only mix the echo, if requested
for (auto& streamPair : nodeData->getAudioStreams()) {
auto nodeStream = streamPair.second;
if (nodeStream->shouldLoopbackForNode()) {
mixStream(*listenerData, node->getUUID(), *listenerAudioStream, *nodeStream);
}
}
} else if (!shouldIgnoreNode(listener, node)) {
if (!isThrottling) {
allStreams(node, &AudioMixerSlave::mixStream);
} else {
AudioMixerClientData* nodeData = static_cast<AudioMixerClientData*>(node->getLinkedData());
// compute the node's max relative volume
float nodeVolume;
for (auto& streamPair : nodeData->getAudioStreams()) {
auto nodeStream = streamPair.second;
float distance = glm::length(nodeStream->getPosition() - listenerAudioStream->getPosition());
nodeVolume = std::max(nodeStream->getLastPopOutputTrailingLoudness() / distance, nodeVolume);
}
// max-heapify the nodes by relative volume
throttledNodes.push_back(std::make_pair(nodeVolume, node));
if (!throttledNodes.empty()) {
std::push_heap(throttledNodes.begin(), throttledNodes.end());
}
}
}
});
if (isThrottling) {
// pop the loudest nodes off the heap and mix their streams
int numToRetain = (int)(std::distance(_begin, _end) * (1 - _throttlingRatio));
for (int i = 0; i < numToRetain; i++) {
if (throttledNodes.empty()) {
break;
}
std::pop_heap(throttledNodes.begin(), throttledNodes.end());
auto& node = throttledNodes.back().second;
allStreams(node, &AudioMixerSlave::mixStream);
throttledNodes.pop_back();
}
// throttle the remaining nodes' streams
for (const std::pair<float, SharedNodePointer>& nodePair : throttledNodes) {
auto& node = nodePair.second;
allStreams(node, &AudioMixerSlave::throttleStream);
}
}
// use the per listener AudioLimiter to render the mixed data...
listenerData->audioLimiter.render(_mixSamples, _bufferSamples, AudioConstants::NETWORK_FRAME_SAMPLES_PER_CHANNEL);
// check for silent audio after the peak limiter has converted the samples
bool hasAudio = false;
for (int i = 0; i < AudioConstants::NETWORK_FRAME_SAMPLES_STEREO; ++i) {
if (_bufferSamples[i] != 0) {
hasAudio = true;
break;
}
}
return hasAudio;
}
void AudioMixerSlave::throttleStream(AudioMixerClientData& listenerNodeData, const QUuid& sourceNodeID,
const AvatarAudioStream& listeningNodeStream, const PositionalAudioStream& streamToAdd) {
addStream(listenerNodeData, sourceNodeID, listeningNodeStream, streamToAdd, true);
}
void AudioMixerSlave::mixStream(AudioMixerClientData& listenerNodeData, const QUuid& sourceNodeID,
const AvatarAudioStream& listeningNodeStream, const PositionalAudioStream& streamToAdd) {
addStream(listenerNodeData, sourceNodeID, listeningNodeStream, streamToAdd, false);
}
void AudioMixerSlave::addStream(AudioMixerClientData& listenerNodeData, const QUuid& sourceNodeID,
const AvatarAudioStream& listeningNodeStream, const PositionalAudioStream& streamToAdd,
bool throttle) {
++stats.totalMixes;
// to reduce artifacts we call the HRTF functor for every source, even if throttled or silent
// this ensures the correct tail from last mixed block and the correct spatialization of next first block
// check if this is a server echo of a source back to itself
bool isEcho = (&streamToAdd == &listeningNodeStream);
glm::vec3 relativePosition = streamToAdd.getPosition() - listeningNodeStream.getPosition();
float distance = glm::max(glm::length(relativePosition), EPSILON);
float gain = gainForSource(listeningNodeStream, streamToAdd, relativePosition, isEcho);
float azimuth = isEcho ? 0.0f : azimuthForSource(listeningNodeStream, listeningNodeStream, relativePosition);
static const int HRTF_DATASET_INDEX = 1;
if (!streamToAdd.lastPopSucceeded()) {
bool forceSilentBlock = true;
if (!streamToAdd.getLastPopOutput().isNull()) {
bool isInjector = dynamic_cast<const InjectedAudioStream*>(&streamToAdd);
// in an injector, just go silent - the injector has likely ended
// in other inputs (microphone, &c.), repeat with fade to avoid the harsh jump to silence
if (!isInjector) {
// calculate its fade factor, which depends on how many times it's already been repeated.
float fadeFactor = calculateRepeatedFrameFadeFactor(streamToAdd.getConsecutiveNotMixedCount() - 1);
if (fadeFactor > 0.0f) {
// apply the fadeFactor to the gain
gain *= fadeFactor;
forceSilentBlock = false;
}
}
}
if (forceSilentBlock) {
// call renderSilent with a forced silent block to reduce artifacts
// (this is not done for stereo streams since they do not go through the HRTF)
if (!streamToAdd.isStereo() && !isEcho) {
// get the existing listener-source HRTF object, or create a new one
auto& hrtf = listenerNodeData.hrtfForStream(sourceNodeID, streamToAdd.getStreamIdentifier());
static int16_t silentMonoBlock[AudioConstants::NETWORK_FRAME_SAMPLES_PER_CHANNEL] = {};
hrtf.renderSilent(silentMonoBlock, _mixSamples, HRTF_DATASET_INDEX, azimuth, distance, gain,
AudioConstants::NETWORK_FRAME_SAMPLES_PER_CHANNEL);
++stats.hrtfSilentRenders;
}
return;
}
}
// grab the stream from the ring buffer
AudioRingBuffer::ConstIterator streamPopOutput = streamToAdd.getLastPopOutput();
// stereo sources are not passed through HRTF
if (streamToAdd.isStereo()) {
for (int i = 0; i < AudioConstants::NETWORK_FRAME_SAMPLES_STEREO; ++i) {
_mixSamples[i] += float(streamPopOutput[i] * gain / AudioConstants::MAX_SAMPLE_VALUE);
}
++stats.manualStereoMixes;
return;
}
// echo sources are not passed through HRTF
if (isEcho) {
for (int i = 0; i < AudioConstants::NETWORK_FRAME_SAMPLES_STEREO; i += 2) {
auto monoSample = float(streamPopOutput[i / 2] * gain / AudioConstants::MAX_SAMPLE_VALUE);
_mixSamples[i] += monoSample;
_mixSamples[i + 1] += monoSample;
}
++stats.manualEchoMixes;
return;
}
// get the existing listener-source HRTF object, or create a new one
auto& hrtf = listenerNodeData.hrtfForStream(sourceNodeID, streamToAdd.getStreamIdentifier());
streamPopOutput.readSamples(_bufferSamples, AudioConstants::NETWORK_FRAME_SAMPLES_PER_CHANNEL);
if (streamToAdd.getLastPopOutputLoudness() == 0.0f) {
// call renderSilent to reduce artifacts
hrtf.renderSilent(_bufferSamples, _mixSamples, HRTF_DATASET_INDEX, azimuth, distance, gain,
AudioConstants::NETWORK_FRAME_SAMPLES_PER_CHANNEL);
++stats.hrtfSilentRenders;
return;
}
if (throttle) {
// call renderSilent with actual frame data and a gain of 0.0f to reduce artifacts
hrtf.renderSilent(_bufferSamples, _mixSamples, HRTF_DATASET_INDEX, azimuth, distance, 0.0f,
AudioConstants::NETWORK_FRAME_SAMPLES_PER_CHANNEL);
++stats.hrtfThrottleRenders;
return;
}
hrtf.render(_bufferSamples, _mixSamples, HRTF_DATASET_INDEX, azimuth, distance, gain,
AudioConstants::NETWORK_FRAME_SAMPLES_PER_CHANNEL);
++stats.hrtfRenders;
}
std::unique_ptr<NLPacket> createAudioPacket(PacketType type, int size, quint16 sequence, QString codec) {
auto audioPacket = NLPacket::create(type, size);
audioPacket->writePrimitive(sequence);
@ -73,6 +359,14 @@ void sendSilentPacket(const SharedNodePointer& node, AudioMixerClientData& data)
data.incrementOutgoingMixedAudioSequenceNumber();
}
void sendMutePacket(const SharedNodePointer& node, AudioMixerClientData& data) {
auto mutePacket = NLPacket::create(PacketType::NoisyMute, 0);
DependencyManager::get<NodeList>()->sendPacket(std::move(mutePacket), *node);
// probably now we just reset the flag, once should do it (?)
data.setShouldMuteClient(false);
}
void sendEnvironmentPacket(const SharedNodePointer& node, AudioMixerClientData& data) {
bool hasReverb = false;
float reverbTime, wetLevel;
@ -134,285 +428,54 @@ void sendEnvironmentPacket(const SharedNodePointer& node, AudioMixerClientData&
}
}
void AudioMixerSlave::configure(ConstIter begin, ConstIter end, unsigned int frame) {
_begin = begin;
_end = end;
_frame = frame;
}
void AudioMixerSlave::mix(const SharedNodePointer& node) {
// check that the node is valid
AudioMixerClientData* data = (AudioMixerClientData*)node->getLinkedData();
if (data == nullptr) {
return;
}
auto avatarStream = data->getAvatarAudioStream();
if (avatarStream == nullptr) {
return;
}
// send mute packet, if necessary
if (AudioMixer::shouldMute(avatarStream->getQuietestFrameLoudness()) || data->shouldMuteClient()) {
auto mutePacket = NLPacket::create(PacketType::NoisyMute, 0);
DependencyManager::get<NodeList>()->sendPacket(std::move(mutePacket), *node);
// probably now we just reset the flag, once should do it (?)
data->setShouldMuteClient(false);
}
// send audio packets, if necessary
if (node->getType() == NodeType::Agent && node->getActiveSocket()) {
++stats.sumListeners;
// mix the audio
bool mixHasAudio = prepareMix(node);
// send audio packet
if (mixHasAudio || data->shouldFlushEncoder()) {
// encode the audio
QByteArray encodedBuffer;
if (mixHasAudio) {
QByteArray decodedBuffer(reinterpret_cast<char*>(_bufferSamples), AudioConstants::NETWORK_FRAME_BYTES_STEREO);
data->encode(decodedBuffer, encodedBuffer);
} else {
// time to flush, which resets the shouldFlush until next time we encode something
data->encodeFrameOfZeros(encodedBuffer);
}
sendMixPacket(node, *data, encodedBuffer);
} else {
sendSilentPacket(node, *data);
}
// send environment packet
sendEnvironmentPacket(node, *data);
// send stats packet (about every second)
static const unsigned int NUM_FRAMES_PER_SEC = (int) ceil(AudioConstants::NETWORK_FRAMES_PER_SEC);
if (data->shouldSendStats(_frame % NUM_FRAMES_PER_SEC)) {
data->sendAudioStreamStatsPackets(node);
}
}
}
bool AudioMixerSlave::prepareMix(const SharedNodePointer& node) {
AvatarAudioStream* nodeAudioStream = static_cast<AudioMixerClientData*>(node->getLinkedData())->getAvatarAudioStream();
bool shouldIgnoreNode(const SharedNodePointer& listener, const SharedNodePointer& node) {
AudioMixerClientData* listenerData = static_cast<AudioMixerClientData*>(listener->getLinkedData());
AudioMixerClientData* nodeData = static_cast<AudioMixerClientData*>(node->getLinkedData());
// zero out the client mix for this node
memset(_mixSamples, 0, sizeof(_mixSamples));
// when this is true, the AudioMixer will send Audio data to a client about avatars that have ignored them
bool getsAnyIgnored = listenerData->getRequestsDomainListData() && listener->getCanKick();
// loop through all other nodes that have sufficient audio to mix
std::for_each(_begin, _end, [&](const SharedNodePointer& otherNode){
// make sure that we have audio data for this other node
// and that it isn't being ignored by our listening node
// and that it isn't ignoring our listening node
AudioMixerClientData* otherData = static_cast<AudioMixerClientData*>(otherNode->getLinkedData());
bool ignore = true;
// When this is true, the AudioMixer will send Audio data to a client about avatars that have ignored them
bool getsAnyIgnored = nodeData->getRequestsDomainListData() && node->getCanKick();
if (nodeData &&
// make sure that it isn't being ignored by our listening node
(!listener->isIgnoringNodeWithID(node->getUUID()) || (nodeData->getRequestsDomainListData() && node->getCanKick())) &&
// and that it isn't ignoring our listening node
(!node->isIgnoringNodeWithID(listener->getUUID()) || getsAnyIgnored)) {
if (otherData
&& (!node->isIgnoringNodeWithID(otherNode->getUUID()) || (otherData->getRequestsDomainListData() && otherNode->getCanKick()))
&& (!otherNode->isIgnoringNodeWithID(node->getUUID()) || getsAnyIgnored)) {
// is either node enabling the space bubble / ignore radius?
if ((listener->isIgnoreRadiusEnabled() || node->isIgnoreRadiusEnabled())) {
// define the minimum bubble size
static const glm::vec3 minBubbleSize = glm::vec3(0.3f, 1.3f, 0.3f);
// check to see if we're ignoring in radius
bool insideIgnoreRadius = false;
// If the otherNode equals the node, we're doing a comparison on ourselves
if (*otherNode == *node) {
// We'll always be inside the radius in that case.
insideIgnoreRadius = true;
// Check to see if the space bubble is enabled
} else if ((node->isIgnoreRadiusEnabled() || otherNode->isIgnoreRadiusEnabled())) {
// Define the minimum bubble size
static const glm::vec3 minBubbleSize = glm::vec3(0.3f, 1.3f, 0.3f);
AudioMixerClientData* nodeData = reinterpret_cast<AudioMixerClientData*>(node->getLinkedData());
// Set up the bounding box for the current node
AABox nodeBox(nodeData->getAvatarBoundingBoxCorner(), nodeData->getAvatarBoundingBoxScale());
// Clamp the size of the bounding box to a minimum scale
if (glm::any(glm::lessThan(nodeData->getAvatarBoundingBoxScale(), minBubbleSize))) {
nodeBox.setScaleStayCentered(minBubbleSize);
}
// Set up the bounding box for the current other node
AABox otherNodeBox(otherData->getAvatarBoundingBoxCorner(), otherData->getAvatarBoundingBoxScale());
// Clamp the size of the bounding box to a minimum scale
if (glm::any(glm::lessThan(otherData->getAvatarBoundingBoxScale(), minBubbleSize))) {
otherNodeBox.setScaleStayCentered(minBubbleSize);
}
// Quadruple the scale of both bounding boxes
nodeBox.embiggen(4.0f);
otherNodeBox.embiggen(4.0f);
// Perform the collision check between the two bounding boxes
if (nodeBox.touches(otherNodeBox)) {
insideIgnoreRadius = true;
}
// set up the bounding box for the listener
AABox listenerBox(listenerData->getAvatarBoundingBoxCorner(), listenerData->getAvatarBoundingBoxScale());
if (glm::any(glm::lessThan(listenerData->getAvatarBoundingBoxScale(), minBubbleSize))) {
listenerBox.setScaleStayCentered(minBubbleSize);
}
// Enumerate the audio streams attached to the otherNode
auto streamsCopy = otherData->getAudioStreams();
for (auto& streamPair : streamsCopy) {
auto otherNodeStream = streamPair.second;
bool isSelfWithEcho = ((*otherNode == *node) && (otherNodeStream->shouldLoopbackForNode()));
// Add all audio streams that should be added to the mix
if (isSelfWithEcho || (!isSelfWithEcho && !insideIgnoreRadius)) {
addStreamToMix(*nodeData, otherNode->getUUID(), *nodeAudioStream, *otherNodeStream);
}
}
}
});
// use the per listener AudioLimiter to render the mixed data...
nodeData->audioLimiter.render(_mixSamples, _bufferSamples, AudioConstants::NETWORK_FRAME_SAMPLES_PER_CHANNEL);
// check for silent audio after the peak limiter has converted the samples
bool hasAudio = false;
for (int i = 0; i < AudioConstants::NETWORK_FRAME_SAMPLES_STEREO; ++i) {
if (_bufferSamples[i] != 0) {
hasAudio = true;
break;
}
}
return hasAudio;
}
void AudioMixerSlave::addStreamToMix(AudioMixerClientData& listenerNodeData, const QUuid& sourceNodeID,
const AvatarAudioStream& listeningNodeStream, const PositionalAudioStream& streamToAdd) {
// to reduce artifacts we calculate the gain and azimuth for every source for this listener
// even if we are not going to end up mixing in this source
++stats.totalMixes;
// this ensures that the tail of any previously mixed audio or the first block of new audio sounds correct
// check if this is a server echo of a source back to itself
bool isEcho = (&streamToAdd == &listeningNodeStream);
glm::vec3 relativePosition = streamToAdd.getPosition() - listeningNodeStream.getPosition();
// figure out the distance between source and listener
float distance = glm::max(glm::length(relativePosition), EPSILON);
// figure out the gain for this source at the listener
float gain = gainForSource(listeningNodeStream, streamToAdd, relativePosition, isEcho);
// figure out the azimuth to this source at the listener
float azimuth = isEcho ? 0.0f : azimuthForSource(listeningNodeStream, listeningNodeStream, relativePosition);
float repeatedFrameFadeFactor = 1.0f;
static const int HRTF_DATASET_INDEX = 1;
if (!streamToAdd.lastPopSucceeded()) {
bool forceSilentBlock = true;
if (!streamToAdd.getLastPopOutput().isNull()) {
bool isInjector = dynamic_cast<const InjectedAudioStream*>(&streamToAdd);
// in an injector, just go silent - the injector has likely ended
// in other inputs (microphone, &c.), repeat with fade to avoid the harsh jump to silence
// we'll repeat the last block until it has a block to mix
// and we'll gradually fade that repeated block into silence.
// calculate its fade factor, which depends on how many times it's already been repeated.
repeatedFrameFadeFactor = calculateRepeatedFrameFadeFactor(streamToAdd.getConsecutiveNotMixedCount() - 1);
if (!isInjector && repeatedFrameFadeFactor > 0.0f) {
// apply the repeatedFrameFadeFactor to the gain
gain *= repeatedFrameFadeFactor;
forceSilentBlock = false;
}
}
if (forceSilentBlock) {
// we're deciding not to repeat either since we've already done it enough times or repetition with fade is disabled
// in this case we will call renderSilent with a forced silent block
// this ensures the correct tail from the previously mixed block and the correct spatialization of first block
// of any upcoming audio
if (!streamToAdd.isStereo() && !isEcho) {
// get the existing listener-source HRTF object, or create a new one
auto& hrtf = listenerNodeData.hrtfForStream(sourceNodeID, streamToAdd.getStreamIdentifier());
// this is not done for stereo streams since they do not go through the HRTF
static int16_t silentMonoBlock[AudioConstants::NETWORK_FRAME_SAMPLES_PER_CHANNEL] = {};
hrtf.renderSilent(silentMonoBlock, _mixSamples, HRTF_DATASET_INDEX, azimuth, distance, gain,
AudioConstants::NETWORK_FRAME_SAMPLES_PER_CHANNEL);
++stats.hrtfSilentRenders;
// set up the bounding box for the node
AABox nodeBox(nodeData->getAvatarBoundingBoxCorner(), nodeData->getAvatarBoundingBoxScale());
// Clamp the size of the bounding box to a minimum scale
if (glm::any(glm::lessThan(nodeData->getAvatarBoundingBoxScale(), minBubbleSize))) {
nodeBox.setScaleStayCentered(minBubbleSize);
}
return;
}
}
// quadruple the scale of both bounding boxes
listenerBox.embiggen(4.0f);
nodeBox.embiggen(4.0f);
// grab the stream from the ring buffer
AudioRingBuffer::ConstIterator streamPopOutput = streamToAdd.getLastPopOutput();
if (streamToAdd.isStereo() || isEcho) {
// this is a stereo source or server echo so we do not pass it through the HRTF
// simply apply our calculated gain to each sample
if (streamToAdd.isStereo()) {
for (int i = 0; i < AudioConstants::NETWORK_FRAME_SAMPLES_STEREO; ++i) {
_mixSamples[i] += float(streamPopOutput[i] * gain / AudioConstants::MAX_SAMPLE_VALUE);
}
++stats.manualStereoMixes;
// perform the collision check between the two bounding boxes
ignore = listenerBox.touches(nodeBox);
} else {
for (int i = 0; i < AudioConstants::NETWORK_FRAME_SAMPLES_STEREO; i += 2) {
auto monoSample = float(streamPopOutput[i / 2] * gain / AudioConstants::MAX_SAMPLE_VALUE);
_mixSamples[i] += monoSample;
_mixSamples[i + 1] += monoSample;
}
++stats.manualEchoMixes;
ignore = false;
}
return;
}
// get the existing listener-source HRTF object, or create a new one
auto& hrtf = listenerNodeData.hrtfForStream(sourceNodeID, streamToAdd.getStreamIdentifier());
streamPopOutput.readSamples(_bufferSamples, AudioConstants::NETWORK_FRAME_SAMPLES_PER_CHANNEL);
// if the frame we're about to mix is silent, simply call render silent and move on
if (streamToAdd.getLastPopOutputLoudness() == 0.0f) {
// silent frame from source
// we still need to call renderSilent via the HRTF for mono source
hrtf.renderSilent(_bufferSamples, _mixSamples, HRTF_DATASET_INDEX, azimuth, distance, gain,
AudioConstants::NETWORK_FRAME_SAMPLES_PER_CHANNEL);
++stats.hrtfSilentRenders;
return;
}
float audibilityThreshold = AudioMixer::getMinimumAudibilityThreshold();
if (audibilityThreshold > 0.0f &&
streamToAdd.getLastPopOutputTrailingLoudness() / glm::length(relativePosition) <= audibilityThreshold) {
// the mixer is struggling so we're going to drop off some streams
// we call renderSilent via the HRTF with the actual frame data and a gain of 0.0
hrtf.renderSilent(_bufferSamples, _mixSamples, HRTF_DATASET_INDEX, azimuth, distance, 0.0f,
AudioConstants::NETWORK_FRAME_SAMPLES_PER_CHANNEL);
++stats.hrtfStruggleRenders;
return;
}
++stats.hrtfRenders;
// mono stream, call the HRTF with our block and calculated azimuth and gain
hrtf.render(_bufferSamples, _mixSamples, HRTF_DATASET_INDEX, azimuth, distance, gain,
AudioConstants::NETWORK_FRAME_SAMPLES_PER_CHANNEL);
return ignore;
}
float AudioMixerSlave::gainForSource(const AvatarAudioStream& listeningNodeStream, const PositionalAudioStream& streamToAdd,
float gainForSource(const AvatarAudioStream& listeningNodeStream, const PositionalAudioStream& streamToAdd,
const glm::vec3& relativePosition, bool isEcho) {
float gain = 1.0f;
@ -472,7 +535,7 @@ float AudioMixerSlave::gainForSource(const AvatarAudioStream& listeningNodeStrea
return gain;
}
float AudioMixerSlave::azimuthForSource(const AvatarAudioStream& listeningNodeStream, const PositionalAudioStream& streamToAdd,
float azimuthForSource(const AvatarAudioStream& listeningNodeStream, const PositionalAudioStream& streamToAdd,
const glm::vec3& relativePosition) {
glm::quat inverseOrientation = glm::inverse(listeningNodeStream.getOrientation());
@ -482,7 +545,7 @@ float AudioMixerSlave::azimuthForSource(const AvatarAudioStream& listeningNodeSt
// project the rotated source position vector onto the XZ plane
rotatedSourcePosition.y = 0.0f;
static const float SOURCE_DISTANCE_THRESHOLD = 1e-30f;
const float SOURCE_DISTANCE_THRESHOLD = 1e-30f;
if (glm::length2(rotatedSourcePosition) > SOURCE_DISTANCE_THRESHOLD) {
// produce an oriented angle about the y-axis

18
assignment-client/src/audio/AudioMixerSlave.h
View file

@ -30,7 +30,7 @@ class AudioMixerSlave {
public:
using ConstIter = NodeList::const_iterator;
void configure(ConstIter begin, ConstIter end, unsigned int frame);
void configure(ConstIter begin, ConstIter end, unsigned int frame, float throttlingRatio);
// mix and broadcast non-ignored streams to the node
// returns true if a mixed packet was sent to the node
@ -40,15 +40,14 @@ public:
private:
// create mix, returns true if mix has audio
bool prepareMix(const SharedNodePointer& node);
// add a stream to the mix
void addStreamToMix(AudioMixerClientData& listenerData, const QUuid& streamerID,
bool prepareMix(const SharedNodePointer& listener);
void throttleStream(AudioMixerClientData& listenerData, const QUuid& streamerID,
const AvatarAudioStream& listenerStream, const PositionalAudioStream& streamer);
float gainForSource(const AvatarAudioStream& listener, const PositionalAudioStream& streamer,
const glm::vec3& relativePosition, bool isEcho);
float azimuthForSource(const AvatarAudioStream& listener, const PositionalAudioStream& streamer,
const glm::vec3& relativePosition);
void mixStream(AudioMixerClientData& listenerData, const QUuid& streamerID,
const AvatarAudioStream& listenerStream, const PositionalAudioStream& streamer);
void addStream(AudioMixerClientData& listenerData, const QUuid& streamerID,
const AvatarAudioStream& listenerStream, const PositionalAudioStream& streamer,
bool throttle);
// mixing buffers
float _mixSamples[AudioConstants::NETWORK_FRAME_SAMPLES_STEREO];
@ -58,6 +57,7 @@ private:
ConstIter _begin;
ConstIter _end;
unsigned int _frame { 0 };
float _throttlingRatio { 0.0f };
};
#endif // hifi_AudioMixerSlave_h

7
assignment-client/src/audio/AudioMixerSlavePool.cpp
View file

@ -41,7 +41,7 @@ void AudioMixerSlaveThread::wait() {
});
++_pool._numStarted;
}
configure(_pool._begin, _pool._end, _pool._frame);
configure(_pool._begin, _pool._end, _pool._frame, _pool._throttlingRatio);
}
void AudioMixerSlaveThread::notify(bool stopping) {
@ -64,13 +64,14 @@ bool AudioMixerSlaveThread::try_pop(SharedNodePointer& node) {
static AudioMixerSlave slave;
#endif
void AudioMixerSlavePool::mix(ConstIter begin, ConstIter end, unsigned int frame) {
void AudioMixerSlavePool::mix(ConstIter begin, ConstIter end, unsigned int frame, float throttlingRatio) {
_begin = begin;
_end = end;
_frame = frame;
_throttlingRatio = throttlingRatio;
#ifdef AUDIO_SINGLE_THREADED
slave.configure(_begin, _end, frame);
slave.configure(_begin, _end, frame, throttlingRatio);
std::for_each(begin, end, [&](const SharedNodePointer& node) {
slave.mix(node);
});

3
assignment-client/src/audio/AudioMixerSlavePool.h
View file

@ -61,7 +61,7 @@ public:
~AudioMixerSlavePool() { resize(0); }
// mix on slave threads
void mix(ConstIter begin, ConstIter end, unsigned int frame);
void mix(ConstIter begin, ConstIter end, unsigned int frame, float throttlingRatio);
// iterate over all slaves
void each(std::function<void(AudioMixerSlave& slave)> functor);
@ -90,6 +90,7 @@ private:
// frame state
Queue _queue;
unsigned int _frame { 0 };
float _throttlingRatio { 0.0f };
ConstIter _begin;
ConstIter _end;
};

4
assignment-client/src/audio/AudioMixerStats.cpp
View file

@ -17,7 +17,7 @@ void AudioMixerStats::reset() {
totalMixes = 0;
hrtfRenders = 0;
hrtfSilentRenders = 0;
hrtfStruggleRenders = 0;
hrtfThrottleRenders = 0;
manualStereoMixes = 0;
manualEchoMixes = 0;
}
@ -28,7 +28,7 @@ void AudioMixerStats::accumulate(const AudioMixerStats& otherStats) {
totalMixes += otherStats.totalMixes;
hrtfRenders += otherStats.hrtfRenders;
hrtfSilentRenders += otherStats.hrtfSilentRenders;
hrtfStruggleRenders += otherStats.hrtfStruggleRenders;
hrtfThrottleRenders += otherStats.hrtfThrottleRenders;
manualStereoMixes += otherStats.manualStereoMixes;
manualEchoMixes += otherStats.manualEchoMixes;
}

2
assignment-client/src/audio/AudioMixerStats.h
View file

@ -20,7 +20,7 @@ struct AudioMixerStats {
int hrtfRenders { 0 };
int hrtfSilentRenders { 0 };
int hrtfStruggleRenders { 0 };
int hrtfThrottleRenders { 0 };
int manualStereoMixes { 0 };
int manualEchoMixes { 0 };