diff --git a/assignment-client/src/audio/AudioMixer.cpp b/assignment-client/src/audio/AudioMixer.cpp index 5001e4419d..deff1dfa6f 100644 --- a/assignment-client/src/audio/AudioMixer.cpp +++ b/assignment-client/src/audio/AudioMixer.cpp @@ -36,7 +36,6 @@ #include "AudioMixer.h" -static const float LOUDNESS_TO_DISTANCE_RATIO = 0.00001f; static const float DEFAULT_ATTENUATION_PER_DOUBLING_IN_DISTANCE = 0.5f; // attenuation = -6dB * log2(distance) static const float DEFAULT_NOISE_MUTING_THRESHOLD = 1.0f; static const QString AUDIO_MIXER_LOGGING_TARGET_NAME = "audio-mixer"; @@ -47,9 +46,6 @@ static const QString AUDIO_THREADING_GROUP_KEY = "audio_threading"; int AudioMixer::_numStaticJitterFrames{ -1 }; float AudioMixer::_noiseMutingThreshold{ DEFAULT_NOISE_MUTING_THRESHOLD }; float AudioMixer::_attenuationPerDoublingInDistance{ DEFAULT_ATTENUATION_PER_DOUBLING_IN_DISTANCE }; -float AudioMixer::_trailingSleepRatio{ 1.0f }; -float AudioMixer::_performanceThrottlingRatio{ 0.0f }; -float AudioMixer::_minAudibilityThreshold{ LOUDNESS_TO_DISTANCE_RATIO / 2.0f }; QHash AudioMixer::_audioZones; QVector AudioMixer::_zoneSettings; QVector AudioMixer::_zoneReverbSettings; @@ -294,35 +290,31 @@ void AudioMixer::sendStatsPacket() { // general stats statsObject["useDynamicJitterBuffers"] = _numStaticJitterFrames == -1; - statsObject["trailing_sleep_percentage"] = _trailingSleepRatio * 100.0f; - statsObject["performance_throttling_ratio"] = _performanceThrottlingRatio; + + statsObject["threads"] = _slavePool.numThreads(); + + statsObject["trailing_mix_ratio"] = _trailingMixRatio; + statsObject["throttling_ratio"] = _throttlingRatio; statsObject["avg_streams_per_frame"] = (float)_stats.sumStreams / (float)_numStatFrames; statsObject["avg_listeners_per_frame"] = (float)_stats.sumListeners / (float)_numStatFrames; // timing stats QJsonObject timingStats; - uint64_t timing, trailing; - _sleepTiming.get(timing, trailing); - timingStats["us_per_sleep"] = (qint64)(timing / _numStatFrames); - timingStats["us_per_sleep_trailing"] = (qint64)(trailing / _numStatFrames); + auto addTiming = [&](Timer& timer, std::string name) { + uint64_t timing, trailing; + timer.get(timing, trailing); + timingStats[("us_per_" + name).c_str()] = (qint64)(timing / _numStatFrames); + timingStats[("us_per_" + name + "_trailing").c_str()] = (qint64)(trailing / _numStatFrames); + }; - _frameTiming.get(timing, trailing); - timingStats["us_per_frame"] = (qint64)(timing / _numStatFrames); - timingStats["us_per_frame_trailing"] = (qint64)(trailing / _numStatFrames); - - _prepareTiming.get(timing, trailing); - timingStats["us_per_prepare"] = (qint64)(timing / _numStatFrames); - timingStats["us_per_prepare_trailing"] = (qint64)(trailing / _numStatFrames); - - _mixTiming.get(timing, trailing); - timingStats["us_per_mix"] = (qint64)(timing / _numStatFrames); - timingStats["us_per_mix_trailing"] = (qint64)(trailing / _numStatFrames); - - _eventsTiming.get(timing, trailing); - timingStats["us_per_events"] = (qint64)(timing / _numStatFrames); - timingStats["us_per_events_trailing"] = (qint64)(trailing / _numStatFrames); + addTiming(_ticTiming, "tic"); + addTiming(_sleepTiming, "sleep"); + addTiming(_frameTiming, "frame"); + addTiming(_prepareTiming, "prepare"); + addTiming(_mixTiming, "mix"); + addTiming(_eventsTiming, "events"); // call it "avg_..." to keep it higher in the display, sorted alphabetically statsObject["avg_timing_stats"] = timingStats; @@ -332,7 +324,7 @@ void AudioMixer::sendStatsPacket() { mixStats["%_hrtf_mixes"] = percentageForMixStats(_stats.hrtfRenders); mixStats["%_hrtf_silent_mixes"] = percentageForMixStats(_stats.hrtfSilentRenders); - mixStats["%_hrtf_struggle_mixes"] = percentageForMixStats(_stats.hrtfStruggleRenders); + mixStats["%_hrtf_throttle_mixes"] = percentageForMixStats(_stats.hrtfThrottleRenders); mixStats["%_manual_stereo_mixes"] = percentageForMixStats(_stats.manualStereoMixes); mixStats["%_manual_echo_mixes"] = percentageForMixStats(_stats.manualEchoMixes); @@ -408,25 +400,25 @@ void AudioMixer::start() { parseSettingsObject(settingsObject); } - // manageLoad state - auto frameTimestamp = p_high_resolution_clock::time_point::min(); - unsigned int framesSinceManagement = std::numeric_limits::max(); - // mix state unsigned int frame = 1; + auto frameTimestamp = p_high_resolution_clock::now(); while (!_isFinished) { + auto ticTimer = _ticTiming.timer(); + { auto timer = _sleepTiming.timer(); - manageLoad(frameTimestamp, framesSinceManagement); + auto frameDuration = timeFrame(frameTimestamp); + throttle(frameDuration, frame); } - auto timer = _frameTiming.timer(); + auto frameTimer = _frameTiming.timer(); nodeList->nestedEach([&](NodeList::const_iterator cbegin, NodeList::const_iterator cend) { // prepare frames; pop off any new audio from their streams { - auto timer = _prepareTiming.timer(); + auto prepareTimer = _prepareTiming.timer(); std::for_each(cbegin, cend, [&](const SharedNodePointer& node) { _stats.sumStreams += prepareFrame(node, frame); }); @@ -434,8 +426,8 @@ void AudioMixer::start() { // mix across slave threads { - auto timer = _mixTiming.timer(); - _slavePool.mix(cbegin, cend, frame); + auto mixTimer = _mixTiming.timer(); + _slavePool.mix(cbegin, cend, frame, _throttlingRatio); } }); @@ -450,7 +442,7 @@ void AudioMixer::start() { // play nice with qt event-looping { - auto timer = _eventsTiming.timer(); + auto eventsTimer = _eventsTiming.timer(); // since we're a while loop we need to yield to qt's event processing QCoreApplication::processEvents(); @@ -464,67 +456,66 @@ void AudioMixer::start() { } } -void AudioMixer::manageLoad(p_high_resolution_clock::time_point& frameTimestamp, unsigned int& framesSinceCutoffEvent) { - auto timeToSleep = std::chrono::microseconds(0); +std::chrono::microseconds AudioMixer::timeFrame(p_high_resolution_clock::time_point& timestamp) { + // advance the next frame + auto nextTimestamp = timestamp + std::chrono::microseconds(AudioConstants::NETWORK_FRAME_USECS); + auto now = p_high_resolution_clock::now(); - // sleep until the next frame, if necessary - { - // advance the next frame - frameTimestamp += std::chrono::microseconds(AudioConstants::NETWORK_FRAME_USECS); - auto now = p_high_resolution_clock::now(); + // compute how long the last frame took + auto duration = std::chrono::duration_cast(now - timestamp); - // calculate sleep - if (frameTimestamp < now) { - frameTimestamp = now; - } else { - timeToSleep = std::chrono::duration_cast(frameTimestamp - now); - std::this_thread::sleep_for(timeToSleep); - } - } + // set the new frame timestamp + timestamp = std::max(now, nextTimestamp); - // manage mixer load - { - const int TRAILING_AVERAGE_FRAMES = 100; - const float CURRENT_FRAME_RATIO = 1.0f / TRAILING_AVERAGE_FRAMES; - const float PREVIOUS_FRAMES_RATIO = 1.0f - CURRENT_FRAME_RATIO; + // sleep until the next frame should start + std::this_thread::sleep_until(timestamp); - const float STRUGGLE_TRIGGER_SLEEP_PERCENTAGE_THRESHOLD = 0.10f; - const float BACK_OFF_TRIGGER_SLEEP_PERCENTAGE_THRESHOLD = 0.20f; + return duration; +} - const float RATIO_BACK_OFF = 0.02f; +void AudioMixer::throttle(std::chrono::microseconds duration, int frame) { + // throttle using a modified proportional-integral controller + const float FRAME_TIME = 10000.0f; + float mixRatio = duration.count() / FRAME_TIME; - _trailingSleepRatio = (PREVIOUS_FRAMES_RATIO * _trailingSleepRatio) + - // ratio of frame spent sleeping / total frame time - ((CURRENT_FRAME_RATIO * timeToSleep.count()) / (float) AudioConstants::NETWORK_FRAME_USECS); + // constants are determined based on a "regular" 16-CPU EC2 server - bool hasRatioChanged = false; + // target different mix and backoff ratios (they also have different backoff rates) + // this is to prevent oscillation, and encourage throttling to find a steady state + const float TARGET = 0.9f; + // on a "regular" machine with 100 avatars, this is the largest value where + // - overthrottling can be recovered + // - oscillations will not occur after the recovery + const float BACKOFF_TARGET = 0.44f; - if (framesSinceCutoffEvent >= TRAILING_AVERAGE_FRAMES) { - if (_trailingSleepRatio <= STRUGGLE_TRIGGER_SLEEP_PERCENTAGE_THRESHOLD) { - qDebug() << "Mixer is struggling"; - // change our min required loudness to reduce some load - _performanceThrottlingRatio = _performanceThrottlingRatio + (0.5f * (1.0f - _performanceThrottlingRatio)); - hasRatioChanged = true; - } else if (_trailingSleepRatio >= BACK_OFF_TRIGGER_SLEEP_PERCENTAGE_THRESHOLD && _performanceThrottlingRatio != 0) { - qDebug() << "Mixer is recovering"; - // back off the required loudness - _performanceThrottlingRatio = std::max(0.0f, _performanceThrottlingRatio - RATIO_BACK_OFF); - hasRatioChanged = true; - } + // the mixer is known to struggle at about 80 on a "regular" machine + // so throttle 2/80 the streams to ensure smooth audio (throttling is linear) + const float THROTTLE_RATE = 2 / 80.0f; + const float BACKOFF_RATE = THROTTLE_RATE / 4; - if (hasRatioChanged) { - // set out min audability threshold from the new ratio - _minAudibilityThreshold = LOUDNESS_TO_DISTANCE_RATIO / (2.0f * (1.0f - _performanceThrottlingRatio)); - framesSinceCutoffEvent = 0; + // recovery should be bounded so that large changes in user count is a tolerable experience + // throttling is linear, so most cases will not need a full recovery + const int RECOVERY_TIME = 180; - qDebug() << "Sleeping" << _trailingSleepRatio << "of frame"; - qDebug() << "Cutoff is" << _performanceThrottlingRatio; - qDebug() << "Minimum audibility to be mixed is" << _minAudibilityThreshold; - } - } + // weight more recent frames to determine if throttling is necessary, + const int TRAILING_FRAMES = (int)(100 * RECOVERY_TIME * BACKOFF_RATE); + const float CURRENT_FRAME_RATIO = 1.0f / TRAILING_FRAMES; + const float PREVIOUS_FRAMES_RATIO = 1.0f - CURRENT_FRAME_RATIO; + _trailingMixRatio = PREVIOUS_FRAMES_RATIO * _trailingMixRatio + CURRENT_FRAME_RATIO * mixRatio; - if (!hasRatioChanged) { - ++framesSinceCutoffEvent; + if (frame % TRAILING_FRAMES == 0) { + if (_trailingMixRatio > TARGET) { + int proportionalTerm = 1 + (_trailingMixRatio - TARGET) / 0.1f; + _throttlingRatio += THROTTLE_RATE * proportionalTerm; + _throttlingRatio = std::min(_throttlingRatio, 1.0f); + qDebug("audio-mixer is struggling (%f mix/sleep) - throttling %f of streams", + (double)_trailingMixRatio, (double)_throttlingRatio); + } else if (_throttlingRatio > 0.0f && _trailingMixRatio <= BACKOFF_TARGET) { + int proportionalTerm = 1 + (TARGET - _trailingMixRatio) / 0.2f; + _throttlingRatio -= BACKOFF_RATE * proportionalTerm; + _throttlingRatio = std::max(_throttlingRatio, 0.0f); + qDebug("audio-mixer is recovering (%f mix/sleep) - throttling %f of streams", + (double)_trailingMixRatio, (double)_throttlingRatio); } } } diff --git a/assignment-client/src/audio/AudioMixer.h b/assignment-client/src/audio/AudioMixer.h index d88bc3b917..f9c4252ecf 100644 --- a/assignment-client/src/audio/AudioMixer.h +++ b/assignment-client/src/audio/AudioMixer.h @@ -46,7 +46,6 @@ public: static int getStaticJitterFrames() { return _numStaticJitterFrames; } static bool shouldMute(float quietestFrame) { return quietestFrame > _noiseMutingThreshold; } static float getAttenuationPerDoublingInDistance() { return _attenuationPerDoublingInDistance; } - static float getMinimumAudibilityThreshold() { return _performanceThrottlingRatio > 0.0f ? _minAudibilityThreshold : 0.0f; } static const QHash& getAudioZones() { return _audioZones; } static const QVector& getZoneSettings() { return _zoneSettings; } static const QVector& getReverbSettings() { return _zoneReverbSettings; } @@ -73,8 +72,8 @@ private slots: private: // mixing helpers - // check and maybe throttle mixer load by changing audibility threshold - void manageLoad(p_high_resolution_clock::time_point& frameTimestamp, unsigned int& framesSinceManagement); + std::chrono::microseconds timeFrame(p_high_resolution_clock::time_point& timestamp); + void throttle(std::chrono::microseconds frameDuration, int frame); // pop a frame from any streams on the node // returns the number of available streams int prepareFrame(const SharedNodePointer& node, unsigned int frame); @@ -85,6 +84,9 @@ private: void parseSettingsObject(const QJsonObject& settingsObject); + float _trailingMixRatio { 0.0f }; + float _throttlingRatio { 0.0f }; + int _numStatFrames { 0 }; AudioMixerStats _stats; @@ -113,6 +115,7 @@ private: uint64_t _history[TIMER_TRAILING_SECONDS] {}; int _index { 0 }; }; + Timer _ticTiming; Timer _sleepTiming; Timer _frameTiming; Timer _prepareTiming; @@ -122,9 +125,6 @@ private: static int _numStaticJitterFrames; // -1 denotes dynamic jitter buffering static float _noiseMutingThreshold; static float _attenuationPerDoublingInDistance; - static float _trailingSleepRatio; - static float _performanceThrottlingRatio; - static float _minAudibilityThreshold; static QHash _audioZones; static QVector _zoneSettings; static QVector _zoneReverbSettings; diff --git a/assignment-client/src/audio/AudioMixerSlave.cpp b/assignment-client/src/audio/AudioMixerSlave.cpp index 28d3358eb5..4b02ca1567 100644 --- a/assignment-client/src/audio/AudioMixerSlave.cpp +++ b/assignment-client/src/audio/AudioMixerSlave.cpp @@ -36,6 +36,292 @@ #include "AudioMixerSlave.h" +using AudioStreamMap = AudioMixerClientData::AudioStreamMap; + +// packet helpers +std::unique_ptr createAudioPacket(PacketType type, int size, quint16 sequence, QString codec); +void sendMixPacket(const SharedNodePointer& node, AudioMixerClientData& data, QByteArray& buffer); +void sendSilentPacket(const SharedNodePointer& node, AudioMixerClientData& data); +void sendMutePacket(const SharedNodePointer& node, AudioMixerClientData&); +void sendEnvironmentPacket(const SharedNodePointer& node, AudioMixerClientData& data); + +// mix helpers +bool shouldIgnoreNode(const SharedNodePointer& listener, const SharedNodePointer& node); +float gainForSource(const AvatarAudioStream& listeningNodeStream, const PositionalAudioStream& streamToAdd, + const glm::vec3& relativePosition, bool isEcho); +float azimuthForSource(const AvatarAudioStream& listeningNodeStream, const PositionalAudioStream& streamToAdd, + const glm::vec3& relativePosition); + +void AudioMixerSlave::configure(ConstIter begin, ConstIter end, unsigned int frame, float throttlingRatio) { + _begin = begin; + _end = end; + _frame = frame; + _throttlingRatio = throttlingRatio; +} + +void AudioMixerSlave::mix(const SharedNodePointer& node) { + // check that the node is valid + AudioMixerClientData* data = (AudioMixerClientData*)node->getLinkedData(); + if (data == nullptr) { + return; + } + + // check that the stream is valid + auto avatarStream = data->getAvatarAudioStream(); + if (avatarStream == nullptr) { + return; + } + + // send mute packet, if necessary + if (AudioMixer::shouldMute(avatarStream->getQuietestFrameLoudness()) || data->shouldMuteClient()) { + sendMutePacket(node, *data); + } + + // send audio packets, if necessary + if (node->getType() == NodeType::Agent && node->getActiveSocket()) { + ++stats.sumListeners; + + // mix the audio + bool mixHasAudio = prepareMix(node); + + // send audio packet + if (mixHasAudio || data->shouldFlushEncoder()) { + QByteArray encodedBuffer; + if (mixHasAudio) { + // encode the audio + QByteArray decodedBuffer(reinterpret_cast(_bufferSamples), AudioConstants::NETWORK_FRAME_BYTES_STEREO); + data->encode(decodedBuffer, encodedBuffer); + } else { + // time to flush (resets shouldFlush until the next encode) + data->encodeFrameOfZeros(encodedBuffer); + } + + sendMixPacket(node, *data, encodedBuffer); + } else { + sendSilentPacket(node, *data); + } + + // send environment packet + sendEnvironmentPacket(node, *data); + + // send stats packet (about every second) + const unsigned int NUM_FRAMES_PER_SEC = (int)ceil(AudioConstants::NETWORK_FRAMES_PER_SEC); + if (data->shouldSendStats(_frame % NUM_FRAMES_PER_SEC)) { + data->sendAudioStreamStatsPackets(node); + } + } +} + +bool AudioMixerSlave::prepareMix(const SharedNodePointer& listener) { + AvatarAudioStream* listenerAudioStream = static_cast(listener->getLinkedData())->getAvatarAudioStream(); + AudioMixerClientData* listenerData = static_cast(listener->getLinkedData()); + + // zero out the mix for this listener + memset(_mixSamples, 0, sizeof(_mixSamples)); + + bool isThrottling = _throttlingRatio > 0.0f; + std::vector> throttledNodes; + + typedef void (AudioMixerSlave::*MixFunctor)( + AudioMixerClientData&, const QUuid&, const AvatarAudioStream&, const PositionalAudioStream&); + auto allStreams = [&](const SharedNodePointer& node, MixFunctor mixFunctor) { + AudioMixerClientData* nodeData = static_cast(node->getLinkedData()); + for (auto& streamPair : nodeData->getAudioStreams()) { + auto nodeStream = streamPair.second; + (this->*mixFunctor)(*listenerData, node->getUUID(), *listenerAudioStream, *nodeStream); + } + }; + + std::for_each(_begin, _end, [&](const SharedNodePointer& node) { + if (*node == *listener) { + AudioMixerClientData* nodeData = static_cast(node->getLinkedData()); + + // only mix the echo, if requested + for (auto& streamPair : nodeData->getAudioStreams()) { + auto nodeStream = streamPair.second; + if (nodeStream->shouldLoopbackForNode()) { + mixStream(*listenerData, node->getUUID(), *listenerAudioStream, *nodeStream); + } + } + } else if (!shouldIgnoreNode(listener, node)) { + if (!isThrottling) { + allStreams(node, &AudioMixerSlave::mixStream); + } else { + AudioMixerClientData* nodeData = static_cast(node->getLinkedData()); + + // compute the node's max relative volume + float nodeVolume; + for (auto& streamPair : nodeData->getAudioStreams()) { + auto nodeStream = streamPair.second; + float distance = glm::length(nodeStream->getPosition() - listenerAudioStream->getPosition()); + nodeVolume = std::max(nodeStream->getLastPopOutputTrailingLoudness() / distance, nodeVolume); + } + + // max-heapify the nodes by relative volume + throttledNodes.push_back(std::make_pair(nodeVolume, node)); + if (!throttledNodes.empty()) { + std::push_heap(throttledNodes.begin(), throttledNodes.end()); + } + } + } + }); + + if (isThrottling) { + // pop the loudest nodes off the heap and mix their streams + int numToRetain = (int)(std::distance(_begin, _end) * (1 - _throttlingRatio)); + for (int i = 0; i < numToRetain; i++) { + if (throttledNodes.empty()) { + break; + } + + std::pop_heap(throttledNodes.begin(), throttledNodes.end()); + + auto& node = throttledNodes.back().second; + allStreams(node, &AudioMixerSlave::mixStream); + + throttledNodes.pop_back(); + } + + // throttle the remaining nodes' streams + for (const std::pair& nodePair : throttledNodes) { + auto& node = nodePair.second; + allStreams(node, &AudioMixerSlave::throttleStream); + } + } + + // use the per listener AudioLimiter to render the mixed data... + listenerData->audioLimiter.render(_mixSamples, _bufferSamples, AudioConstants::NETWORK_FRAME_SAMPLES_PER_CHANNEL); + + // check for silent audio after the peak limiter has converted the samples + bool hasAudio = false; + for (int i = 0; i < AudioConstants::NETWORK_FRAME_SAMPLES_STEREO; ++i) { + if (_bufferSamples[i] != 0) { + hasAudio = true; + break; + } + } + return hasAudio; +} + +void AudioMixerSlave::throttleStream(AudioMixerClientData& listenerNodeData, const QUuid& sourceNodeID, + const AvatarAudioStream& listeningNodeStream, const PositionalAudioStream& streamToAdd) { + addStream(listenerNodeData, sourceNodeID, listeningNodeStream, streamToAdd, true); +} + +void AudioMixerSlave::mixStream(AudioMixerClientData& listenerNodeData, const QUuid& sourceNodeID, + const AvatarAudioStream& listeningNodeStream, const PositionalAudioStream& streamToAdd) { + addStream(listenerNodeData, sourceNodeID, listeningNodeStream, streamToAdd, false); +} + +void AudioMixerSlave::addStream(AudioMixerClientData& listenerNodeData, const QUuid& sourceNodeID, + const AvatarAudioStream& listeningNodeStream, const PositionalAudioStream& streamToAdd, + bool throttle) { + ++stats.totalMixes; + + // to reduce artifacts we call the HRTF functor for every source, even if throttled or silent + // this ensures the correct tail from last mixed block and the correct spatialization of next first block + + // check if this is a server echo of a source back to itself + bool isEcho = (&streamToAdd == &listeningNodeStream); + + glm::vec3 relativePosition = streamToAdd.getPosition() - listeningNodeStream.getPosition(); + + float distance = glm::max(glm::length(relativePosition), EPSILON); + float gain = gainForSource(listeningNodeStream, streamToAdd, relativePosition, isEcho); + float azimuth = isEcho ? 0.0f : azimuthForSource(listeningNodeStream, listeningNodeStream, relativePosition); + static const int HRTF_DATASET_INDEX = 1; + + if (!streamToAdd.lastPopSucceeded()) { + bool forceSilentBlock = true; + + if (!streamToAdd.getLastPopOutput().isNull()) { + bool isInjector = dynamic_cast(&streamToAdd); + + // in an injector, just go silent - the injector has likely ended + // in other inputs (microphone, &c.), repeat with fade to avoid the harsh jump to silence + if (!isInjector) { + // calculate its fade factor, which depends on how many times it's already been repeated. + float fadeFactor = calculateRepeatedFrameFadeFactor(streamToAdd.getConsecutiveNotMixedCount() - 1); + if (fadeFactor > 0.0f) { + // apply the fadeFactor to the gain + gain *= fadeFactor; + forceSilentBlock = false; + } + } + } + + if (forceSilentBlock) { + // call renderSilent with a forced silent block to reduce artifacts + // (this is not done for stereo streams since they do not go through the HRTF) + if (!streamToAdd.isStereo() && !isEcho) { + // get the existing listener-source HRTF object, or create a new one + auto& hrtf = listenerNodeData.hrtfForStream(sourceNodeID, streamToAdd.getStreamIdentifier()); + + static int16_t silentMonoBlock[AudioConstants::NETWORK_FRAME_SAMPLES_PER_CHANNEL] = {}; + hrtf.renderSilent(silentMonoBlock, _mixSamples, HRTF_DATASET_INDEX, azimuth, distance, gain, + AudioConstants::NETWORK_FRAME_SAMPLES_PER_CHANNEL); + + ++stats.hrtfSilentRenders; + } + + return; + } + } + + // grab the stream from the ring buffer + AudioRingBuffer::ConstIterator streamPopOutput = streamToAdd.getLastPopOutput(); + + // stereo sources are not passed through HRTF + if (streamToAdd.isStereo()) { + for (int i = 0; i < AudioConstants::NETWORK_FRAME_SAMPLES_STEREO; ++i) { + _mixSamples[i] += float(streamPopOutput[i] * gain / AudioConstants::MAX_SAMPLE_VALUE); + } + + ++stats.manualStereoMixes; + return; + } + + // echo sources are not passed through HRTF + if (isEcho) { + for (int i = 0; i < AudioConstants::NETWORK_FRAME_SAMPLES_STEREO; i += 2) { + auto monoSample = float(streamPopOutput[i / 2] * gain / AudioConstants::MAX_SAMPLE_VALUE); + _mixSamples[i] += monoSample; + _mixSamples[i + 1] += monoSample; + } + + ++stats.manualEchoMixes; + return; + } + + // get the existing listener-source HRTF object, or create a new one + auto& hrtf = listenerNodeData.hrtfForStream(sourceNodeID, streamToAdd.getStreamIdentifier()); + + streamPopOutput.readSamples(_bufferSamples, AudioConstants::NETWORK_FRAME_SAMPLES_PER_CHANNEL); + + if (streamToAdd.getLastPopOutputLoudness() == 0.0f) { + // call renderSilent to reduce artifacts + hrtf.renderSilent(_bufferSamples, _mixSamples, HRTF_DATASET_INDEX, azimuth, distance, gain, + AudioConstants::NETWORK_FRAME_SAMPLES_PER_CHANNEL); + + ++stats.hrtfSilentRenders; + return; + } + + if (throttle) { + // call renderSilent with actual frame data and a gain of 0.0f to reduce artifacts + hrtf.renderSilent(_bufferSamples, _mixSamples, HRTF_DATASET_INDEX, azimuth, distance, 0.0f, + AudioConstants::NETWORK_FRAME_SAMPLES_PER_CHANNEL); + + ++stats.hrtfThrottleRenders; + return; + } + + hrtf.render(_bufferSamples, _mixSamples, HRTF_DATASET_INDEX, azimuth, distance, gain, + AudioConstants::NETWORK_FRAME_SAMPLES_PER_CHANNEL); + + ++stats.hrtfRenders; +} + std::unique_ptr createAudioPacket(PacketType type, int size, quint16 sequence, QString codec) { auto audioPacket = NLPacket::create(type, size); audioPacket->writePrimitive(sequence); @@ -73,6 +359,14 @@ void sendSilentPacket(const SharedNodePointer& node, AudioMixerClientData& data) data.incrementOutgoingMixedAudioSequenceNumber(); } +void sendMutePacket(const SharedNodePointer& node, AudioMixerClientData& data) { + auto mutePacket = NLPacket::create(PacketType::NoisyMute, 0); + DependencyManager::get()->sendPacket(std::move(mutePacket), *node); + + // probably now we just reset the flag, once should do it (?) + data.setShouldMuteClient(false); +} + void sendEnvironmentPacket(const SharedNodePointer& node, AudioMixerClientData& data) { bool hasReverb = false; float reverbTime, wetLevel; @@ -134,285 +428,54 @@ void sendEnvironmentPacket(const SharedNodePointer& node, AudioMixerClientData& } } -void AudioMixerSlave::configure(ConstIter begin, ConstIter end, unsigned int frame) { - _begin = begin; - _end = end; - _frame = frame; -} - -void AudioMixerSlave::mix(const SharedNodePointer& node) { - // check that the node is valid - AudioMixerClientData* data = (AudioMixerClientData*)node->getLinkedData(); - if (data == nullptr) { - return; - } - - auto avatarStream = data->getAvatarAudioStream(); - if (avatarStream == nullptr) { - return; - } - - // send mute packet, if necessary - if (AudioMixer::shouldMute(avatarStream->getQuietestFrameLoudness()) || data->shouldMuteClient()) { - auto mutePacket = NLPacket::create(PacketType::NoisyMute, 0); - DependencyManager::get()->sendPacket(std::move(mutePacket), *node); - - // probably now we just reset the flag, once should do it (?) - data->setShouldMuteClient(false); - } - - // send audio packets, if necessary - if (node->getType() == NodeType::Agent && node->getActiveSocket()) { - ++stats.sumListeners; - - // mix the audio - bool mixHasAudio = prepareMix(node); - - // send audio packet - if (mixHasAudio || data->shouldFlushEncoder()) { - // encode the audio - QByteArray encodedBuffer; - if (mixHasAudio) { - QByteArray decodedBuffer(reinterpret_cast(_bufferSamples), AudioConstants::NETWORK_FRAME_BYTES_STEREO); - data->encode(decodedBuffer, encodedBuffer); - } else { - // time to flush, which resets the shouldFlush until next time we encode something - data->encodeFrameOfZeros(encodedBuffer); - } - - sendMixPacket(node, *data, encodedBuffer); - } else { - sendSilentPacket(node, *data); - } - - // send environment packet - sendEnvironmentPacket(node, *data); - - // send stats packet (about every second) - static const unsigned int NUM_FRAMES_PER_SEC = (int) ceil(AudioConstants::NETWORK_FRAMES_PER_SEC); - if (data->shouldSendStats(_frame % NUM_FRAMES_PER_SEC)) { - data->sendAudioStreamStatsPackets(node); - } - } -} - -bool AudioMixerSlave::prepareMix(const SharedNodePointer& node) { - AvatarAudioStream* nodeAudioStream = static_cast(node->getLinkedData())->getAvatarAudioStream(); +bool shouldIgnoreNode(const SharedNodePointer& listener, const SharedNodePointer& node) { + AudioMixerClientData* listenerData = static_cast(listener->getLinkedData()); AudioMixerClientData* nodeData = static_cast(node->getLinkedData()); - // zero out the client mix for this node - memset(_mixSamples, 0, sizeof(_mixSamples)); + // when this is true, the AudioMixer will send Audio data to a client about avatars that have ignored them + bool getsAnyIgnored = listenerData->getRequestsDomainListData() && listener->getCanKick(); - // loop through all other nodes that have sufficient audio to mix - std::for_each(_begin, _end, [&](const SharedNodePointer& otherNode){ - // make sure that we have audio data for this other node - // and that it isn't being ignored by our listening node - // and that it isn't ignoring our listening node - AudioMixerClientData* otherData = static_cast(otherNode->getLinkedData()); + bool ignore = true; - // When this is true, the AudioMixer will send Audio data to a client about avatars that have ignored them - bool getsAnyIgnored = nodeData->getRequestsDomainListData() && node->getCanKick(); + if (nodeData && + // make sure that it isn't being ignored by our listening node + (!listener->isIgnoringNodeWithID(node->getUUID()) || (nodeData->getRequestsDomainListData() && node->getCanKick())) && + // and that it isn't ignoring our listening node + (!node->isIgnoringNodeWithID(listener->getUUID()) || getsAnyIgnored)) { - if (otherData - && (!node->isIgnoringNodeWithID(otherNode->getUUID()) || (otherData->getRequestsDomainListData() && otherNode->getCanKick())) - && (!otherNode->isIgnoringNodeWithID(node->getUUID()) || getsAnyIgnored)) { + // is either node enabling the space bubble / ignore radius? + if ((listener->isIgnoreRadiusEnabled() || node->isIgnoreRadiusEnabled())) { + // define the minimum bubble size + static const glm::vec3 minBubbleSize = glm::vec3(0.3f, 1.3f, 0.3f); - // check to see if we're ignoring in radius - bool insideIgnoreRadius = false; - // If the otherNode equals the node, we're doing a comparison on ourselves - if (*otherNode == *node) { - // We'll always be inside the radius in that case. - insideIgnoreRadius = true; - // Check to see if the space bubble is enabled - } else if ((node->isIgnoreRadiusEnabled() || otherNode->isIgnoreRadiusEnabled())) { - // Define the minimum bubble size - static const glm::vec3 minBubbleSize = glm::vec3(0.3f, 1.3f, 0.3f); - AudioMixerClientData* nodeData = reinterpret_cast(node->getLinkedData()); - // Set up the bounding box for the current node - AABox nodeBox(nodeData->getAvatarBoundingBoxCorner(), nodeData->getAvatarBoundingBoxScale()); - // Clamp the size of the bounding box to a minimum scale - if (glm::any(glm::lessThan(nodeData->getAvatarBoundingBoxScale(), minBubbleSize))) { - nodeBox.setScaleStayCentered(minBubbleSize); - } - // Set up the bounding box for the current other node - AABox otherNodeBox(otherData->getAvatarBoundingBoxCorner(), otherData->getAvatarBoundingBoxScale()); - // Clamp the size of the bounding box to a minimum scale - if (glm::any(glm::lessThan(otherData->getAvatarBoundingBoxScale(), minBubbleSize))) { - otherNodeBox.setScaleStayCentered(minBubbleSize); - } - // Quadruple the scale of both bounding boxes - nodeBox.embiggen(4.0f); - otherNodeBox.embiggen(4.0f); - - // Perform the collision check between the two bounding boxes - if (nodeBox.touches(otherNodeBox)) { - insideIgnoreRadius = true; - } + // set up the bounding box for the listener + AABox listenerBox(listenerData->getAvatarBoundingBoxCorner(), listenerData->getAvatarBoundingBoxScale()); + if (glm::any(glm::lessThan(listenerData->getAvatarBoundingBoxScale(), minBubbleSize))) { + listenerBox.setScaleStayCentered(minBubbleSize); } - // Enumerate the audio streams attached to the otherNode - auto streamsCopy = otherData->getAudioStreams(); - for (auto& streamPair : streamsCopy) { - auto otherNodeStream = streamPair.second; - bool isSelfWithEcho = ((*otherNode == *node) && (otherNodeStream->shouldLoopbackForNode())); - // Add all audio streams that should be added to the mix - if (isSelfWithEcho || (!isSelfWithEcho && !insideIgnoreRadius)) { - addStreamToMix(*nodeData, otherNode->getUUID(), *nodeAudioStream, *otherNodeStream); - } - } - } - }); - - // use the per listener AudioLimiter to render the mixed data... - nodeData->audioLimiter.render(_mixSamples, _bufferSamples, AudioConstants::NETWORK_FRAME_SAMPLES_PER_CHANNEL); - - // check for silent audio after the peak limiter has converted the samples - bool hasAudio = false; - for (int i = 0; i < AudioConstants::NETWORK_FRAME_SAMPLES_STEREO; ++i) { - if (_bufferSamples[i] != 0) { - hasAudio = true; - break; - } - } - return hasAudio; -} - -void AudioMixerSlave::addStreamToMix(AudioMixerClientData& listenerNodeData, const QUuid& sourceNodeID, - const AvatarAudioStream& listeningNodeStream, const PositionalAudioStream& streamToAdd) { - // to reduce artifacts we calculate the gain and azimuth for every source for this listener - // even if we are not going to end up mixing in this source - - ++stats.totalMixes; - - // this ensures that the tail of any previously mixed audio or the first block of new audio sounds correct - - // check if this is a server echo of a source back to itself - bool isEcho = (&streamToAdd == &listeningNodeStream); - - glm::vec3 relativePosition = streamToAdd.getPosition() - listeningNodeStream.getPosition(); - - // figure out the distance between source and listener - float distance = glm::max(glm::length(relativePosition), EPSILON); - - // figure out the gain for this source at the listener - float gain = gainForSource(listeningNodeStream, streamToAdd, relativePosition, isEcho); - - // figure out the azimuth to this source at the listener - float azimuth = isEcho ? 0.0f : azimuthForSource(listeningNodeStream, listeningNodeStream, relativePosition); - - float repeatedFrameFadeFactor = 1.0f; - - static const int HRTF_DATASET_INDEX = 1; - - if (!streamToAdd.lastPopSucceeded()) { - bool forceSilentBlock = true; - - if (!streamToAdd.getLastPopOutput().isNull()) { - bool isInjector = dynamic_cast(&streamToAdd); - - // in an injector, just go silent - the injector has likely ended - // in other inputs (microphone, &c.), repeat with fade to avoid the harsh jump to silence - - // we'll repeat the last block until it has a block to mix - // and we'll gradually fade that repeated block into silence. - - // calculate its fade factor, which depends on how many times it's already been repeated. - repeatedFrameFadeFactor = calculateRepeatedFrameFadeFactor(streamToAdd.getConsecutiveNotMixedCount() - 1); - if (!isInjector && repeatedFrameFadeFactor > 0.0f) { - // apply the repeatedFrameFadeFactor to the gain - gain *= repeatedFrameFadeFactor; - - forceSilentBlock = false; - } - } - - if (forceSilentBlock) { - // we're deciding not to repeat either since we've already done it enough times or repetition with fade is disabled - // in this case we will call renderSilent with a forced silent block - // this ensures the correct tail from the previously mixed block and the correct spatialization of first block - // of any upcoming audio - - if (!streamToAdd.isStereo() && !isEcho) { - // get the existing listener-source HRTF object, or create a new one - auto& hrtf = listenerNodeData.hrtfForStream(sourceNodeID, streamToAdd.getStreamIdentifier()); - - // this is not done for stereo streams since they do not go through the HRTF - static int16_t silentMonoBlock[AudioConstants::NETWORK_FRAME_SAMPLES_PER_CHANNEL] = {}; - hrtf.renderSilent(silentMonoBlock, _mixSamples, HRTF_DATASET_INDEX, azimuth, distance, gain, - AudioConstants::NETWORK_FRAME_SAMPLES_PER_CHANNEL); - - ++stats.hrtfSilentRenders; + // set up the bounding box for the node + AABox nodeBox(nodeData->getAvatarBoundingBoxCorner(), nodeData->getAvatarBoundingBoxScale()); + // Clamp the size of the bounding box to a minimum scale + if (glm::any(glm::lessThan(nodeData->getAvatarBoundingBoxScale(), minBubbleSize))) { + nodeBox.setScaleStayCentered(minBubbleSize); } - return; - } - } + // quadruple the scale of both bounding boxes + listenerBox.embiggen(4.0f); + nodeBox.embiggen(4.0f); - // grab the stream from the ring buffer - AudioRingBuffer::ConstIterator streamPopOutput = streamToAdd.getLastPopOutput(); - - if (streamToAdd.isStereo() || isEcho) { - // this is a stereo source or server echo so we do not pass it through the HRTF - // simply apply our calculated gain to each sample - if (streamToAdd.isStereo()) { - for (int i = 0; i < AudioConstants::NETWORK_FRAME_SAMPLES_STEREO; ++i) { - _mixSamples[i] += float(streamPopOutput[i] * gain / AudioConstants::MAX_SAMPLE_VALUE); - } - - ++stats.manualStereoMixes; + // perform the collision check between the two bounding boxes + ignore = listenerBox.touches(nodeBox); } else { - for (int i = 0; i < AudioConstants::NETWORK_FRAME_SAMPLES_STEREO; i += 2) { - auto monoSample = float(streamPopOutput[i / 2] * gain / AudioConstants::MAX_SAMPLE_VALUE); - _mixSamples[i] += monoSample; - _mixSamples[i + 1] += monoSample; - } - - ++stats.manualEchoMixes; + ignore = false; } - - return; } - // get the existing listener-source HRTF object, or create a new one - auto& hrtf = listenerNodeData.hrtfForStream(sourceNodeID, streamToAdd.getStreamIdentifier()); - - streamPopOutput.readSamples(_bufferSamples, AudioConstants::NETWORK_FRAME_SAMPLES_PER_CHANNEL); - - // if the frame we're about to mix is silent, simply call render silent and move on - if (streamToAdd.getLastPopOutputLoudness() == 0.0f) { - // silent frame from source - - // we still need to call renderSilent via the HRTF for mono source - hrtf.renderSilent(_bufferSamples, _mixSamples, HRTF_DATASET_INDEX, azimuth, distance, gain, - AudioConstants::NETWORK_FRAME_SAMPLES_PER_CHANNEL); - - ++stats.hrtfSilentRenders; - - return; - } - - float audibilityThreshold = AudioMixer::getMinimumAudibilityThreshold(); - if (audibilityThreshold > 0.0f && - streamToAdd.getLastPopOutputTrailingLoudness() / glm::length(relativePosition) <= audibilityThreshold) { - // the mixer is struggling so we're going to drop off some streams - - // we call renderSilent via the HRTF with the actual frame data and a gain of 0.0 - hrtf.renderSilent(_bufferSamples, _mixSamples, HRTF_DATASET_INDEX, azimuth, distance, 0.0f, - AudioConstants::NETWORK_FRAME_SAMPLES_PER_CHANNEL); - - ++stats.hrtfStruggleRenders; - - return; - } - - ++stats.hrtfRenders; - - // mono stream, call the HRTF with our block and calculated azimuth and gain - hrtf.render(_bufferSamples, _mixSamples, HRTF_DATASET_INDEX, azimuth, distance, gain, - AudioConstants::NETWORK_FRAME_SAMPLES_PER_CHANNEL); + return ignore; } -float AudioMixerSlave::gainForSource(const AvatarAudioStream& listeningNodeStream, const PositionalAudioStream& streamToAdd, +float gainForSource(const AvatarAudioStream& listeningNodeStream, const PositionalAudioStream& streamToAdd, const glm::vec3& relativePosition, bool isEcho) { float gain = 1.0f; @@ -472,7 +535,7 @@ float AudioMixerSlave::gainForSource(const AvatarAudioStream& listeningNodeStrea return gain; } -float AudioMixerSlave::azimuthForSource(const AvatarAudioStream& listeningNodeStream, const PositionalAudioStream& streamToAdd, +float azimuthForSource(const AvatarAudioStream& listeningNodeStream, const PositionalAudioStream& streamToAdd, const glm::vec3& relativePosition) { glm::quat inverseOrientation = glm::inverse(listeningNodeStream.getOrientation()); @@ -482,7 +545,7 @@ float AudioMixerSlave::azimuthForSource(const AvatarAudioStream& listeningNodeSt // project the rotated source position vector onto the XZ plane rotatedSourcePosition.y = 0.0f; - static const float SOURCE_DISTANCE_THRESHOLD = 1e-30f; + const float SOURCE_DISTANCE_THRESHOLD = 1e-30f; if (glm::length2(rotatedSourcePosition) > SOURCE_DISTANCE_THRESHOLD) { // produce an oriented angle about the y-axis diff --git a/assignment-client/src/audio/AudioMixerSlave.h b/assignment-client/src/audio/AudioMixerSlave.h index c4aabfbb4a..7b59500629 100644 --- a/assignment-client/src/audio/AudioMixerSlave.h +++ b/assignment-client/src/audio/AudioMixerSlave.h @@ -30,7 +30,7 @@ class AudioMixerSlave { public: using ConstIter = NodeList::const_iterator; - void configure(ConstIter begin, ConstIter end, unsigned int frame); + void configure(ConstIter begin, ConstIter end, unsigned int frame, float throttlingRatio); // mix and broadcast non-ignored streams to the node // returns true if a mixed packet was sent to the node @@ -40,15 +40,14 @@ public: private: // create mix, returns true if mix has audio - bool prepareMix(const SharedNodePointer& node); - // add a stream to the mix - void addStreamToMix(AudioMixerClientData& listenerData, const QUuid& streamerID, + bool prepareMix(const SharedNodePointer& listener); + void throttleStream(AudioMixerClientData& listenerData, const QUuid& streamerID, const AvatarAudioStream& listenerStream, const PositionalAudioStream& streamer); - - float gainForSource(const AvatarAudioStream& listener, const PositionalAudioStream& streamer, - const glm::vec3& relativePosition, bool isEcho); - float azimuthForSource(const AvatarAudioStream& listener, const PositionalAudioStream& streamer, - const glm::vec3& relativePosition); + void mixStream(AudioMixerClientData& listenerData, const QUuid& streamerID, + const AvatarAudioStream& listenerStream, const PositionalAudioStream& streamer); + void addStream(AudioMixerClientData& listenerData, const QUuid& streamerID, + const AvatarAudioStream& listenerStream, const PositionalAudioStream& streamer, + bool throttle); // mixing buffers float _mixSamples[AudioConstants::NETWORK_FRAME_SAMPLES_STEREO]; @@ -58,6 +57,7 @@ private: ConstIter _begin; ConstIter _end; unsigned int _frame { 0 }; + float _throttlingRatio { 0.0f }; }; #endif // hifi_AudioMixerSlave_h diff --git a/assignment-client/src/audio/AudioMixerSlavePool.cpp b/assignment-client/src/audio/AudioMixerSlavePool.cpp index 1b884fa089..9b20572b84 100644 --- a/assignment-client/src/audio/AudioMixerSlavePool.cpp +++ b/assignment-client/src/audio/AudioMixerSlavePool.cpp @@ -41,7 +41,7 @@ void AudioMixerSlaveThread::wait() { }); ++_pool._numStarted; } - configure(_pool._begin, _pool._end, _pool._frame); + configure(_pool._begin, _pool._end, _pool._frame, _pool._throttlingRatio); } void AudioMixerSlaveThread::notify(bool stopping) { @@ -64,13 +64,14 @@ bool AudioMixerSlaveThread::try_pop(SharedNodePointer& node) { static AudioMixerSlave slave; #endif -void AudioMixerSlavePool::mix(ConstIter begin, ConstIter end, unsigned int frame) { +void AudioMixerSlavePool::mix(ConstIter begin, ConstIter end, unsigned int frame, float throttlingRatio) { _begin = begin; _end = end; _frame = frame; + _throttlingRatio = throttlingRatio; #ifdef AUDIO_SINGLE_THREADED - slave.configure(_begin, _end, frame); + slave.configure(_begin, _end, frame, throttlingRatio); std::for_each(begin, end, [&](const SharedNodePointer& node) { slave.mix(node); }); diff --git a/assignment-client/src/audio/AudioMixerSlavePool.h b/assignment-client/src/audio/AudioMixerSlavePool.h index e8781950f3..19d2315d12 100644 --- a/assignment-client/src/audio/AudioMixerSlavePool.h +++ b/assignment-client/src/audio/AudioMixerSlavePool.h @@ -61,7 +61,7 @@ public: ~AudioMixerSlavePool() { resize(0); } // mix on slave threads - void mix(ConstIter begin, ConstIter end, unsigned int frame); + void mix(ConstIter begin, ConstIter end, unsigned int frame, float throttlingRatio); // iterate over all slaves void each(std::function functor); @@ -90,6 +90,7 @@ private: // frame state Queue _queue; unsigned int _frame { 0 }; + float _throttlingRatio { 0.0f }; ConstIter _begin; ConstIter _end; }; diff --git a/assignment-client/src/audio/AudioMixerStats.cpp b/assignment-client/src/audio/AudioMixerStats.cpp index 94115ad5ff..a50c0d26c1 100644 --- a/assignment-client/src/audio/AudioMixerStats.cpp +++ b/assignment-client/src/audio/AudioMixerStats.cpp @@ -17,7 +17,7 @@ void AudioMixerStats::reset() { totalMixes = 0; hrtfRenders = 0; hrtfSilentRenders = 0; - hrtfStruggleRenders = 0; + hrtfThrottleRenders = 0; manualStereoMixes = 0; manualEchoMixes = 0; } @@ -28,7 +28,7 @@ void AudioMixerStats::accumulate(const AudioMixerStats& otherStats) { totalMixes += otherStats.totalMixes; hrtfRenders += otherStats.hrtfRenders; hrtfSilentRenders += otherStats.hrtfSilentRenders; - hrtfStruggleRenders += otherStats.hrtfStruggleRenders; + hrtfThrottleRenders += otherStats.hrtfThrottleRenders; manualStereoMixes += otherStats.manualStereoMixes; manualEchoMixes += otherStats.manualEchoMixes; } diff --git a/assignment-client/src/audio/AudioMixerStats.h b/assignment-client/src/audio/AudioMixerStats.h index 5aefe611f0..cb85006061 100644 --- a/assignment-client/src/audio/AudioMixerStats.h +++ b/assignment-client/src/audio/AudioMixerStats.h @@ -20,7 +20,7 @@ struct AudioMixerStats { int hrtfRenders { 0 }; int hrtfSilentRenders { 0 }; - int hrtfStruggleRenders { 0 }; + int hrtfThrottleRenders { 0 }; int manualStereoMixes { 0 }; int manualEchoMixes { 0 };