From aed79b3b177a59d60ecdf52c1913a07a3146eb52 Mon Sep 17 00:00:00 2001 From: Stephen Birarda Date: Fri, 3 Aug 2018 10:38:11 -0700 Subject: [PATCH 01/21] use a vector for ignored node IDs --- .../src/audio/AudioMixerClientData.cpp | 25 ------------------- .../src/audio/AudioMixerClientData.h | 21 +--------------- libraries/networking/src/Node.cpp | 25 +++++++++++++------ libraries/networking/src/Node.h | 5 ++-- 4 files changed, 22 insertions(+), 54 deletions(-) diff --git a/assignment-client/src/audio/AudioMixerClientData.cpp b/assignment-client/src/audio/AudioMixerClientData.cpp index 07cc5493b0..bbaf04c6a5 100644 --- a/assignment-client/src/audio/AudioMixerClientData.cpp +++ b/assignment-client/src/audio/AudioMixerClientData.cpp @@ -647,32 +647,9 @@ AudioMixerClientData::IgnoreZone& AudioMixerClientData::IgnoreZoneMemo::get(unsi return _zone; } -void AudioMixerClientData::IgnoreNodeCache::cache(bool shouldIgnore) { - if (!_isCached) { - _shouldIgnore = shouldIgnore; - _isCached = true; - } -} - -bool AudioMixerClientData::IgnoreNodeCache::isCached() { - return _isCached; -} - -bool AudioMixerClientData::IgnoreNodeCache::shouldIgnore() { - bool ignore = _shouldIgnore; - _isCached = false; - return ignore; -} - bool AudioMixerClientData::shouldIgnore(const SharedNodePointer self, const SharedNodePointer node, unsigned int frame) { // this is symmetric over self / node; if computed, it is cached in the other - // check the cache to avoid computation - auto& cache = _nodeSourcesIgnoreMap[node->getUUID()]; - if (cache.isCached()) { - return cache.shouldIgnore(); - } - AudioMixerClientData* nodeData = static_cast(node->getLinkedData()); if (!nodeData) { return false; @@ -696,8 +673,6 @@ bool AudioMixerClientData::shouldIgnore(const SharedNodePointer self, const Shar } } - // cache in node - nodeData->_nodeSourcesIgnoreMap[self->getUUID()].cache(shouldIgnore); return shouldIgnore; } diff --git a/assignment-client/src/audio/AudioMixerClientData.h b/assignment-client/src/audio/AudioMixerClientData.h index 82bdc0e5c5..7d77169eaf 100644 --- a/assignment-client/src/audio/AudioMixerClientData.h +++ b/assignment-client/src/audio/AudioMixerClientData.h @@ -57,7 +57,7 @@ public: void removeHRTFForStream(const QUuid& nodeID, const QUuid& streamID = QUuid()); // remove all sources and data from this node - void removeNode(const QUuid& nodeID) { _nodeSourcesIgnoreMap.unsafe_erase(nodeID); _nodeSourcesHRTFMap.erase(nodeID); } + void removeNode(const QUuid& nodeID) { _nodeSourcesHRTFMap.erase(nodeID); } void removeAgentAvatarAudioStream(); @@ -150,25 +150,6 @@ private: }; IgnoreZoneMemo _ignoreZone; - class IgnoreNodeCache { - public: - // std::atomic is not copyable - always initialize uncached - IgnoreNodeCache() {} - IgnoreNodeCache(const IgnoreNodeCache& other) {} - - void cache(bool shouldIgnore); - bool isCached(); - bool shouldIgnore(); - - private: - std::atomic _isCached { false }; - bool _shouldIgnore { false }; - }; - struct IgnoreNodeCacheHasher { std::size_t operator()(const QUuid& key) const { return qHash(key); } }; - - using NodeSourcesIgnoreMap = tbb::concurrent_unordered_map; - NodeSourcesIgnoreMap _nodeSourcesIgnoreMap; - using HRTFMap = std::unordered_map; using NodeSourcesHRTFMap = std::unordered_map; NodeSourcesHRTFMap _nodeSourcesHRTFMap; diff --git a/libraries/networking/src/Node.cpp b/libraries/networking/src/Node.cpp index 5bbff103dd..ec72ae9bf6 100644 --- a/libraries/networking/src/Node.cpp +++ b/libraries/networking/src/Node.cpp @@ -131,12 +131,14 @@ void Node::parseIgnoreRequestMessage(QSharedPointer message) { void Node::addIgnoredNode(const QUuid& otherNodeID) { if (!otherNodeID.isNull() && otherNodeID != _uuid) { - QReadLocker lock { &_ignoredNodeIDSetLock }; + QWriteLocker lock { &_ignoredNodeIDSetLock }; qCDebug(networking) << "Adding" << uuidStringWithoutCurlyBraces(otherNodeID) << "to ignore set for" - << uuidStringWithoutCurlyBraces(_uuid); + << uuidStringWithoutCurlyBraces(_uuid); // add the session UUID to the set of ignored ones for this listening node - _ignoredNodeIDSet.insert(otherNodeID); + if (std::find(_ignoredNodeIDs.begin(), _ignoredNodeIDs.end(), otherNodeID) == _ignoredNodeIDs.end()) { + _ignoredNodeIDs.push_back(otherNodeID); + } } else { qCWarning(networking) << "Node::addIgnoredNode called with null ID or ID of ignoring node."; } @@ -144,18 +146,27 @@ void Node::addIgnoredNode(const QUuid& otherNodeID) { void Node::removeIgnoredNode(const QUuid& otherNodeID) { if (!otherNodeID.isNull() && otherNodeID != _uuid) { - // insert/find are read locked concurrently. unsafe_erase is not concurrent, and needs a write lock. QWriteLocker lock { &_ignoredNodeIDSetLock }; qCDebug(networking) << "Removing" << uuidStringWithoutCurlyBraces(otherNodeID) << "from ignore set for" - << uuidStringWithoutCurlyBraces(_uuid); + << uuidStringWithoutCurlyBraces(_uuid); - // remove the session UUID from the set of ignored ones for this listening node - _ignoredNodeIDSet.unsafe_erase(otherNodeID); + // remove the session UUID from the set of ignored ones for this listening node, if it exists + auto it = std::remove(_ignoredNodeIDs.begin(), _ignoredNodeIDs.end(), otherNodeID); + if (it != _ignoredNodeIDs.end()) { + _ignoredNodeIDs.erase(it); + } } else { qCWarning(networking) << "Node::removeIgnoredNode called with null ID or ID of ignoring node."; } } +bool Node::isIgnoringNodeWithID(const QUuid& nodeID) const { + QReadLocker lock { &_ignoredNodeIDSetLock }; + + // check if this node ID is present in the ignore node ID set + return std::find(_ignoredNodeIDs.begin(), _ignoredNodeIDs.end(), nodeID) != _ignoredNodeIDs.end(); +} + void Node::parseIgnoreRadiusRequestMessage(QSharedPointer message) { bool enabled; message->readPrimitive(&enabled); diff --git a/libraries/networking/src/Node.h b/libraries/networking/src/Node.h index bcfe525aa8..72a32d9d18 100644 --- a/libraries/networking/src/Node.h +++ b/libraries/networking/src/Node.h @@ -15,6 +15,7 @@ #include #include #include +#include #include #include @@ -83,7 +84,7 @@ public: void parseIgnoreRequestMessage(QSharedPointer message); void addIgnoredNode(const QUuid& otherNodeID); void removeIgnoredNode(const QUuid& otherNodeID); - bool isIgnoringNodeWithID(const QUuid& nodeID) const { QReadLocker lock { &_ignoredNodeIDSetLock }; return _ignoredNodeIDSet.find(nodeID) != _ignoredNodeIDSet.cend(); } + bool isIgnoringNodeWithID(const QUuid& nodeID) const; void parseIgnoreRadiusRequestMessage(QSharedPointer message); friend QDataStream& operator<<(QDataStream& out, const Node& node); @@ -108,7 +109,7 @@ private: MovingPercentile _clockSkewMovingPercentile; NodePermissions _permissions; bool _isUpstream { false }; - tbb::concurrent_unordered_set _ignoredNodeIDSet; + std::vector _ignoredNodeIDs; mutable QReadWriteLock _ignoredNodeIDSetLock; std::vector _replicatedUsernames { }; From 09cfe4dbc1d65cc5234cd912ca47bedeac874ec4 Mon Sep 17 00:00:00 2001 From: Stephen Birarda Date: Fri, 3 Aug 2018 12:11:41 -0700 Subject: [PATCH 02/21] use a vector for audio streams --- .../src/audio/AudioMixerClientData.cpp | 58 ++++++++++--------- .../src/audio/AudioMixerClientData.h | 7 +-- .../src/audio/AudioMixerSlave.cpp | 13 ++--- 3 files changed, 38 insertions(+), 40 deletions(-) diff --git a/assignment-client/src/audio/AudioMixerClientData.cpp b/assignment-client/src/audio/AudioMixerClientData.cpp index bbaf04c6a5..a8398abcd3 100644 --- a/assignment-client/src/audio/AudioMixerClientData.cpp +++ b/assignment-client/src/audio/AudioMixerClientData.cpp @@ -213,9 +213,12 @@ void AudioMixerClientData::parseRadiusIgnoreRequest(QSharedPointergetStreamIdentifier().isNull(); + }); + if (it != _audioStreams.end()) { - return dynamic_cast(it->second.get()); + return dynamic_cast(it->get()); } // no mic stream found - return NULL @@ -238,11 +241,14 @@ void AudioMixerClientData::removeHRTFForStream(const QUuid& nodeID, const QUuid& void AudioMixerClientData::removeAgentAvatarAudioStream() { QWriteLocker writeLocker { &_streamsLock }; - auto it = _audioStreams.find(QUuid()); + + auto it = std::remove_if(_audioStreams.begin(), _audioStreams.end(), [](const SharedStreamPointer& stream){ + return stream->getStreamIdentifier().isNull(); + }); + if (it != _audioStreams.end()) { _audioStreams.erase(it); } - writeLocker.unlock(); } int AudioMixerClientData::parseData(ReceivedMessage& message) { @@ -271,7 +277,9 @@ int AudioMixerClientData::parseData(ReceivedMessage& message) { QWriteLocker writeLocker { &_streamsLock }; - auto micStreamIt = _audioStreams.find(QUuid()); + auto micStreamIt = std::find_if(_audioStreams.begin(), _audioStreams.end(), [](const SharedStreamPointer& stream){ + return stream->getStreamIdentifier().isNull(); + }); if (micStreamIt == _audioStreams.end()) { // we don't have a mic stream yet, so add it @@ -301,16 +309,12 @@ int AudioMixerClientData::parseData(ReceivedMessage& message) { connect(avatarAudioStream, &InboundAudioStream::mismatchedAudioCodec, this, &AudioMixerClientData::handleMismatchAudioFormat); - auto emplaced = _audioStreams.emplace( - QUuid(), - std::unique_ptr { avatarAudioStream } - ); - - micStreamIt = emplaced.first; + matchingStream = SharedStreamPointer(avatarAudioStream); + _audioStreams.push_back(matchingStream); + } else { + matchingStream = *micStreamIt; } - matchingStream = micStreamIt->second; - writeLocker.unlock(); isMicStream = true; @@ -327,7 +331,9 @@ int AudioMixerClientData::parseData(ReceivedMessage& message) { QWriteLocker writeLock { &_streamsLock }; - auto streamIt = _audioStreams.find(streamIdentifier); + auto streamIt = std::find_if(_audioStreams.begin(), _audioStreams.end(), [&streamIdentifier](const SharedStreamPointer& stream) { + return stream->getStreamIdentifier() == streamIdentifier; + }); if (streamIt == _audioStreams.end()) { // we don't have this injected stream yet, so add it @@ -338,16 +344,12 @@ int AudioMixerClientData::parseData(ReceivedMessage& message) { qCDebug(audio) << "creating new injectorStream... codec:" << _selectedCodecName << "isStereo:" << isStereo; #endif - auto emplaced = _audioStreams.emplace( - streamIdentifier, - std::unique_ptr { injectorStream } - ); - - streamIt = emplaced.first; + matchingStream = SharedStreamPointer(injectorStream); + _audioStreams.push_back(matchingStream); + } else { + matchingStream = *streamIt; } - matchingStream = streamIt->second; - writeLock.unlock(); } @@ -373,7 +375,7 @@ int AudioMixerClientData::checkBuffersBeforeFrameSend() { auto it = _audioStreams.begin(); while (it != _audioStreams.end()) { - SharedStreamPointer stream = it->second; + SharedStreamPointer stream = *it; if (stream->popFrames(1, true) > 0) { stream->updateLastPopOutputLoudnessAndTrailingLoudness(); @@ -388,7 +390,7 @@ int AudioMixerClientData::checkBuffersBeforeFrameSend() { // this is an inactive injector, pull it from our streams // first emit that it is finished so that the HRTF objects for this source can be cleaned up - emit injectorStreamFinished(it->second->getStreamIdentifier()); + emit injectorStreamFinished(stream->getStreamIdentifier()); // erase the stream to drop our ref to the shared pointer and remove it it = _audioStreams.erase(it); @@ -441,7 +443,7 @@ void AudioMixerClientData::sendAudioStreamStatsPackets(const SharedNodePointer& // pack the calculated number of stream stats for (int i = 0; i < numStreamStatsToPack; i++) { - PositionalAudioStream* stream = it->second.get(); + PositionalAudioStream* stream = it->get(); stream->perSecondCallbackForUpdatingStats(); @@ -513,12 +515,12 @@ QJsonObject AudioMixerClientData::getAudioStreamStats() { QJsonArray injectorArray; auto streamsCopy = getAudioStreams(); for (auto& injectorPair : streamsCopy) { - if (injectorPair.second->getType() == PositionalAudioStream::Injector) { + if (injectorPair->getType() == PositionalAudioStream::Injector) { QJsonObject upstreamStats; - AudioStreamStats streamStats = injectorPair.second->getAudioStreamStats(); + AudioStreamStats streamStats = injectorPair->getAudioStreamStats(); upstreamStats["inj.desired"] = streamStats._desiredJitterBufferFrames; - upstreamStats["desired_calc"] = injectorPair.second->getCalculatedJitterBufferFrames(); + upstreamStats["desired_calc"] = injectorPair->getCalculatedJitterBufferFrames(); upstreamStats["available_avg_10s"] = streamStats._framesAvailableAverage; upstreamStats["available"] = (double) streamStats._framesAvailable; upstreamStats["unplayed"] = (double) streamStats._unplayedMs; diff --git a/assignment-client/src/audio/AudioMixerClientData.h b/assignment-client/src/audio/AudioMixerClientData.h index 7d77169eaf..201ccfe867 100644 --- a/assignment-client/src/audio/AudioMixerClientData.h +++ b/assignment-client/src/audio/AudioMixerClientData.h @@ -34,13 +34,12 @@ public: ~AudioMixerClientData(); using SharedStreamPointer = std::shared_ptr; - using AudioStreamMap = std::unordered_map; + using AudioStreamVector = std::vector; void queuePacket(QSharedPointer packet, SharedNodePointer node); void processPackets(); - // locks the mutex to make a copy - AudioStreamMap getAudioStreams() { QReadLocker readLock { &_streamsLock }; return _audioStreams; } + AudioStreamVector& getAudioStreams() { return _audioStreams; } AvatarAudioStream* getAvatarAudioStream(); // returns whether self (this data's node) should ignore node, memoized by frame @@ -127,7 +126,7 @@ private: PacketQueue _packetQueue; QReadWriteLock _streamsLock; - AudioStreamMap _audioStreams; // microphone stream from avatar is stored under key of null UUID + AudioStreamVector _audioStreams; // microphone stream from avatar is stored under key of null UUID void optionallyReplicatePacket(ReceivedMessage& packet, const Node& node); diff --git a/assignment-client/src/audio/AudioMixerSlave.cpp b/assignment-client/src/audio/AudioMixerSlave.cpp index b447048ac9..4b97049758 100644 --- a/assignment-client/src/audio/AudioMixerSlave.cpp +++ b/assignment-client/src/audio/AudioMixerSlave.cpp @@ -36,7 +36,7 @@ #include "InjectedAudioStream.h" #include "AudioHelpers.h" -using AudioStreamMap = AudioMixerClientData::AudioStreamMap; +using AudioStreamVector = AudioMixerClientData::AudioStreamVector; // packet helpers std::unique_ptr createAudioPacket(PacketType type, int size, quint16 sequence, QString codec); @@ -146,8 +146,7 @@ bool AudioMixerSlave::prepareMix(const SharedNodePointer& listener) { auto forAllStreams = [&](const SharedNodePointer& node, AudioMixerClientData* nodeData, MixFunctor mixFunctor) { auto nodeID = node->getUUID(); for (auto& streamPair : nodeData->getAudioStreams()) { - auto nodeStream = streamPair.second; - (this->*mixFunctor)(*listenerData, nodeID, *listenerAudioStream, *nodeStream); + (this->*mixFunctor)(*listenerData, nodeID, *listenerAudioStream, *streamPair); } }; @@ -164,9 +163,8 @@ bool AudioMixerSlave::prepareMix(const SharedNodePointer& listener) { if (*node == *listener) { // only mix the echo, if requested for (auto& streamPair : nodeData->getAudioStreams()) { - auto nodeStream = streamPair.second; - if (nodeStream->shouldLoopbackForNode()) { - mixStream(*listenerData, node->getUUID(), *listenerAudioStream, *nodeStream); + if (streamPair->shouldLoopbackForNode()) { + mixStream(*listenerData, node->getUUID(), *listenerAudioStream, *streamPair); } } } else if (!listenerData->shouldIgnore(listener, node, _frame)) { @@ -177,8 +175,7 @@ bool AudioMixerSlave::prepareMix(const SharedNodePointer& listener) { // compute the node's max relative volume float nodeVolume = 0.0f; - for (auto& streamPair : nodeData->getAudioStreams()) { - auto nodeStream = streamPair.second; + for (auto& nodeStream : nodeData->getAudioStreams()) { // approximate the gain glm::vec3 relativePosition = nodeStream->getPosition() - listenerAudioStream->getPosition(); From d15ef295cff1eccf643c11c49f885486940829a2 Mon Sep 17 00:00:00 2001 From: Clement Date: Fri, 3 Aug 2018 12:50:22 -0700 Subject: [PATCH 03/21] Replace AudioZones lookup with vectors and indexes --- assignment-client/src/audio/AudioMixer.cpp | 91 +++++++++++-------- assignment-client/src/audio/AudioMixer.h | 25 +++-- .../src/audio/AudioMixerSlave.cpp | 16 ++-- 3 files changed, 78 insertions(+), 54 deletions(-) diff --git a/assignment-client/src/audio/AudioMixer.cpp b/assignment-client/src/audio/AudioMixer.cpp index 0d42cc83be..7b91916bbb 100644 --- a/assignment-client/src/audio/AudioMixer.cpp +++ b/assignment-client/src/audio/AudioMixer.cpp @@ -38,6 +38,8 @@ #include "AvatarAudioStream.h" #include "InjectedAudioStream.h" +using namespace std; + static const float DEFAULT_ATTENUATION_PER_DOUBLING_IN_DISTANCE = 0.5f; // attenuation = -6dB * log2(distance) static const int DISABLE_STATIC_JITTER_FRAMES = -1; static const float DEFAULT_NOISE_MUTING_THRESHOLD = 1.0f; @@ -49,11 +51,11 @@ static const QString AUDIO_THREADING_GROUP_KEY = "audio_threading"; int AudioMixer::_numStaticJitterFrames{ DISABLE_STATIC_JITTER_FRAMES }; float AudioMixer::_noiseMutingThreshold{ DEFAULT_NOISE_MUTING_THRESHOLD }; float AudioMixer::_attenuationPerDoublingInDistance{ DEFAULT_ATTENUATION_PER_DOUBLING_IN_DISTANCE }; -std::map> AudioMixer::_availableCodecs{ }; +map> AudioMixer::_availableCodecs{ }; QStringList AudioMixer::_codecPreferenceOrder{}; -QHash AudioMixer::_audioZones; -QVector AudioMixer::_zoneSettings; -QVector AudioMixer::_zoneReverbSettings; +vector AudioMixer::_audioZones; +vector AudioMixer::_zoneSettings; +vector AudioMixer::_zoneReverbSettings; AudioMixer::AudioMixer(ReceivedMessage& message) : ThreadedAssignment(message) @@ -67,7 +69,7 @@ AudioMixer::AudioMixer(ReceivedMessage& message) : _availableCodecs.clear(); // Make sure struct is clean auto pluginManager = DependencyManager::set(); auto codecPlugins = pluginManager->getCodecPlugins(); - std::for_each(codecPlugins.cbegin(), codecPlugins.cend(), + for_each(codecPlugins.cbegin(), codecPlugins.cend(), [&](const CodecPluginPointer& codec) { _availableCodecs[codec->getName()] = codec; }); @@ -122,7 +124,7 @@ void AudioMixer::queueAudioPacket(QSharedPointer message, Share void AudioMixer::queueReplicatedAudioPacket(QSharedPointer message) { // make sure we have a replicated node for the original sender of the packet auto nodeList = DependencyManager::get(); - + // Node ID is now part of user data, since replicated audio packets are non-sourced. QUuid nodeID = QUuid::fromRfc4122(message->readWithoutCopy(NUM_BYTES_RFC4122_UUID)); @@ -173,12 +175,12 @@ void AudioMixer::handleMuteEnvironmentPacket(QSharedPointer mes } } -const std::pair AudioMixer::negotiateCodec(std::vector codecs) { +const pair AudioMixer::negotiateCodec(vector codecs) { QString selectedCodecName; CodecPluginPointer selectedCodec; // read the codecs requested (by the client) - int minPreference = std::numeric_limits::max(); + int minPreference = numeric_limits::max(); for (auto& codec : codecs) { if (_availableCodecs.count(codec) > 0) { int preference = _codecPreferenceOrder.indexOf(codec); @@ -191,7 +193,7 @@ const std::pair AudioMixer::negotiateCodec(std::vec } } - return std::make_pair(selectedCodecName, _availableCodecs[selectedCodecName]); + return make_pair(selectedCodecName, _availableCodecs[selectedCodecName]); } void AudioMixer::handleNodeKilled(SharedNodePointer killedNode) { @@ -285,7 +287,7 @@ void AudioMixer::sendStatsPacket() { // timing stats QJsonObject timingStats; - auto addTiming = [&](Timer& timer, std::string name) { + auto addTiming = [&](Timer& timer, string name) { uint64_t timing, trailing; timer.get(timing, trailing); timingStats[("us_per_" + name).c_str()] = (qint64)(timing / _numStatFrames); @@ -366,7 +368,7 @@ AudioMixerClientData* AudioMixer::getOrCreateClientData(Node* node) { auto clientData = dynamic_cast(node->getLinkedData()); if (!clientData) { - node->setLinkedData(std::unique_ptr { new AudioMixerClientData(node->getUUID(), node->getLocalID()) }); + node->setLinkedData(unique_ptr { new AudioMixerClientData(node->getUUID(), node->getLocalID()) }); clientData = dynamic_cast(node->getLinkedData()); connect(clientData, &AudioMixerClientData::injectorStreamFinished, this, &AudioMixer::removeHRTFsForFinishedInjector); } @@ -410,7 +412,7 @@ void AudioMixer::start() { // prepare frames; pop off any new audio from their streams { auto prepareTimer = _prepareTiming.timer(); - std::for_each(cbegin, cend, [&](const SharedNodePointer& node) { + for_each(cbegin, cend, [&](const SharedNodePointer& node) { _stats.sumStreams += prepareFrame(node, frame); }); } @@ -455,26 +457,26 @@ void AudioMixer::start() { } } -std::chrono::microseconds AudioMixer::timeFrame(p_high_resolution_clock::time_point& timestamp) { +chrono::microseconds AudioMixer::timeFrame(p_high_resolution_clock::time_point& timestamp) { // advance the next frame - auto nextTimestamp = timestamp + std::chrono::microseconds(AudioConstants::NETWORK_FRAME_USECS); + auto nextTimestamp = timestamp + chrono::microseconds(AudioConstants::NETWORK_FRAME_USECS); auto now = p_high_resolution_clock::now(); // compute how long the last frame took - auto duration = std::chrono::duration_cast(now - timestamp); + auto duration = chrono::duration_cast(now - timestamp); // set the new frame timestamp - timestamp = std::max(now, nextTimestamp); + timestamp = max(now, nextTimestamp); // sleep until the next frame should start // WIN32 sleep_until is broken until VS2015 Update 2 - // instead, std::max (above) guarantees that timestamp >= now, so we can sleep_for - std::this_thread::sleep_for(timestamp - now); + // instead, max (above) guarantees that timestamp >= now, so we can sleep_for + this_thread::sleep_for(timestamp - now); return duration; } -void AudioMixer::throttle(std::chrono::microseconds duration, int frame) { +void AudioMixer::throttle(chrono::microseconds duration, int frame) { // throttle using a modified proportional-integral controller const float FRAME_TIME = 10000.0f; float mixRatio = duration.count() / FRAME_TIME; @@ -508,13 +510,13 @@ void AudioMixer::throttle(std::chrono::microseconds duration, int frame) { if (_trailingMixRatio > TARGET) { int proportionalTerm = 1 + (_trailingMixRatio - TARGET) / 0.1f; _throttlingRatio += THROTTLE_RATE * proportionalTerm; - _throttlingRatio = std::min(_throttlingRatio, 1.0f); + _throttlingRatio = min(_throttlingRatio, 1.0f); qCDebug(audio) << "audio-mixer is struggling (" << _trailingMixRatio << "mix/sleep) - throttling" << _throttlingRatio << "of streams"; } else if (_throttlingRatio > 0.0f && _trailingMixRatio <= BACKOFF_TARGET) { int proportionalTerm = 1 + (TARGET - _trailingMixRatio) / 0.2f; _throttlingRatio -= BACKOFF_RATE * proportionalTerm; - _throttlingRatio = std::max(_throttlingRatio, 0.0f); + _throttlingRatio = max(_throttlingRatio, 0.0f); qCDebug(audio) << "audio-mixer is recovering (" << _trailingMixRatio << "mix/sleep) - throttling" << _throttlingRatio << "of streams"; } @@ -661,8 +663,11 @@ void AudioMixer::parseSettingsObject(const QJsonObject& settingsObject) { const QString Y_MAX = "y_max"; const QString Z_MIN = "z_min"; const QString Z_MAX = "z_max"; - foreach (const QString& zone, zones.keys()) { - QJsonObject zoneObject = zones[zone].toObject(); + + auto zoneNames = zones.keys(); + _audioZones.reserve(zoneNames.length()); + foreach (const QString& zoneName, zoneNames) { + QJsonObject zoneObject = zones[zoneName].toObject(); if (zoneObject.contains(X_MIN) && zoneObject.contains(X_MAX) && zoneObject.contains(Y_MIN) && zoneObject.contains(Y_MAX) && zoneObject.contains(Z_MIN) && zoneObject.contains(Z_MAX)) { @@ -686,8 +691,8 @@ void AudioMixer::parseSettingsObject(const QJsonObject& settingsObject) { glm::vec3 corner(xMin, yMin, zMin); glm::vec3 dimensions(xMax - xMin, yMax - yMin, zMax - zMin); AABox zoneAABox(corner, dimensions); - _audioZones.insert(zone, zoneAABox); - qCDebug(audio) << "Added zone:" << zone << "(corner:" << corner << ", dimensions:" << dimensions << ")"; + _audioZones.push_back({ zoneName, zoneAABox }); + qCDebug(audio) << "Added zone:" << zoneName << "(corner:" << corner << ", dimensions:" << dimensions << ")"; } } } @@ -707,18 +712,28 @@ void AudioMixer::parseSettingsObject(const QJsonObject& settingsObject) { coefficientObject.contains(LISTENER) && coefficientObject.contains(COEFFICIENT)) { - ZoneSettings settings; + auto itSource = find_if(begin(_audioZones), end(_audioZones), [&](const ZoneDescription& description) { + return description.name == coefficientObject.value(SOURCE).toString(); + }); + auto itListener = find_if(begin(_audioZones), end(_audioZones), [&](const ZoneDescription& description) { + return description.name == coefficientObject.value(LISTENER).toString(); + }); bool ok; - settings.source = coefficientObject.value(SOURCE).toString(); - settings.listener = coefficientObject.value(LISTENER).toString(); - settings.coefficient = coefficientObject.value(COEFFICIENT).toString().toFloat(&ok); + float coefficient = coefficientObject.value(COEFFICIENT).toString().toFloat(&ok); - if (ok && settings.coefficient >= 0.0f && settings.coefficient <= 1.0f && - _audioZones.contains(settings.source) && _audioZones.contains(settings.listener)) { + + if (ok && coefficient >= 0.0f && coefficient <= 1.0f && + itSource != end(_audioZones) && + itListener != end(_audioZones)) { + + ZoneSettings settings; + settings.source = itSource - begin(_audioZones); + settings.listener = itListener - begin(_audioZones); + settings.coefficient = coefficient; _zoneSettings.push_back(settings); - qCDebug(audio) << "Added Coefficient:" << settings.source << settings.listener << settings.coefficient; + qCDebug(audio) << "Added Coefficient:" << itSource->name << itListener->name << settings.coefficient; } } } @@ -739,19 +754,21 @@ void AudioMixer::parseSettingsObject(const QJsonObject& settingsObject) { reverbObject.contains(WET_LEVEL)) { bool okReverbTime, okWetLevel; - QString zone = reverbObject.value(ZONE).toString(); + auto itZone = find_if(begin(_audioZones), end(_audioZones), [&](const ZoneDescription& description) { + return description.name == reverbObject.value(ZONE).toString(); + }); float reverbTime = reverbObject.value(REVERB_TIME).toString().toFloat(&okReverbTime); float wetLevel = reverbObject.value(WET_LEVEL).toString().toFloat(&okWetLevel); - if (okReverbTime && okWetLevel && _audioZones.contains(zone)) { + if (okReverbTime && okWetLevel && itZone != end(_audioZones)) { ReverbSettings settings; - settings.zone = zone; + settings.zone = itZone - begin(_audioZones); settings.reverbTime = reverbTime; settings.wetLevel = wetLevel; _zoneReverbSettings.push_back(settings); - qCDebug(audio) << "Added Reverb:" << zone << reverbTime << wetLevel; + qCDebug(audio) << "Added Reverb:" << itZone->name << reverbTime << wetLevel; } } } @@ -764,7 +781,7 @@ AudioMixer::Timer::Timing::Timing(uint64_t& sum) : _sum(sum) { } AudioMixer::Timer::Timing::~Timing() { - _sum += std::chrono::duration_cast(p_high_resolution_clock::now() - _timing).count(); + _sum += chrono::duration_cast(p_high_resolution_clock::now() - _timing).count(); } void AudioMixer::Timer::get(uint64_t& timing, uint64_t& trailing) { diff --git a/assignment-client/src/audio/AudioMixer.h b/assignment-client/src/audio/AudioMixer.h index f9eb18da6d..d201ce5ac3 100644 --- a/assignment-client/src/audio/AudioMixer.h +++ b/assignment-client/src/audio/AudioMixer.h @@ -34,13 +34,18 @@ class AudioMixer : public ThreadedAssignment { public: AudioMixer(ReceivedMessage& message); + + struct ZoneDescription { + QString name; + AABox area; + }; struct ZoneSettings { - QString source; - QString listener; + int source; + int listener; float coefficient; }; struct ReverbSettings { - QString zone; + int zone; float reverbTime; float wetLevel; }; @@ -48,9 +53,9 @@ public: static int getStaticJitterFrames() { return _numStaticJitterFrames; } static bool shouldMute(float quietestFrame) { return quietestFrame > _noiseMutingThreshold; } static float getAttenuationPerDoublingInDistance() { return _attenuationPerDoublingInDistance; } - static const QHash& getAudioZones() { return _audioZones; } - static const QVector& getZoneSettings() { return _zoneSettings; } - static const QVector& getReverbSettings() { return _zoneReverbSettings; } + static const std::vector& getAudioZones() { return _audioZones; } + static const std::vector& getZoneSettings() { return _zoneSettings; } + static const std::vector& getReverbSettings() { return _zoneReverbSettings; } static const std::pair negotiateCodec(std::vector codecs); static bool shouldReplicateTo(const Node& from, const Node& to) { @@ -136,9 +141,11 @@ private: static float _attenuationPerDoublingInDistance; static std::map _availableCodecs; static QStringList _codecPreferenceOrder; - static QHash _audioZones; - static QVector _zoneSettings; - static QVector _zoneReverbSettings; + + + static std::vector _audioZones; + static std::vector _zoneSettings; + static std::vector _zoneReverbSettings; }; diff --git a/assignment-client/src/audio/AudioMixerSlave.cpp b/assignment-client/src/audio/AudioMixerSlave.cpp index 4b97049758..869c7361e6 100644 --- a/assignment-client/src/audio/AudioMixerSlave.cpp +++ b/assignment-client/src/audio/AudioMixerSlave.cpp @@ -440,12 +440,12 @@ void sendEnvironmentPacket(const SharedNodePointer& node, AudioMixerClientData& glm::vec3 streamPosition = stream->getPosition(); // find reverb properties - for (int i = 0; i < reverbSettings.size(); ++i) { - AABox box = audioZones[reverbSettings[i].zone]; + for (const auto& settings : reverbSettings) { + AABox box = audioZones[settings.zone].area; if (box.contains(streamPosition)) { hasReverb = true; - reverbTime = reverbSettings[i].reverbTime; - wetLevel = reverbSettings[i].wetLevel; + reverbTime = settings.reverbTime; + wetLevel = settings.wetLevel; break; } } @@ -539,10 +539,10 @@ float computeGain(const AudioMixerClientData& listenerNodeData, const AvatarAudi // find distance attenuation coefficient float attenuationPerDoublingInDistance = AudioMixer::getAttenuationPerDoublingInDistance(); - for (int i = 0; i < zoneSettings.length(); ++i) { - if (audioZones[zoneSettings[i].source].contains(streamToAdd.getPosition()) && - audioZones[zoneSettings[i].listener].contains(listeningNodeStream.getPosition())) { - attenuationPerDoublingInDistance = zoneSettings[i].coefficient; + for (const auto& settings : zoneSettings) { + if (audioZones[settings.source].area.contains(streamToAdd.getPosition()) && + audioZones[settings.listener].area.contains(listeningNodeStream.getPosition())) { + attenuationPerDoublingInDistance = settings.coefficient; break; } } From bcba2a1cf16269160386bbed64ed6d822254188c Mon Sep 17 00:00:00 2001 From: Stephen Birarda Date: Fri, 3 Aug 2018 13:14:19 -0700 Subject: [PATCH 04/21] use local ID for stream hash --- assignment-client/src/audio/AudioMixer.cpp | 7 ++--- .../src/audio/AudioMixerClientData.cpp | 5 ++-- .../src/audio/AudioMixerClientData.h | 13 ++++++--- .../src/audio/AudioMixerSlave.cpp | 27 +++++++++---------- assignment-client/src/audio/AudioMixerSlave.h | 6 ++--- 5 files changed, 32 insertions(+), 26 deletions(-) diff --git a/assignment-client/src/audio/AudioMixer.cpp b/assignment-client/src/audio/AudioMixer.cpp index 7b91916bbb..5dabfaaa59 100644 --- a/assignment-client/src/audio/AudioMixer.cpp +++ b/assignment-client/src/audio/AudioMixer.cpp @@ -203,7 +203,7 @@ void AudioMixer::handleNodeKilled(SharedNodePointer killedNode) { nodeList->eachNode([&killedNode](const SharedNodePointer& node) { auto clientData = dynamic_cast(node->getLinkedData()); if (clientData) { - clientData->removeNode(killedNode->getUUID()); + clientData->removeNode(killedNode->getLocalID()); } }); } @@ -233,7 +233,7 @@ void AudioMixer::handleKillAvatarPacket(QSharedPointer packet, nodeList->eachNode([sendingNode](const SharedNodePointer& node){ auto listenerClientData = dynamic_cast(node->getLinkedData()); if (listenerClientData) { - listenerClientData->removeHRTFForStream(sendingNode->getUUID()); + listenerClientData->removeHRTFForStream(sendingNode->getLocalID()); } }); } @@ -248,7 +248,7 @@ void AudioMixer::removeHRTFsForFinishedInjector(const QUuid& streamID) { nodeList->eachNode([injectorClientData, &streamID](const SharedNodePointer& node){ auto listenerClientData = dynamic_cast(node->getLinkedData()); if (listenerClientData) { - listenerClientData->removeHRTFForStream(injectorClientData->getNodeID(), streamID); + listenerClientData->removeHRTFForStream(injectorClientData->getNodeLocalID(), streamID); } }); } @@ -370,6 +370,7 @@ AudioMixerClientData* AudioMixer::getOrCreateClientData(Node* node) { if (!clientData) { node->setLinkedData(unique_ptr { new AudioMixerClientData(node->getUUID(), node->getLocalID()) }); clientData = dynamic_cast(node->getLinkedData()); + clientData->setNodeLocalID(node->getLocalID()); connect(clientData, &AudioMixerClientData::injectorStreamFinished, this, &AudioMixer::removeHRTFsForFinishedInjector); } diff --git a/assignment-client/src/audio/AudioMixerClientData.cpp b/assignment-client/src/audio/AudioMixerClientData.cpp index a8398abcd3..07fa337a57 100644 --- a/assignment-client/src/audio/AudioMixerClientData.cpp +++ b/assignment-client/src/audio/AudioMixerClientData.cpp @@ -197,7 +197,8 @@ void AudioMixerClientData::parsePerAvatarGainSet(ReceivedMessage& message, const qCDebug(audio) << "Setting MASTER avatar gain for " << uuid << " to " << gain; } else { // set the per-source avatar gain - hrtfForStream(avatarUuid, QUuid()).setGainAdjustment(gain); + auto nodeList = DependencyManager::get(); + hrtfForStream(nodeList->nodeWithUUID(avatarUuid)->getLocalID(), QUuid()).setGainAdjustment(gain); qCDebug(audio) << "Setting avatar gain adjustment for hrtf[" << uuid << "][" << avatarUuid << "] to " << gain; } } @@ -225,7 +226,7 @@ AvatarAudioStream* AudioMixerClientData::getAvatarAudioStream() { return NULL; } -void AudioMixerClientData::removeHRTFForStream(const QUuid& nodeID, const QUuid& streamID) { +void AudioMixerClientData::removeHRTFForStream(Node::LocalID nodeID, const QUuid& streamID) { auto it = _nodeSourcesHRTFMap.find(nodeID); if (it != _nodeSourcesHRTFMap.end()) { // erase the stream with the given ID from the given node diff --git a/assignment-client/src/audio/AudioMixerClientData.h b/assignment-client/src/audio/AudioMixerClientData.h index 201ccfe867..d74c8bb6a4 100644 --- a/assignment-client/src/audio/AudioMixerClientData.h +++ b/assignment-client/src/audio/AudioMixerClientData.h @@ -50,13 +50,13 @@ public: // they are not thread-safe // returns a new or existing HRTF object for the given stream from the given node - AudioHRTF& hrtfForStream(const QUuid& nodeID, const QUuid& streamID = QUuid()) { return _nodeSourcesHRTFMap[nodeID][streamID]; } + AudioHRTF& hrtfForStream(Node::LocalID nodeID, const QUuid& streamID = QUuid()) { return _nodeSourcesHRTFMap[nodeID][streamID]; } // removes an AudioHRTF object for a given stream - void removeHRTFForStream(const QUuid& nodeID, const QUuid& streamID = QUuid()); + void removeHRTFForStream(Node::LocalID nodeID, const QUuid& streamID = QUuid()); // remove all sources and data from this node - void removeNode(const QUuid& nodeID) { _nodeSourcesHRTFMap.erase(nodeID); } + void removeNode(Node::LocalID nodeID) { _nodeSourcesHRTFMap.erase(nodeID); } void removeAgentAvatarAudioStream(); @@ -75,6 +75,9 @@ public: QJsonObject getAudioStreamStats(); + void setNodeLocalID(Node::LocalID localNodeID) { _localNodeID = localNodeID; } + Node::LocalID getNodeLocalID() { return _localNodeID; } + void sendAudioStreamStatsPackets(const SharedNodePointer& destinationNode); void incrementOutgoingMixedAudioSequenceNumber() { _outgoingMixedAudioSequenceNumber++; } @@ -150,7 +153,7 @@ private: IgnoreZoneMemo _ignoreZone; using HRTFMap = std::unordered_map; - using NodeSourcesHRTFMap = std::unordered_map; + using NodeSourcesHRTFMap = std::unordered_map; NodeSourcesHRTFMap _nodeSourcesHRTFMap; quint16 _outgoingMixedAudioSequenceNumber; @@ -170,6 +173,8 @@ private: bool _shouldMuteClient { false }; bool _requestsDomainListData { false }; + + Node::LocalID _localNodeID; }; #endif // hifi_AudioMixerClientData_h diff --git a/assignment-client/src/audio/AudioMixerSlave.cpp b/assignment-client/src/audio/AudioMixerSlave.cpp index 869c7361e6..d350f2b27a 100644 --- a/assignment-client/src/audio/AudioMixerSlave.cpp +++ b/assignment-client/src/audio/AudioMixerSlave.cpp @@ -142,9 +142,9 @@ bool AudioMixerSlave::prepareMix(const SharedNodePointer& listener) { std::vector> throttledNodes; typedef void (AudioMixerSlave::*MixFunctor)( - AudioMixerClientData&, const QUuid&, const AvatarAudioStream&, const PositionalAudioStream&); + AudioMixerClientData&, Node::LocalID, const AvatarAudioStream&, const PositionalAudioStream&); auto forAllStreams = [&](const SharedNodePointer& node, AudioMixerClientData* nodeData, MixFunctor mixFunctor) { - auto nodeID = node->getUUID(); + auto nodeID = node->getLocalID(); for (auto& streamPair : nodeData->getAudioStreams()) { (this->*mixFunctor)(*listenerData, nodeID, *listenerAudioStream, *streamPair); } @@ -164,14 +164,13 @@ bool AudioMixerSlave::prepareMix(const SharedNodePointer& listener) { // only mix the echo, if requested for (auto& streamPair : nodeData->getAudioStreams()) { if (streamPair->shouldLoopbackForNode()) { - mixStream(*listenerData, node->getUUID(), *listenerAudioStream, *streamPair); + mixStream(*listenerData, node->getLocalID(), *listenerAudioStream, *streamPair); } } } else if (!listenerData->shouldIgnore(listener, node, _frame)) { if (!isThrottling) { forAllStreams(node, nodeData, &AudioMixerSlave::mixStream); } else { - auto nodeID = node->getUUID(); // compute the node's max relative volume float nodeVolume = 0.0f; @@ -182,7 +181,7 @@ bool AudioMixerSlave::prepareMix(const SharedNodePointer& listener) { float gain = approximateGain(*listenerAudioStream, *nodeStream, relativePosition); // modify by hrtf gain adjustment - auto& hrtf = listenerData->hrtfForStream(nodeID, nodeStream->getStreamIdentifier()); + auto& hrtf = listenerData->hrtfForStream(node->getLocalID(), nodeStream->getStreamIdentifier()); gain *= hrtf.getGainAdjustment(); auto streamVolume = nodeStream->getLastPopOutputTrailingLoudness() * gain; @@ -243,23 +242,23 @@ bool AudioMixerSlave::prepareMix(const SharedNodePointer& listener) { return hasAudio; } -void AudioMixerSlave::throttleStream(AudioMixerClientData& listenerNodeData, const QUuid& sourceNodeID, +void AudioMixerSlave::throttleStream(AudioMixerClientData& listenerNodeData, Node::LocalID sourceNodeLocalID, const AvatarAudioStream& listeningNodeStream, const PositionalAudioStream& streamToAdd) { // only throttle this stream to the mix if it has a valid position, we won't know how to mix it otherwise if (streamToAdd.hasValidPosition()) { - addStream(listenerNodeData, sourceNodeID, listeningNodeStream, streamToAdd, true); + addStream(listenerNodeData, sourceNodeLocalID, listeningNodeStream, streamToAdd, true); } } -void AudioMixerSlave::mixStream(AudioMixerClientData& listenerNodeData, const QUuid& sourceNodeID, +void AudioMixerSlave::mixStream(AudioMixerClientData& listenerNodeData, Node::LocalID sourceNodeLocalID, const AvatarAudioStream& listeningNodeStream, const PositionalAudioStream& streamToAdd) { // only add the stream to the mix if it has a valid position, we won't know how to mix it otherwise if (streamToAdd.hasValidPosition()) { - addStream(listenerNodeData, sourceNodeID, listeningNodeStream, streamToAdd, false); + addStream(listenerNodeData, sourceNodeLocalID, listeningNodeStream, streamToAdd, false); } } -void AudioMixerSlave::addStream(AudioMixerClientData& listenerNodeData, const QUuid& sourceNodeID, +void AudioMixerSlave::addStream(AudioMixerClientData& listenerNodeData, Node::LocalID sourceNodeLocalID, const AvatarAudioStream& listeningNodeStream, const PositionalAudioStream& streamToAdd, bool throttle) { ++stats.totalMixes; @@ -301,7 +300,7 @@ void AudioMixerSlave::addStream(AudioMixerClientData& listenerNodeData, const QU // (this is not done for stereo streams since they do not go through the HRTF) if (!streamToAdd.isStereo() && !isEcho) { // get the existing listener-source HRTF object, or create a new one - auto& hrtf = listenerNodeData.hrtfForStream(sourceNodeID, streamToAdd.getStreamIdentifier()); + auto& hrtf = listenerNodeData.hrtfForStream(sourceNodeLocalID, streamToAdd.getStreamIdentifier()); static int16_t silentMonoBlock[AudioConstants::NETWORK_FRAME_SAMPLES_PER_CHANNEL] = {}; hrtf.renderSilent(silentMonoBlock, _mixSamples, HRTF_DATASET_INDEX, azimuth, distance, gain, @@ -321,7 +320,7 @@ void AudioMixerSlave::addStream(AudioMixerClientData& listenerNodeData, const QU if (streamToAdd.isStereo()) { // apply the avatar gain adjustment - auto& hrtf = listenerNodeData.hrtfForStream(sourceNodeID, streamToAdd.getStreamIdentifier()); + auto& hrtf = listenerNodeData.hrtfForStream(sourceNodeLocalID, streamToAdd.getStreamIdentifier()); gain *= hrtf.getGainAdjustment(); const float scale = 1/32768.0f; // int16_t to float @@ -351,7 +350,7 @@ void AudioMixerSlave::addStream(AudioMixerClientData& listenerNodeData, const QU } // get the existing listener-source HRTF object, or create a new one - auto& hrtf = listenerNodeData.hrtfForStream(sourceNodeID, streamToAdd.getStreamIdentifier()); + auto& hrtf = listenerNodeData.hrtfForStream(sourceNodeLocalID, streamToAdd.getStreamIdentifier()); streamPopOutput.readSamples(_bufferSamples, AudioConstants::NETWORK_FRAME_SAMPLES_PER_CHANNEL); @@ -375,7 +374,7 @@ void AudioMixerSlave::addStream(AudioMixerClientData& listenerNodeData, const QU if (streamToAdd.getType() == PositionalAudioStream::Injector) { // apply per-avatar gain to positional audio injectors, which wouldn't otherwise be affected by PAL sliders - hrtf.setGainAdjustment(listenerNodeData.hrtfForStream(sourceNodeID, QUuid()).getGainAdjustment()); + hrtf.setGainAdjustment(listenerNodeData.hrtfForStream(sourceNodeLocalID, QUuid()).getGainAdjustment()); } hrtf.render(_bufferSamples, _mixSamples, HRTF_DATASET_INDEX, azimuth, distance, gain, diff --git a/assignment-client/src/audio/AudioMixerSlave.h b/assignment-client/src/audio/AudioMixerSlave.h index 074d10ff40..a0044743b0 100644 --- a/assignment-client/src/audio/AudioMixerSlave.h +++ b/assignment-client/src/audio/AudioMixerSlave.h @@ -45,11 +45,11 @@ public: private: // create mix, returns true if mix has audio bool prepareMix(const SharedNodePointer& listener); - void throttleStream(AudioMixerClientData& listenerData, const QUuid& streamerID, + void throttleStream(AudioMixerClientData& listenerData, Node::LocalID streamerID, const AvatarAudioStream& listenerStream, const PositionalAudioStream& streamer); - void mixStream(AudioMixerClientData& listenerData, const QUuid& streamerID, + void mixStream(AudioMixerClientData& listenerData, Node::LocalID streamerID, const AvatarAudioStream& listenerStream, const PositionalAudioStream& streamer); - void addStream(AudioMixerClientData& listenerData, const QUuid& streamerID, + void addStream(AudioMixerClientData& listenerData, Node::LocalID streamerID, const AvatarAudioStream& listenerStream, const PositionalAudioStream& streamer, bool throttle); From c992150c10bd715eb3e4ee3bae66fa4110c4466b Mon Sep 17 00:00:00 2001 From: Stephen Birarda Date: Fri, 3 Aug 2018 17:07:55 -0700 Subject: [PATCH 05/21] change stream HRTF map to use a vector --- .../src/audio/AudioMixerClientData.cpp | 22 ++++++++++++++++++- .../src/audio/AudioMixerClientData.h | 11 +++++++--- 2 files changed, 29 insertions(+), 4 deletions(-) diff --git a/assignment-client/src/audio/AudioMixerClientData.cpp b/assignment-client/src/audio/AudioMixerClientData.cpp index 07fa337a57..a5f5521a2d 100644 --- a/assignment-client/src/audio/AudioMixerClientData.cpp +++ b/assignment-client/src/audio/AudioMixerClientData.cpp @@ -226,11 +226,31 @@ AvatarAudioStream* AudioMixerClientData::getAvatarAudioStream() { return NULL; } +AudioHRTF& AudioMixerClientData::hrtfForStream(Node::LocalID nodeID, const QUuid& streamID) { + auto& hrtfVector = _nodeSourcesHRTFMap[nodeID]; + + auto streamIt = std::find_if(hrtfVector.begin(), hrtfVector.end(), [&streamID](IdentifiedHRTF& identifiedHRTF){ + return identifiedHRTF.streamIdentifier == streamID; + }); + + if (streamIt == hrtfVector.end()) { + hrtfVector.push_back({ streamID, std::unique_ptr(new AudioHRTF) }); + + return *hrtfVector.back().hrtf; + } else { + return *streamIt->hrtf; + } +} + void AudioMixerClientData::removeHRTFForStream(Node::LocalID nodeID, const QUuid& streamID) { auto it = _nodeSourcesHRTFMap.find(nodeID); if (it != _nodeSourcesHRTFMap.end()) { + auto streamIt = std::find_if(it->second.begin(), it->second.end(), [&streamID](IdentifiedHRTF& identifiedHRTF){ + return identifiedHRTF.streamIdentifier == streamID; + }); + // erase the stream with the given ID from the given node - it->second.erase(streamID); + it->second.erase(streamIt); // is the map for this node now empty? // if so we can remove it diff --git a/assignment-client/src/audio/AudioMixerClientData.h b/assignment-client/src/audio/AudioMixerClientData.h index d74c8bb6a4..a35f3dea5a 100644 --- a/assignment-client/src/audio/AudioMixerClientData.h +++ b/assignment-client/src/audio/AudioMixerClientData.h @@ -50,7 +50,7 @@ public: // they are not thread-safe // returns a new or existing HRTF object for the given stream from the given node - AudioHRTF& hrtfForStream(Node::LocalID nodeID, const QUuid& streamID = QUuid()) { return _nodeSourcesHRTFMap[nodeID][streamID]; } + AudioHRTF& hrtfForStream(Node::LocalID nodeID, const QUuid& streamID = QUuid()); // removes an AudioHRTF object for a given stream void removeHRTFForStream(Node::LocalID nodeID, const QUuid& streamID = QUuid()); @@ -152,8 +152,13 @@ private: }; IgnoreZoneMemo _ignoreZone; - using HRTFMap = std::unordered_map; - using NodeSourcesHRTFMap = std::unordered_map; + struct IdentifiedHRTF { + QUuid streamIdentifier; + std::unique_ptr hrtf; + }; + + using HRTFVector = std::vector; + using NodeSourcesHRTFMap = std::unordered_map; NodeSourcesHRTFMap _nodeSourcesHRTFMap; quint16 _outgoingMixedAudioSequenceNumber; From 371de312ccaee517045f24e0fa2934539ff5bde5 Mon Sep 17 00:00:00 2001 From: Stephen Birarda Date: Fri, 24 Aug 2018 18:23:41 -0700 Subject: [PATCH 06/21] enumerate a vector of mixable streams for each listener --- assignment-client/src/audio/AudioMixer.cpp | 82 ++-- assignment-client/src/audio/AudioMixer.h | 4 +- .../src/audio/AudioMixerClientData.cpp | 393 +++++++++--------- .../src/audio/AudioMixerClientData.h | 120 +++--- .../src/audio/AudioMixerSlave.cpp | 347 ++++++++++------ assignment-client/src/audio/AudioMixerSlave.h | 25 +- .../src/audio/AudioMixerSlavePool.cpp | 2 +- .../src/audio/AudioMixerSlavePool.h | 8 +- assignment-client/src/avatars/AvatarMixer.cpp | 8 +- .../src/avatars/AvatarMixerClientData.cpp | 2 +- .../src/avatars/AvatarMixerClientData.h | 5 + .../src/avatars/AvatarMixerSlave.cpp | 2 +- libraries/audio/src/PositionalAudioStream.cpp | 31 ++ libraries/audio/src/PositionalAudioStream.h | 32 +- libraries/networking/src/Node.cpp | 16 +- libraries/networking/src/Node.h | 15 +- 16 files changed, 646 insertions(+), 446 deletions(-) diff --git a/assignment-client/src/audio/AudioMixer.cpp b/assignment-client/src/audio/AudioMixer.cpp index 5dabfaaa59..a7a5ac95bc 100644 --- a/assignment-client/src/audio/AudioMixer.cpp +++ b/assignment-client/src/audio/AudioMixer.cpp @@ -196,17 +196,6 @@ const pair AudioMixer::negotiateCodec(vector(); - - nodeList->eachNode([&killedNode](const SharedNodePointer& node) { - auto clientData = dynamic_cast(node->getLinkedData()); - if (clientData) { - clientData->removeNode(killedNode->getLocalID()); - } - }); -} void AudioMixer::handleNodeMuteRequestPacket(QSharedPointer packet, SharedNodePointer sendingNode) { auto nodeList = DependencyManager::get(); @@ -225,32 +214,31 @@ void AudioMixer::handleNodeMuteRequestPacket(QSharedPointer pac } } +void AudioMixer::handleNodeKilled(SharedNodePointer killedNode) { + auto clientData = dynamic_cast(killedNode->getLinkedData()); + if (clientData) { + // stage the removal of all streams from this node, workers handle when preparing mixes for listeners + _workerSharedData.removedNodes.emplace_back(killedNode->getLocalID()); + } +} + void AudioMixer::handleKillAvatarPacket(QSharedPointer packet, SharedNodePointer sendingNode) { auto clientData = dynamic_cast(sendingNode->getLinkedData()); if (clientData) { clientData->removeAgentAvatarAudioStream(); - auto nodeList = DependencyManager::get(); - nodeList->eachNode([sendingNode](const SharedNodePointer& node){ - auto listenerClientData = dynamic_cast(node->getLinkedData()); - if (listenerClientData) { - listenerClientData->removeHRTFForStream(sendingNode->getLocalID()); - } - }); + + // stage a removal of the avatar audio stream from this Agent, workers handle when preparing mixes for listeners + _workerSharedData.removedStreams.emplace_back(sendingNode->getUUID(), sendingNode->getLocalID(), QUuid()); } } void AudioMixer::removeHRTFsForFinishedInjector(const QUuid& streamID) { auto injectorClientData = qobject_cast(sender()); - if (injectorClientData) { - // enumerate the connected listeners to remove HRTF objects for the disconnected injector - auto nodeList = DependencyManager::get(); - nodeList->eachNode([injectorClientData, &streamID](const SharedNodePointer& node){ - auto listenerClientData = dynamic_cast(node->getLinkedData()); - if (listenerClientData) { - listenerClientData->removeHRTFForStream(injectorClientData->getNodeLocalID(), streamID); - } - }); + if (injectorClientData) { + // stage the removal of this stream, workers handle when preparing mixes for listeners + _workerSharedData.removedStreams.emplace_back(injectorClientData->getNodeID(), injectorClientData->getNodeLocalID(), + streamID); } } @@ -370,7 +358,6 @@ AudioMixerClientData* AudioMixer::getOrCreateClientData(Node* node) { if (!clientData) { node->setLinkedData(unique_ptr { new AudioMixerClientData(node->getUUID(), node->getLocalID()) }); clientData = dynamic_cast(node->getLinkedData()); - clientData->setNodeLocalID(node->getLocalID()); connect(clientData, &AudioMixerClientData::injectorStreamFinished, this, &AudioMixer::removeHRTFsForFinishedInjector); } @@ -409,6 +396,30 @@ void AudioMixer::start() { auto frameTimer = _frameTiming.timer(); + // process (node-isolated) audio packets across slave threads + { + auto packetsTimer = _packetsTiming.timer(); + + // first clear the concurrent vector of added streams that the slaves will add to when they process packets + _workerSharedData.addedStreams.clear(); + + nodeList->nestedEach([&](NodeList::const_iterator cbegin, NodeList::const_iterator cend) { + _slavePool.processPackets(cbegin, cend); + }); + } + + // process queued events (networking, global audio packets, &c.) + { + auto eventsTimer = _eventsTiming.timer(); + + // clear removed nodes and removed streams before we process events that will setup the new set + _workerSharedData.removedNodes.clear(); + _workerSharedData.removedStreams.clear(); + + // since we're a while loop we need to yield to qt's event processing + QCoreApplication::processEvents(); + } + nodeList->nestedEach([&](NodeList::const_iterator cbegin, NodeList::const_iterator cend) { // prepare frames; pop off any new audio from their streams { @@ -434,21 +445,6 @@ void AudioMixer::start() { ++frame; ++_numStatFrames; - // process queued events (networking, global audio packets, &c.) - { - auto eventsTimer = _eventsTiming.timer(); - - // since we're a while loop we need to yield to qt's event processing - QCoreApplication::processEvents(); - - // process (node-isolated) audio packets across slave threads - { - nodeList->nestedEach([&](NodeList::const_iterator cbegin, NodeList::const_iterator cend) { - auto packetsTimer = _packetsTiming.timer(); - _slavePool.processPackets(cbegin, cend); - }); - } - } if (_isFinished) { // alert qt eventing that this is finished diff --git a/assignment-client/src/audio/AudioMixer.h b/assignment-client/src/audio/AudioMixer.h index d201ce5ac3..6cddd539da 100644 --- a/assignment-client/src/audio/AudioMixer.h +++ b/assignment-client/src/audio/AudioMixer.h @@ -86,6 +86,7 @@ private: // mixing helpers std::chrono::microseconds timeFrame(p_high_resolution_clock::time_point& timestamp); void throttle(std::chrono::microseconds frameDuration, int frame); + // pop a frame from any streams on the node // returns the number of available streams int prepareFrame(const SharedNodePointer& node, unsigned int frame); @@ -105,7 +106,7 @@ private: int _numStatFrames { 0 }; AudioMixerStats _stats; - AudioMixerSlavePool _slavePool; + AudioMixerSlavePool _slavePool { _workerSharedData }; class Timer { public: @@ -147,6 +148,7 @@ private: static std::vector _zoneSettings; static std::vector _zoneReverbSettings; + AudioMixerSlave::SharedData _workerSharedData; }; #endif // hifi_AudioMixer_h diff --git a/assignment-client/src/audio/AudioMixerClientData.cpp b/assignment-client/src/audio/AudioMixerClientData.cpp index a5f5521a2d..e0b156473a 100644 --- a/assignment-client/src/audio/AudioMixerClientData.cpp +++ b/assignment-client/src/audio/AudioMixerClientData.cpp @@ -28,7 +28,6 @@ AudioMixerClientData::AudioMixerClientData(const QUuid& nodeID, Node::LocalID nodeLocalID) : NodeData(nodeID, nodeLocalID), audioLimiter(AudioConstants::SAMPLE_RATE, AudioConstants::STEREO), - _ignoreZone(*this), _outgoingMixedAudioSequenceNumber(0), _downstreamAudioStreamStats() { @@ -56,7 +55,7 @@ void AudioMixerClientData::queuePacket(QSharedPointer message, _packetQueue.push(message); } -void AudioMixerClientData::processPackets() { +void AudioMixerClientData::processPackets(ConcurrentAddedStreams& addedStreams) { SharedNodePointer node = _packetQueue.node; assert(_packetQueue.empty() || node); _packetQueue.node.clear(); @@ -69,22 +68,17 @@ void AudioMixerClientData::processPackets() { case PacketType::MicrophoneAudioWithEcho: case PacketType::InjectAudio: case PacketType::SilentAudioFrame: { - if (node->isUpstream()) { setupCodecForReplicatedAgent(packet); } - QMutexLocker lock(&getMutex()); - parseData(*packet); + processStreamPacket(*packet, addedStreams); optionallyReplicatePacket(*packet, *node); - break; } case PacketType::AudioStreamStats: { - QMutexLocker lock(&getMutex()); parseData(*packet); - break; } case PacketType::NegotiateAudioFormat: @@ -186,29 +180,113 @@ void AudioMixerClientData::parseRequestsDomainListData(ReceivedMessage& message) void AudioMixerClientData::parsePerAvatarGainSet(ReceivedMessage& message, const SharedNodePointer& node) { QUuid uuid = node->getUUID(); // parse the UUID from the packet - QUuid avatarUuid = QUuid::fromRfc4122(message.readWithoutCopy(NUM_BYTES_RFC4122_UUID)); + QUuid avatarUUID = QUuid::fromRfc4122(message.readWithoutCopy(NUM_BYTES_RFC4122_UUID)); uint8_t packedGain; message.readPrimitive(&packedGain); float gain = unpackFloatGainFromByte(packedGain); - if (avatarUuid.isNull()) { + if (avatarUUID.isNull()) { // set the MASTER avatar gain setMasterAvatarGain(gain); qCDebug(audio) << "Setting MASTER avatar gain for " << uuid << " to " << gain; } else { // set the per-source avatar gain - auto nodeList = DependencyManager::get(); - hrtfForStream(nodeList->nodeWithUUID(avatarUuid)->getLocalID(), QUuid()).setGainAdjustment(gain); - qCDebug(audio) << "Setting avatar gain adjustment for hrtf[" << uuid << "][" << avatarUuid << "] to " << gain; + setGainForAvatar(avatarUUID, gain); + qCDebug(audio) << "Setting avatar gain adjustment for hrtf[" << uuid << "][" << avatarUUID << "] to " << gain; + } +} + +void AudioMixerClientData::setGainForAvatar(QUuid nodeID, uint8_t gain) { + auto it = std::find_if(_mixableStreams.cbegin(), _mixableStreams.cend(), [nodeID](const MixableStream& mixableStream){ + return mixableStream.nodeStreamID.nodeID == nodeID && mixableStream.nodeStreamID.streamID.isNull(); + }); + + if (it != _mixableStreams.cend()) { + it->hrtf->setGainAdjustment(gain); } } void AudioMixerClientData::parseNodeIgnoreRequest(QSharedPointer message, const SharedNodePointer& node) { - node->parseIgnoreRequestMessage(message); + auto ignoredNodesPair = node->parseIgnoreRequestMessage(message); + + // we have a vector of ignored or unignored node UUIDs - update our internal data structures so that + // streams can be included or excluded next time a mix is being created + if (ignoredNodesPair.second) { + // we have newly ignored nodes, add them to our vector + _newIgnoredNodeIDs.insert(std::end(_newIgnoredNodeIDs), + std::begin(ignoredNodesPair.first), std::end(ignoredNodesPair.first)); + } else { + // we have newly unignored nodes, add them to our vector + _newUnignoredNodeIDs.insert(std::end(_newUnignoredNodeIDs), + std::begin(ignoredNodesPair.first), std::end(ignoredNodesPair.first)); + } + + auto nodeList = DependencyManager::get(); + for (auto& nodeID : ignoredNodesPair.first) { + auto otherNode = nodeList->nodeWithUUID(nodeID); + if (otherNode) { + auto otherNodeMixerClientData = static_cast(otherNode->getLinkedData()); + if (otherNodeMixerClientData) { + if (ignoredNodesPair.second) { + otherNodeMixerClientData->ignoredByNode(getNodeID()); + } else { + otherNodeMixerClientData->unignoredByNode(getNodeID()); + } + } + } + } +} + +void AudioMixerClientData::ignoredByNode(QUuid nodeID) { + // first add this ID to the concurrent vector for newly ignoring nodes + _newIgnoringNodeIDs.push_back(nodeID); + + // now take a lock and on the consistent vector of ignoring nodes and make sure this node is in it + std::lock_guard lock(_ignoringNodeIDsMutex); + if (std::find(_ignoringNodeIDs.begin(), _ignoringNodeIDs.end(), nodeID) == _ignoringNodeIDs.end()) { + _ignoringNodeIDs.push_back(nodeID); + } +} + +void AudioMixerClientData::unignoredByNode(QUuid nodeID) { + // first add this ID to the concurrent vector for newly unignoring nodes + _newUnignoringNodeIDs.push_back(nodeID); + + // now take a lock on the consistent vector of ignoring nodes and make sure this node isn't in it + std::lock_guard lock(_ignoringNodeIDsMutex); + auto it = _ignoringNodeIDs.begin(); + while (it != _ignoringNodeIDs.end()) { + if (*it == nodeID) { + it = _ignoringNodeIDs.erase(it); + } else { + ++it; + } + } +} + +void AudioMixerClientData::clearStagedIgnoreChanges() { + _newIgnoredNodeIDs.clear(); + _newUnignoredNodeIDs.clear(); + _newIgnoringNodeIDs.clear(); + _newUnignoringNodeIDs.clear(); } void AudioMixerClientData::parseRadiusIgnoreRequest(QSharedPointer message, const SharedNodePointer& node) { - node->parseIgnoreRadiusRequestMessage(message); + bool enabled; + message->readPrimitive(&enabled); + + _isIgnoreRadiusEnabled = enabled; + + auto avatarAudioStream = getAvatarAudioStream(); + + // if we have an avatar audio stream, tell it wether its ignore box should be enabled or disabled + if (avatarAudioStream) { + if (_isIgnoreRadiusEnabled) { + avatarAudioStream->enableIgnoreBox(); + } else { + avatarAudioStream->disableIgnoreBox(); + } + } } AvatarAudioStream* AudioMixerClientData::getAvatarAudioStream() { @@ -226,40 +304,6 @@ AvatarAudioStream* AudioMixerClientData::getAvatarAudioStream() { return NULL; } -AudioHRTF& AudioMixerClientData::hrtfForStream(Node::LocalID nodeID, const QUuid& streamID) { - auto& hrtfVector = _nodeSourcesHRTFMap[nodeID]; - - auto streamIt = std::find_if(hrtfVector.begin(), hrtfVector.end(), [&streamID](IdentifiedHRTF& identifiedHRTF){ - return identifiedHRTF.streamIdentifier == streamID; - }); - - if (streamIt == hrtfVector.end()) { - hrtfVector.push_back({ streamID, std::unique_ptr(new AudioHRTF) }); - - return *hrtfVector.back().hrtf; - } else { - return *streamIt->hrtf; - } -} - -void AudioMixerClientData::removeHRTFForStream(Node::LocalID nodeID, const QUuid& streamID) { - auto it = _nodeSourcesHRTFMap.find(nodeID); - if (it != _nodeSourcesHRTFMap.end()) { - auto streamIt = std::find_if(it->second.begin(), it->second.end(), [&streamID](IdentifiedHRTF& identifiedHRTF){ - return identifiedHRTF.streamIdentifier == streamID; - }); - - // erase the stream with the given ID from the given node - it->second.erase(streamIt); - - // is the map for this node now empty? - // if so we can remove it - if (it->second.size() == 0) { - _nodeSourcesHRTFMap.erase(it); - } - } -} - void AudioMixerClientData::removeAgentAvatarAudioStream() { QWriteLocker writeLocker { &_streamsLock }; @@ -283,112 +327,127 @@ int AudioMixerClientData::parseData(ReceivedMessage& message) { message.readPrimitive(&_downstreamAudioStreamStats); return message.getPosition(); + } - } else { - SharedStreamPointer matchingStream; + return 0; +} - bool isMicStream = false; +void AudioMixerClientData::processStreamPacket(ReceivedMessage& message, ConcurrentAddedStreams &addedStreams) { + SharedStreamPointer matchingStream; - if (packetType == PacketType::MicrophoneAudioWithEcho - || packetType == PacketType::ReplicatedMicrophoneAudioWithEcho - || packetType == PacketType::MicrophoneAudioNoEcho - || packetType == PacketType::ReplicatedMicrophoneAudioNoEcho - || packetType == PacketType::SilentAudioFrame - || packetType == PacketType::ReplicatedSilentAudioFrame) { + auto packetType = message.getType(); + bool newStream = false; - QWriteLocker writeLocker { &_streamsLock }; + if (packetType == PacketType::MicrophoneAudioWithEcho + || packetType == PacketType::ReplicatedMicrophoneAudioWithEcho + || packetType == PacketType::MicrophoneAudioNoEcho + || packetType == PacketType::ReplicatedMicrophoneAudioNoEcho + || packetType == PacketType::SilentAudioFrame + || packetType == PacketType::ReplicatedSilentAudioFrame) { - auto micStreamIt = std::find_if(_audioStreams.begin(), _audioStreams.end(), [](const SharedStreamPointer& stream){ - return stream->getStreamIdentifier().isNull(); - }); - if (micStreamIt == _audioStreams.end()) { - // we don't have a mic stream yet, so add it + QWriteLocker writeLocker { &_streamsLock }; - // hop past the sequence number that leads the packet - message.seek(sizeof(quint16)); + auto micStreamIt = std::find_if(_audioStreams.begin(), _audioStreams.end(), [](const SharedStreamPointer& stream){ + return stream->getStreamIdentifier().isNull(); + }); - // pull the codec string from the packet - auto codecString = message.readString(); + if (micStreamIt == _audioStreams.end()) { + // we don't have a mic stream yet, so add it - // determine if the stream is stereo or not - bool isStereo; - if (packetType == PacketType::SilentAudioFrame - || packetType == PacketType::ReplicatedSilentAudioFrame) { - quint16 numSilentSamples; - message.readPrimitive(&numSilentSamples); - isStereo = numSilentSamples == AudioConstants::NETWORK_FRAME_SAMPLES_STEREO; - } else { - quint8 channelFlag; - message.readPrimitive(&channelFlag); - isStereo = channelFlag == 1; - } - - auto avatarAudioStream = new AvatarAudioStream(isStereo, AudioMixer::getStaticJitterFrames()); - avatarAudioStream->setupCodec(_codec, _selectedCodecName, isStereo ? AudioConstants::STEREO : AudioConstants::MONO); - qCDebug(audio) << "creating new AvatarAudioStream... codec:" << _selectedCodecName << "isStereo:" << isStereo; - - connect(avatarAudioStream, &InboundAudioStream::mismatchedAudioCodec, - this, &AudioMixerClientData::handleMismatchAudioFormat); - - matchingStream = SharedStreamPointer(avatarAudioStream); - _audioStreams.push_back(matchingStream); - } else { - matchingStream = *micStreamIt; - } - - writeLocker.unlock(); - - isMicStream = true; - } else if (packetType == PacketType::InjectAudio - || packetType == PacketType::ReplicatedInjectAudio) { - // this is injected audio - // grab the stream identifier for this injected audio + // hop past the sequence number that leads the packet message.seek(sizeof(quint16)); - QUuid streamIdentifier = QUuid::fromRfc4122(message.readWithoutCopy(NUM_BYTES_RFC4122_UUID)); + // pull the codec string from the packet + auto codecString = message.readString(); + // determine if the stream is stereo or not bool isStereo; - message.readPrimitive(&isStereo); - - QWriteLocker writeLock { &_streamsLock }; - - auto streamIt = std::find_if(_audioStreams.begin(), _audioStreams.end(), [&streamIdentifier](const SharedStreamPointer& stream) { - return stream->getStreamIdentifier() == streamIdentifier; - }); - - if (streamIt == _audioStreams.end()) { - // we don't have this injected stream yet, so add it - auto injectorStream = new InjectedAudioStream(streamIdentifier, isStereo, AudioMixer::getStaticJitterFrames()); - -#if INJECTORS_SUPPORT_CODECS - injectorStream->setupCodec(_codec, _selectedCodecName, isStereo ? AudioConstants::STEREO : AudioConstants::MONO); - qCDebug(audio) << "creating new injectorStream... codec:" << _selectedCodecName << "isStereo:" << isStereo; -#endif - - matchingStream = SharedStreamPointer(injectorStream); - _audioStreams.push_back(matchingStream); + if (packetType == PacketType::SilentAudioFrame || packetType == PacketType::ReplicatedSilentAudioFrame) { + quint16 numSilentSamples; + message.readPrimitive(&numSilentSamples); + isStereo = numSilentSamples == AudioConstants::NETWORK_FRAME_SAMPLES_STEREO; } else { - matchingStream = *streamIt; + quint8 channelFlag; + message.readPrimitive(&channelFlag); + isStereo = channelFlag == 1; } - writeLock.unlock(); + auto avatarAudioStream = new AvatarAudioStream(isStereo, AudioMixer::getStaticJitterFrames()); + avatarAudioStream->setupCodec(_codec, _selectedCodecName, isStereo ? AudioConstants::STEREO : AudioConstants::MONO); + + if (_isIgnoreRadiusEnabled) { + avatarAudioStream->enableIgnoreBox(); + } else { + avatarAudioStream->disableIgnoreBox(); + } + + qCDebug(audio) << "creating new AvatarAudioStream... codec:" << _selectedCodecName << "isStereo:" << isStereo; + + connect(avatarAudioStream, &InboundAudioStream::mismatchedAudioCodec, + this, &AudioMixerClientData::handleMismatchAudioFormat); + + matchingStream = SharedStreamPointer(avatarAudioStream); + _audioStreams.push_back(matchingStream); + + newStream = true; + } else { + matchingStream = *micStreamIt; } - // seek to the beginning of the packet so that the next reader is in the right spot - message.seek(0); + writeLocker.unlock(); + } else if (packetType == PacketType::InjectAudio + || packetType == PacketType::ReplicatedInjectAudio) { + // this is injected audio + // grab the stream identifier for this injected audio + message.seek(sizeof(quint16)); - // check the overflow count before we parse data - auto overflowBefore = matchingStream->getOverflowCount(); - auto parseResult = matchingStream->parseData(message); + QUuid streamIdentifier = QUuid::fromRfc4122(message.readWithoutCopy(NUM_BYTES_RFC4122_UUID)); - if (matchingStream->getOverflowCount() > overflowBefore) { - qCDebug(audio) << "Just overflowed on stream from" << message.getSourceID() << "at" << message.getSenderSockAddr(); - qCDebug(audio) << "This stream is for" << (isMicStream ? "microphone audio" : "injected audio"); + bool isStereo; + message.readPrimitive(&isStereo); + + QWriteLocker writeLock { &_streamsLock }; + + auto streamIt = std::find_if(_audioStreams.begin(), _audioStreams.end(), [&streamIdentifier](const SharedStreamPointer& stream) { + return stream->getStreamIdentifier() == streamIdentifier; + }); + + if (streamIt == _audioStreams.end()) { + // we don't have this injected stream yet, so add it + auto injectorStream = new InjectedAudioStream(streamIdentifier, isStereo, AudioMixer::getStaticJitterFrames()); + +#if INJECTORS_SUPPORT_CODECS + injectorStream->setupCodec(_codec, _selectedCodecName, isStereo ? AudioConstants::STEREO : AudioConstants::MONO); + qCDebug(audio) << "creating new injectorStream... codec:" << _selectedCodecName << "isStereo:" << isStereo; +#endif + + matchingStream = SharedStreamPointer(injectorStream); + _audioStreams.push_back(matchingStream); + + newStream = true; + } else { + matchingStream = *streamIt; } - return parseResult; + writeLock.unlock(); + } + + // seek to the beginning of the packet so that the next reader is in the right spot + message.seek(0); + + // check the overflow count before we parse data + auto overflowBefore = matchingStream->getOverflowCount(); + matchingStream->parseData(message); + + if (matchingStream->getOverflowCount() > overflowBefore) { + qCDebug(audio) << "Just overflowed on stream" << matchingStream->getStreamIdentifier() + << "from" << message.getSourceID(); + } + + if (newStream) { + // whenever a stream is added, push it to the concurrent vector of streams added this frame + addedStreams.emplace_back(getNodeID(), getNodeLocalID(), matchingStream->getStreamIdentifier(), matchingStream.get()); } - return 0; } int AudioMixerClientData::checkBuffersBeforeFrameSend() { @@ -632,74 +691,6 @@ void AudioMixerClientData::cleanupCodec() { } } -AudioMixerClientData::IgnoreZone& AudioMixerClientData::IgnoreZoneMemo::get(unsigned int frame) { - // check for a memoized zone - if (frame != _frame.load(std::memory_order_acquire)) { - AvatarAudioStream* stream = _data.getAvatarAudioStream(); - - // get the initial dimensions from the stream - glm::vec3 corner = stream ? stream->getAvatarBoundingBoxCorner() : glm::vec3(0); - glm::vec3 scale = stream ? stream->getAvatarBoundingBoxScale() : glm::vec3(0); - - // enforce a minimum scale - static const glm::vec3 MIN_IGNORE_BOX_SCALE = glm::vec3(0.3f, 1.3f, 0.3f); - if (glm::any(glm::lessThan(scale, MIN_IGNORE_BOX_SCALE))) { - scale = MIN_IGNORE_BOX_SCALE; - } - - // (this is arbitrary number determined empirically for comfort) - const float IGNORE_BOX_SCALE_FACTOR = 2.4f; - scale *= IGNORE_BOX_SCALE_FACTOR; - - // create the box (we use a box for the zone for convenience) - AABox box(corner, scale); - - // update the memoized zone - // This may be called by multiple threads concurrently, - // so take a lock and only update the memo if this call is first. - // This prevents concurrent updates from invalidating the returned reference - // (contingent on the preconditions listed in the header). - std::lock_guard lock(_mutex); - if (frame != _frame.load(std::memory_order_acquire)) { - _zone = box; - unsigned int oldFrame = _frame.exchange(frame, std::memory_order_release); - Q_UNUSED(oldFrame); - } - } - - return _zone; -} - -bool AudioMixerClientData::shouldIgnore(const SharedNodePointer self, const SharedNodePointer node, unsigned int frame) { - // this is symmetric over self / node; if computed, it is cached in the other - - AudioMixerClientData* nodeData = static_cast(node->getLinkedData()); - if (!nodeData) { - return false; - } - - // compute shouldIgnore - bool shouldIgnore = true; - if ( // the nodes are not ignoring each other explicitly (or are but get data regardless) - (!self->isIgnoringNodeWithID(node->getUUID()) || - (nodeData->getRequestsDomainListData() && node->getCanKick())) && - (!node->isIgnoringNodeWithID(self->getUUID()) || - (getRequestsDomainListData() && self->getCanKick()))) { - - // if either node is enabling an ignore radius, check their proximity - if ((self->isIgnoreRadiusEnabled() || node->isIgnoreRadiusEnabled())) { - auto& zone = _ignoreZone.get(frame); - auto& nodeZone = nodeData->_ignoreZone.get(frame); - shouldIgnore = zone.touches(nodeZone); - } else { - shouldIgnore = false; - } - } - - - return shouldIgnore; -} - void AudioMixerClientData::setupCodecForReplicatedAgent(QSharedPointer message) { // hop past the sequence number that leads the packet message->seek(sizeof(quint16)); diff --git a/assignment-client/src/audio/AudioMixerClientData.h b/assignment-client/src/audio/AudioMixerClientData.h index a35f3dea5a..7de5309717 100644 --- a/assignment-client/src/audio/AudioMixerClientData.h +++ b/assignment-client/src/audio/AudioMixerClientData.h @@ -14,6 +14,8 @@ #include +#include + #include #include @@ -30,6 +32,17 @@ class AudioMixerClientData : public NodeData { Q_OBJECT public: + struct AddedStream { + NodeIDStreamID nodeIDStreamID; + PositionalAudioStream* positionalStream; + + AddedStream(QUuid nodeID, Node::LocalID localNodeID, + StreamID streamID, PositionalAudioStream* positionalStream) : + nodeIDStreamID(nodeID, localNodeID, streamID), positionalStream(positionalStream) {}; + }; + + using ConcurrentAddedStreams = tbb::concurrent_vector; + AudioMixerClientData(const QUuid& nodeID, Node::LocalID nodeLocalID); ~AudioMixerClientData(); @@ -37,31 +50,16 @@ public: using AudioStreamVector = std::vector; void queuePacket(QSharedPointer packet, SharedNodePointer node); - void processPackets(); + void processPackets(ConcurrentAddedStreams& addedStreams); AudioStreamVector& getAudioStreams() { return _audioStreams; } AvatarAudioStream* getAvatarAudioStream(); - // returns whether self (this data's node) should ignore node, memoized by frame - // precondition: frame is increasing after first call (including overflow wrap) - bool shouldIgnore(SharedNodePointer self, SharedNodePointer node, unsigned int frame); - - // the following methods should be called from the AudioMixer assignment thread ONLY - // they are not thread-safe - - // returns a new or existing HRTF object for the given stream from the given node - AudioHRTF& hrtfForStream(Node::LocalID nodeID, const QUuid& streamID = QUuid()); - - // removes an AudioHRTF object for a given stream - void removeHRTFForStream(Node::LocalID nodeID, const QUuid& streamID = QUuid()); - - // remove all sources and data from this node - void removeNode(Node::LocalID nodeID) { _nodeSourcesHRTFMap.erase(nodeID); } - void removeAgentAvatarAudioStream(); // packet parsers int parseData(ReceivedMessage& message) override; + void processStreamPacket(ReceivedMessage& message, ConcurrentAddedStreams& addedStreams); void negotiateAudioFormat(ReceivedMessage& message, const SharedNodePointer& node); void parseRequestsDomainListData(ReceivedMessage& message); void parsePerAvatarGainSet(ReceivedMessage& message, const SharedNodePointer& node); @@ -75,9 +73,6 @@ public: QJsonObject getAudioStreamStats(); - void setNodeLocalID(Node::LocalID localNodeID) { _localNodeID = localNodeID; } - Node::LocalID getNodeLocalID() { return _localNodeID; } - void sendAudioStreamStatsPackets(const SharedNodePointer& destinationNode); void incrementOutgoingMixedAudioSequenceNumber() { _outgoingMixedAudioSequenceNumber++; } @@ -115,6 +110,48 @@ public: void setupCodecForReplicatedAgent(QSharedPointer message); + struct MixableStream { + float approximateVolume { 0.0f }; + NodeIDStreamID nodeStreamID; + std::unique_ptr hrtf; + PositionalAudioStream* positionalStream; + bool ignoredByListener { false }; + bool ignoringListener { false }; + bool completedSilentRender { false }; + bool skippedStream { false }; + + MixableStream(NodeIDStreamID nodeIDStreamID, PositionalAudioStream* positionalStream) : + nodeStreamID(nodeIDStreamID), hrtf(new AudioHRTF), positionalStream(positionalStream) {}; + MixableStream(QUuid nodeID, Node::LocalID localNodeID, StreamID streamID, PositionalAudioStream* positionalStream) : + nodeStreamID(nodeID, localNodeID, streamID), hrtf(new AudioHRTF), positionalStream(positionalStream) {}; + }; + + using MixableStreamsVector = std::vector; + + MixableStreamsVector& getMixableStreams() { return _mixableStreams; } + + // thread-safe, called from AudioMixerSlave(s) while processing ignore packets for other nodes + void ignoredByNode(QUuid nodeID); + void unignoredByNode(QUuid nodeID); + + // start of methods called non-concurrently from single AudioMixerSlave mixing for the owning node + + const Node::IgnoredNodeIDs& getNewIgnoredNodeIDs() const { return _newIgnoredNodeIDs; } + const Node::IgnoredNodeIDs& getNewUnignoredNodeIDs() const { return _newUnignoredNodeIDs; } + + using ConcurrentIgnoreNodeIDs = tbb::concurrent_vector; + const ConcurrentIgnoreNodeIDs& getNewIgnoringNodeIDs() const { return _newIgnoringNodeIDs; } + const ConcurrentIgnoreNodeIDs& getNewUnignoringNodeIDs() const { return _newUnignoringNodeIDs; } + + void clearStagedIgnoreChanges(); + + const Node::IgnoredNodeIDs& getIgnoringNodeIDs() const { return _ignoringNodeIDs; } + + bool getHasReceivedFirstMix() const { return _hasReceivedFirstMix; } + void setHasReceivedFirstMix(bool hasReceivedFirstMix) { _hasReceivedFirstMix = hasReceivedFirstMix; } + + // end of methods called non-concurrently from single AudioMixerSlave + signals: void injectorStreamFinished(const QUuid& streamIdentifier); @@ -133,33 +170,9 @@ private: void optionallyReplicatePacket(ReceivedMessage& packet, const Node& node); - using IgnoreZone = AABox; - class IgnoreZoneMemo { - public: - IgnoreZoneMemo(AudioMixerClientData& data) : _data(data) {} + void setGainForAvatar(QUuid nodeID, uint8_t gain); - // returns an ignore zone, memoized by frame (lockless if the zone is already memoized) - // preconditions: - // - frame is increasing after first call (including overflow wrap) - // - there are no references left from calls to getIgnoreZone(frame - 1) - IgnoreZone& get(unsigned int frame); - - private: - AudioMixerClientData& _data; - IgnoreZone _zone; - std::atomic _frame { 0 }; - std::mutex _mutex; - }; - IgnoreZoneMemo _ignoreZone; - - struct IdentifiedHRTF { - QUuid streamIdentifier; - std::unique_ptr hrtf; - }; - - using HRTFVector = std::vector; - using NodeSourcesHRTFMap = std::unordered_map; - NodeSourcesHRTFMap _nodeSourcesHRTFMap; + MixableStreamsVector _mixableStreams; quint16 _outgoingMixedAudioSequenceNumber; @@ -179,7 +192,20 @@ private: bool _shouldMuteClient { false }; bool _requestsDomainListData { false }; - Node::LocalID _localNodeID; + std::vector _newAddedStreams; + + Node::IgnoredNodeIDs _newIgnoredNodeIDs; + Node::IgnoredNodeIDs _newUnignoredNodeIDs; + + tbb::concurrent_vector _newIgnoringNodeIDs; + tbb::concurrent_vector _newUnignoringNodeIDs; + + std::mutex _ignoringNodeIDsMutex; + Node::IgnoredNodeIDs _ignoringNodeIDs; + + std::atomic_bool _isIgnoreRadiusEnabled { false }; + + bool _hasReceivedFirstMix { false }; }; #endif // hifi_AudioMixerClientData_h diff --git a/assignment-client/src/audio/AudioMixerSlave.cpp b/assignment-client/src/audio/AudioMixerSlave.cpp index d350f2b27a..547043c0f0 100644 --- a/assignment-client/src/audio/AudioMixerSlave.cpp +++ b/assignment-client/src/audio/AudioMixerSlave.cpp @@ -46,9 +46,8 @@ void sendMutePacket(const SharedNodePointer& node, AudioMixerClientData&); void sendEnvironmentPacket(const SharedNodePointer& node, AudioMixerClientData& data); // mix helpers -inline float approximateGain(const AvatarAudioStream& listeningNodeStream, const PositionalAudioStream& streamToAdd, - const glm::vec3& relativePosition); -inline float computeGain(const AudioMixerClientData& listenerNodeData, const AvatarAudioStream& listeningNodeStream, +inline float approximateGain(const AvatarAudioStream& listeningNodeStream, const PositionalAudioStream& streamToAdd); +inline float computeGain(float masterListenerGain, const AvatarAudioStream& listeningNodeStream, const PositionalAudioStream& streamToAdd, const glm::vec3& relativePosition, float distance, bool isEcho); inline float computeAzimuth(const AvatarAudioStream& listeningNodeStream, const PositionalAudioStream& streamToAdd, const glm::vec3& relativePosition); @@ -56,7 +55,7 @@ inline float computeAzimuth(const AvatarAudioStream& listeningNodeStream, const void AudioMixerSlave::processPackets(const SharedNodePointer& node) { AudioMixerClientData* data = (AudioMixerClientData*)node->getLinkedData(); if (data) { - data->processPackets(); + data->processPackets(_sharedData.addedStreams); } } @@ -125,6 +124,13 @@ void AudioMixerSlave::mix(const SharedNodePointer& node) { } } +template +bool containsNodeID(const V& vector, QUuid nodeID) { + return std::any_of(std::begin(vector), std::end(vector), [&nodeID](const QUuid& vectorID){ + return vectorID == nodeID; + }); +} + bool AudioMixerSlave::prepareMix(const SharedNodePointer& listener) { AvatarAudioStream* listenerAudioStream = static_cast(listener->getLinkedData())->getAvatarAudioStream(); AudioMixerClientData* listenerData = static_cast(listener->getLinkedData()); @@ -139,87 +145,185 @@ bool AudioMixerSlave::prepareMix(const SharedNodePointer& listener) { memset(_mixSamples, 0, sizeof(_mixSamples)); bool isThrottling = _throttlingRatio > 0.0f; - std::vector> throttledNodes; - typedef void (AudioMixerSlave::*MixFunctor)( - AudioMixerClientData&, Node::LocalID, const AvatarAudioStream&, const PositionalAudioStream&); - auto forAllStreams = [&](const SharedNodePointer& node, AudioMixerClientData* nodeData, MixFunctor mixFunctor) { - auto nodeID = node->getLocalID(); - for (auto& streamPair : nodeData->getAudioStreams()) { - (this->*mixFunctor)(*listenerData, nodeID, *listenerAudioStream, *streamPair); - } - }; + auto nodeList = DependencyManager::get(); #ifdef HIFI_AUDIO_MIXER_DEBUG auto mixStart = p_high_resolution_clock::now(); #endif - std::for_each(_begin, _end, [&](const SharedNodePointer& node) { - AudioMixerClientData* nodeData = static_cast(node->getLinkedData()); - if (!nodeData) { - return; - } + auto& mixableStreams = listenerData->getMixableStreams(); + auto& ignoredNodeIDs = listener->getIgnoredNodeIDs(); + auto& ignoringNodeIDs = listenerData->getIgnoringNodeIDs(); - if (*node == *listener) { - // only mix the echo, if requested - for (auto& streamPair : nodeData->getAudioStreams()) { - if (streamPair->shouldLoopbackForNode()) { - mixStream(*listenerData, node->getLocalID(), *listenerAudioStream, *streamPair); + // add data for newly created streams to our vector + if (!listenerData->getHasReceivedFirstMix()) { + // when this listener is new, we need to fill its added streams object with all available streams + std::for_each(_begin, _end, [&](const SharedNodePointer& node) { + AudioMixerClientData* nodeData = static_cast(node->getLinkedData()); + if (nodeData) { + for (auto& stream : nodeData->getAudioStreams()) { + mixableStreams.emplace_back(node->getUUID(), node->getLocalID(), + stream->getStreamIdentifier(), &(*stream)); + + // pre-populate ignored and ignoring flags for this stream + mixableStreams.back().ignoredByListener = containsNodeID(ignoredNodeIDs, node->getUUID()); + mixableStreams.back().ignoringListener = containsNodeID(ignoringNodeIDs, node->getUUID()); } } - } else if (!listenerData->shouldIgnore(listener, node, _frame)) { - if (!isThrottling) { - forAllStreams(node, nodeData, &AudioMixerSlave::mixStream); - } else { + }); - // compute the node's max relative volume - float nodeVolume = 0.0f; - for (auto& nodeStream : nodeData->getAudioStreams()) { + // flag this listener as having received their first mix so we know we don't need to enumerate all nodes again + listenerData->setHasReceivedFirstMix(true); + } else { + for (const auto& newStream : _sharedData.addedStreams) { + mixableStreams.emplace_back(newStream.nodeIDStreamID, newStream.positionalStream); - // approximate the gain - glm::vec3 relativePosition = nodeStream->getPosition() - listenerAudioStream->getPosition(); - float gain = approximateGain(*listenerAudioStream, *nodeStream, relativePosition); - - // modify by hrtf gain adjustment - auto& hrtf = listenerData->hrtfForStream(node->getLocalID(), nodeStream->getStreamIdentifier()); - gain *= hrtf.getGainAdjustment(); - - auto streamVolume = nodeStream->getLastPopOutputTrailingLoudness() * gain; - nodeVolume = std::max(streamVolume, nodeVolume); - } - - // max-heapify the nodes by relative volume - throttledNodes.push_back({ nodeVolume, node }); - std::push_heap(throttledNodes.begin(), throttledNodes.end()); - } - } - }); - - if (isThrottling) { - // pop the loudest nodes off the heap and mix their streams - int numToRetain = (int)(std::distance(_begin, _end) * (1 - _throttlingRatio)); - for (int i = 0; i < numToRetain; i++) { - if (throttledNodes.empty()) { - break; - } - - std::pop_heap(throttledNodes.begin(), throttledNodes.end()); - - auto& node = throttledNodes.back().second; - AudioMixerClientData* nodeData = static_cast(node->getLinkedData()); - forAllStreams(node, nodeData, &AudioMixerSlave::mixStream); - - throttledNodes.pop_back(); - } - - // throttle the remaining nodes' streams - for (const std::pair& nodePair : throttledNodes) { - auto& node = nodePair.second; - AudioMixerClientData* nodeData = static_cast(node->getLinkedData()); - forAllStreams(node, nodeData, &AudioMixerSlave::throttleStream); + // pre-populate ignored and ignoring flags for this stream + mixableStreams.back().ignoredByListener = containsNodeID(ignoredNodeIDs, newStream.nodeIDStreamID.nodeID); + mixableStreams.back().ignoringListener = containsNodeID(ignoringNodeIDs, newStream.nodeIDStreamID.nodeID); } } + // grab the unprocessed ignores and unignores from and for this listener + const auto& nodesIgnoredByListener = listenerData->getNewIgnoredNodeIDs(); + const auto& nodesUnignoredByListener = listenerData->getNewUnignoredNodeIDs(); + const auto& nodesIgnoringListener = listenerData->getNewIgnoringNodeIDs(); + const auto& nodesUnignoringListener = listenerData->getNewUnignoringNodeIDs(); + + // enumerate the available streams + auto it = mixableStreams.begin(); + auto end = mixableStreams.end(); + while (it != end) { + // check if this node (and therefore all of the node's streams) has been removed + auto& nodeIDStreamID = it->nodeStreamID; + auto matchedRemovedNode = std::find(_sharedData.removedNodes.cbegin(), _sharedData.removedNodes.cend(), + nodeIDStreamID.nodeLocalID); + bool streamRemoved = matchedRemovedNode != _sharedData.removedNodes.cend(); + + // if the node wasn't removed, check if this stream was specifically removed + if (!streamRemoved) { + auto matchedRemovedStream = std::find(_sharedData.removedStreams.cbegin(), _sharedData.removedStreams.cend(), + nodeIDStreamID); + streamRemoved = matchedRemovedStream != _sharedData.removedStreams.cend(); + } + + if (streamRemoved) { + // this stream was removed, so swap it with the last item and decrease the end iterator + --end; + std::swap(*it, *end); + + // process the it element (which is now the element that was the last item before the swap) + continue; + } + + if (it->nodeStreamID.nodeLocalID == listener->getLocalID()) { + // streams from this node should be skipped unless loopback is specifically requested + if (it->positionalStream->shouldLoopbackForNode()) { + it->skippedStream = false; + } else { + it->approximateVolume = 0.0f; + it->skippedStream = true; + it->completedSilentRender = true; + + // if we know we're skipping this stream, no more processing is required + // since we don't do silent HRTF renders for echo streams + ++it; + continue; + } + } else { + if (it->ignoredByListener && nodesUnignoredByListener.size() > 0) { + // this stream was previously ignored by the listener and we have some unignored streams + // check now if it is one of the unignored streams and flag it as such + it->ignoredByListener = !containsNodeID(nodesUnignoredByListener, nodeIDStreamID.nodeID); + + } else if (!it->ignoredByListener && nodesIgnoredByListener.size() > 0) { + // this stream was previously not ignored by the listener and we have some newly ignored streams + // check now if it is one of the ignored streams and flag it as such + it->ignoredByListener = containsNodeID(nodesIgnoredByListener, nodeIDStreamID.nodeID); + } + + if (it->ignoringListener && nodesUnignoringListener.size() > 0) { + // this stream was previously ignoring the listener and we have some new un-ignoring nodes + // check now if it is one of the unignoring streams and flag it as such + it->ignoringListener = !containsNodeID(nodesUnignoringListener, nodeIDStreamID.nodeID); + } else if (!it->ignoringListener && nodesIgnoringListener.size() > 0) { + it->ignoringListener = containsNodeID(nodesIgnoringListener, nodeIDStreamID.nodeID); + } + + if (it->ignoredByListener + || (it->ignoringListener && !(listenerData->getRequestsDomainListData() && listener->getCanKick()))) { + // this is a stream ignoring by the listener + // or ignoring the listener (and the listener is not an admin asking for (the poorly named) "domain list" data) + // mark it skipped and move on + it->skippedStream = true; + } else { + it->skippedStream = false; + } + + if (!it->skippedStream) { + if ((listenerAudioStream->isIgnoreBoxEnabled() || it->positionalStream->isIgnoreBoxEnabled()) + && listenerAudioStream->getIgnoreBox().touches(it->positionalStream->getIgnoreBox())) { + // the listener is ignoring audio sources within a radius, and this source is in that radius + // so we mark it skipped + it->skippedStream = true; + } else { + it->skippedStream = false; + } + } + } + + if (!isThrottling) { + // we aren't throttling, so we already know that we can add this stream to the mix + addStream(*it, *listenerAudioStream, listenerData->getMasterAvatarGain(), false); + } else { + // we're throttling, so we need to update the approximate volume for any un-skipped streams + // unless this is simply for an echo (in which case the approx volume is 1.0) + if (!it->skippedStream) { + if (it->positionalStream != listenerAudioStream) { + // approximate the gain + float gain = approximateGain(*listenerAudioStream, *(it->positionalStream)); + + // for avatar streams, modify by the set gain adjustment + if (nodeIDStreamID.streamID.isNull()) { + gain *= it->hrtf->getGainAdjustment(); + } + + it->approximateVolume = it->positionalStream->getLastPopOutputTrailingLoudness() * gain; + } else { + it->approximateVolume = 1.0f; + } + } else { + it->approximateVolume = 0.0f; + } + } + + ++it; + } + + // erase any removed streams that were swapped to the end + mixableStreams.erase(end, mixableStreams.end()); + + if (isThrottling) { + // since we're throttling, we need to partition the mixable into throttled and unthrottled streams + auto numToRetain = std::distance(_begin, _end) * (1 - _throttlingRatio); + auto throttlePoint = mixableStreams.begin() + numToRetain; + + std::nth_element(mixableStreams.begin(), throttlePoint, mixableStreams.end(), + [](const auto& a, const auto& b) + { + return a.approximateVolume > b.approximateVolume; + }); + + for (auto it = mixableStreams.begin(); it != mixableStreams.end(); ++it) { + // add this stream, it is throttled if it is at or past the throttle iterator in the vector + addStream(*it, *listenerAudioStream, listenerData->getMasterAvatarGain(), it >= throttlePoint); + } + } + + // clear the newly ignored, un-ignored, ignoring, and un-ignoring streams now that we've processed them + listenerData->clearStagedIgnoreChanges(); + #ifdef HIFI_AUDIO_MIXER_DEBUG auto mixEnd = p_high_resolution_clock::now(); auto mixTime = std::chrono::duration_cast(mixEnd - mixStart); @@ -242,51 +346,59 @@ bool AudioMixerSlave::prepareMix(const SharedNodePointer& listener) { return hasAudio; } -void AudioMixerSlave::throttleStream(AudioMixerClientData& listenerNodeData, Node::LocalID sourceNodeLocalID, - const AvatarAudioStream& listeningNodeStream, const PositionalAudioStream& streamToAdd) { - // only throttle this stream to the mix if it has a valid position, we won't know how to mix it otherwise - if (streamToAdd.hasValidPosition()) { - addStream(listenerNodeData, sourceNodeLocalID, listeningNodeStream, streamToAdd, true); - } -} +void AudioMixerSlave::addStream(AudioMixerClientData::MixableStream& mixableStream, AvatarAudioStream& listeningNodeStream, + float masterListenerGain, bool throttle) { -void AudioMixerSlave::mixStream(AudioMixerClientData& listenerNodeData, Node::LocalID sourceNodeLocalID, - const AvatarAudioStream& listeningNodeStream, const PositionalAudioStream& streamToAdd) { - // only add the stream to the mix if it has a valid position, we won't know how to mix it otherwise - if (streamToAdd.hasValidPosition()) { - addStream(listenerNodeData, sourceNodeLocalID, listeningNodeStream, streamToAdd, false); + if (mixableStream.skippedStream) { + // any skipped stream gets no processing and no silent render - early return + return; } -} -void AudioMixerSlave::addStream(AudioMixerClientData& listenerNodeData, Node::LocalID sourceNodeLocalID, - const AvatarAudioStream& listeningNodeStream, const PositionalAudioStream& streamToAdd, - bool throttle) { ++stats.totalMixes; - // to reduce artifacts we call the HRTF functor for every source, even if throttled or silent + auto streamToAdd = mixableStream.positionalStream; + + // to reduce artifacts we still call the HRTF functor for every silent or throttled source + // for the first frame where the source becomes throttled or silent // this ensures the correct tail from last mixed block and the correct spatialization of next first block + if (throttle || mixableStream.skippedStream || streamToAdd->getLastPopOutputLoudness() == 0.0f) { + if (mixableStream.completedSilentRender) { + + if (throttle) { + ++stats.hrtfThrottleRenders; + } + + return; + } else { + mixableStream.completedSilentRender = true; + } + } else if (mixableStream.completedSilentRender) { + // a stream that is no longer throttled or silent should have its silent render flag reset to false + // so that we complete a silent render for the stream next time it is throttled or otherwise goes silent + mixableStream.completedSilentRender = false; + } // check if this is a server echo of a source back to itself - bool isEcho = (&streamToAdd == &listeningNodeStream); + bool isEcho = (streamToAdd == &listeningNodeStream); - glm::vec3 relativePosition = streamToAdd.getPosition() - listeningNodeStream.getPosition(); + glm::vec3 relativePosition = streamToAdd->getPosition() - listeningNodeStream.getPosition(); float distance = glm::max(glm::length(relativePosition), EPSILON); - float gain = computeGain(listenerNodeData, listeningNodeStream, streamToAdd, relativePosition, distance, isEcho); + float gain = computeGain(masterListenerGain, listeningNodeStream, *streamToAdd, relativePosition, distance, isEcho); float azimuth = isEcho ? 0.0f : computeAzimuth(listeningNodeStream, listeningNodeStream, relativePosition); const int HRTF_DATASET_INDEX = 1; - if (!streamToAdd.lastPopSucceeded()) { + if (!streamToAdd->lastPopSucceeded()) { bool forceSilentBlock = true; - if (!streamToAdd.getLastPopOutput().isNull()) { - bool isInjector = dynamic_cast(&streamToAdd); + if (!streamToAdd->getLastPopOutput().isNull()) { + bool isInjector = dynamic_cast(streamToAdd); // in an injector, just go silent - the injector has likely ended // in other inputs (microphone, &c.), repeat with fade to avoid the harsh jump to silence if (!isInjector) { // calculate its fade factor, which depends on how many times it's already been repeated. - float fadeFactor = calculateRepeatedFrameFadeFactor(streamToAdd.getConsecutiveNotMixedCount() - 1); + float fadeFactor = calculateRepeatedFrameFadeFactor(streamToAdd->getConsecutiveNotMixedCount() - 1); if (fadeFactor > 0.0f) { // apply the fadeFactor to the gain gain *= fadeFactor; @@ -298,13 +410,10 @@ void AudioMixerSlave::addStream(AudioMixerClientData& listenerNodeData, Node::Lo if (forceSilentBlock) { // call renderSilent with a forced silent block to reduce artifacts // (this is not done for stereo streams since they do not go through the HRTF) - if (!streamToAdd.isStereo() && !isEcho) { - // get the existing listener-source HRTF object, or create a new one - auto& hrtf = listenerNodeData.hrtfForStream(sourceNodeLocalID, streamToAdd.getStreamIdentifier()); - + if (!streamToAdd->isStereo() && !isEcho) { static int16_t silentMonoBlock[AudioConstants::NETWORK_FRAME_SAMPLES_PER_CHANNEL] = {}; - hrtf.renderSilent(silentMonoBlock, _mixSamples, HRTF_DATASET_INDEX, azimuth, distance, gain, - AudioConstants::NETWORK_FRAME_SAMPLES_PER_CHANNEL); + mixableStream.hrtf->renderSilent(silentMonoBlock, _mixSamples, HRTF_DATASET_INDEX, azimuth, distance, gain, + AudioConstants::NETWORK_FRAME_SAMPLES_PER_CHANNEL); ++stats.hrtfSilentRenders; } @@ -314,16 +423,15 @@ void AudioMixerSlave::addStream(AudioMixerClientData& listenerNodeData, Node::Lo } // grab the stream from the ring buffer - AudioRingBuffer::ConstIterator streamPopOutput = streamToAdd.getLastPopOutput(); + AudioRingBuffer::ConstIterator streamPopOutput = streamToAdd->getLastPopOutput(); // stereo sources are not passed through HRTF - if (streamToAdd.isStereo()) { + if (streamToAdd->isStereo()) { // apply the avatar gain adjustment - auto& hrtf = listenerNodeData.hrtfForStream(sourceNodeLocalID, streamToAdd.getStreamIdentifier()); - gain *= hrtf.getGainAdjustment(); + gain *= mixableStream.hrtf->getGainAdjustment(); - const float scale = 1/32768.0f; // int16_t to float + const float scale = 1 / 32768.0f; // int16_t to float for (int i = 0; i < AudioConstants::NETWORK_FRAME_SAMPLES_PER_CHANNEL; i++) { _mixSamples[2*i+0] += (float)streamPopOutput[2*i+0] * gain * scale; @@ -349,15 +457,13 @@ void AudioMixerSlave::addStream(AudioMixerClientData& listenerNodeData, Node::Lo return; } - // get the existing listener-source HRTF object, or create a new one - auto& hrtf = listenerNodeData.hrtfForStream(sourceNodeLocalID, streamToAdd.getStreamIdentifier()); streamPopOutput.readSamples(_bufferSamples, AudioConstants::NETWORK_FRAME_SAMPLES_PER_CHANNEL); - if (streamToAdd.getLastPopOutputLoudness() == 0.0f) { + if (streamToAdd->getLastPopOutputLoudness() == 0.0f || mixableStream.skippedStream) { // call renderSilent to reduce artifacts - hrtf.renderSilent(_bufferSamples, _mixSamples, HRTF_DATASET_INDEX, azimuth, distance, gain, - AudioConstants::NETWORK_FRAME_SAMPLES_PER_CHANNEL); + mixableStream.hrtf->renderSilent(_bufferSamples, _mixSamples, HRTF_DATASET_INDEX, azimuth, distance, gain, + AudioConstants::NETWORK_FRAME_SAMPLES_PER_CHANNEL); ++stats.hrtfSilentRenders; return; @@ -365,19 +471,14 @@ void AudioMixerSlave::addStream(AudioMixerClientData& listenerNodeData, Node::Lo if (throttle) { // call renderSilent with actual frame data and a gain of 0.0f to reduce artifacts - hrtf.renderSilent(_bufferSamples, _mixSamples, HRTF_DATASET_INDEX, azimuth, distance, 0.0f, - AudioConstants::NETWORK_FRAME_SAMPLES_PER_CHANNEL); + mixableStream.hrtf->renderSilent(_bufferSamples, _mixSamples, HRTF_DATASET_INDEX, azimuth, distance, 0.0f, + AudioConstants::NETWORK_FRAME_SAMPLES_PER_CHANNEL); ++stats.hrtfThrottleRenders; return; } - if (streamToAdd.getType() == PositionalAudioStream::Injector) { - // apply per-avatar gain to positional audio injectors, which wouldn't otherwise be affected by PAL sliders - hrtf.setGainAdjustment(listenerNodeData.hrtfForStream(sourceNodeLocalID, QUuid()).getGainAdjustment()); - } - - hrtf.render(_bufferSamples, _mixSamples, HRTF_DATASET_INDEX, azimuth, distance, gain, + mixableStream.hrtf->render(_bufferSamples, _mixSamples, HRTF_DATASET_INDEX, azimuth, distance, gain, AudioConstants::NETWORK_FRAME_SAMPLES_PER_CHANNEL); ++stats.hrtfRenders; @@ -489,8 +590,7 @@ void sendEnvironmentPacket(const SharedNodePointer& node, AudioMixerClientData& } } -float approximateGain(const AvatarAudioStream& listeningNodeStream, const PositionalAudioStream& streamToAdd, - const glm::vec3& relativePosition) { +float approximateGain(const AvatarAudioStream& listeningNodeStream, const PositionalAudioStream& streamToAdd) { float gain = 1.0f; // injector: apply attenuation @@ -501,13 +601,14 @@ float approximateGain(const AvatarAudioStream& listeningNodeStream, const Positi // avatar: skip attenuation - it is too costly to approximate // distance attenuation: approximate, ignore zone-specific attenuations + glm::vec3 relativePosition = streamToAdd.getPosition() - listeningNodeStream.getPosition(); float distance = glm::length(relativePosition); return gain / distance; // avatar: skip master gain - it is constant for all streams } -float computeGain(const AudioMixerClientData& listenerNodeData, const AvatarAudioStream& listeningNodeStream, +float computeGain(float masterListenerGain, const AvatarAudioStream& listeningNodeStream, const PositionalAudioStream& streamToAdd, const glm::vec3& relativePosition, float distance, bool isEcho) { float gain = 1.0f; @@ -530,7 +631,7 @@ float computeGain(const AudioMixerClientData& listenerNodeData, const AvatarAudi gain *= offAxisCoefficient; // apply master gain, only to avatars - gain *= listenerNodeData.getMasterAvatarGain(); + gain *= masterListenerGain; } auto& audioZones = AudioMixer::getAudioZones(); diff --git a/assignment-client/src/audio/AudioMixerSlave.h b/assignment-client/src/audio/AudioMixerSlave.h index a0044743b0..da960cb1fd 100644 --- a/assignment-client/src/audio/AudioMixerSlave.h +++ b/assignment-client/src/audio/AudioMixerSlave.h @@ -12,23 +12,33 @@ #ifndef hifi_AudioMixerSlave_h #define hifi_AudioMixerSlave_h +#include + #include #include #include #include #include #include +#include +#include "AudioMixerClientData.h" #include "AudioMixerStats.h" -class PositionalAudioStream; class AvatarAudioStream; class AudioHRTF; -class AudioMixerClientData; class AudioMixerSlave { public: using ConstIter = NodeList::const_iterator; + + struct SharedData { + AudioMixerClientData::ConcurrentAddedStreams addedStreams; + std::vector removedNodes; + std::vector removedStreams; + }; + + AudioMixerSlave(SharedData& sharedData) : _sharedData(sharedData) {}; // process packets for a given node (requires no configuration) void processPackets(const SharedNodePointer& node); @@ -45,13 +55,8 @@ public: private: // create mix, returns true if mix has audio bool prepareMix(const SharedNodePointer& listener); - void throttleStream(AudioMixerClientData& listenerData, Node::LocalID streamerID, - const AvatarAudioStream& listenerStream, const PositionalAudioStream& streamer); - void mixStream(AudioMixerClientData& listenerData, Node::LocalID streamerID, - const AvatarAudioStream& listenerStream, const PositionalAudioStream& streamer); - void addStream(AudioMixerClientData& listenerData, Node::LocalID streamerID, - const AvatarAudioStream& listenerStream, const PositionalAudioStream& streamer, - bool throttle); + void addStream(AudioMixerClientData::MixableStream& mixableStream, AvatarAudioStream& listeningNodeStream, + float masterListenerGain, bool throttle); // mixing buffers float _mixSamples[AudioConstants::NETWORK_FRAME_SAMPLES_STEREO]; @@ -62,6 +67,8 @@ private: ConstIter _end; unsigned int _frame { 0 }; float _throttlingRatio { 0.0f }; + + SharedData& _sharedData; }; #endif // hifi_AudioMixerSlave_h diff --git a/assignment-client/src/audio/AudioMixerSlavePool.cpp b/assignment-client/src/audio/AudioMixerSlavePool.cpp index dfe7ef56aa..d51ba010d7 100644 --- a/assignment-client/src/audio/AudioMixerSlavePool.cpp +++ b/assignment-client/src/audio/AudioMixerSlavePool.cpp @@ -167,7 +167,7 @@ void AudioMixerSlavePool::resize(int numThreads) { if (numThreads > _numThreads) { // start new slaves for (int i = 0; i < numThreads - _numThreads; ++i) { - auto slave = new AudioMixerSlaveThread(*this); + auto slave = new AudioMixerSlaveThread(*this, _workerSharedData); slave->start(); _slaves.emplace_back(slave); } diff --git a/assignment-client/src/audio/AudioMixerSlavePool.h b/assignment-client/src/audio/AudioMixerSlavePool.h index 25047faa89..c9487686b5 100644 --- a/assignment-client/src/audio/AudioMixerSlavePool.h +++ b/assignment-client/src/audio/AudioMixerSlavePool.h @@ -31,7 +31,8 @@ class AudioMixerSlaveThread : public QThread, public AudioMixerSlave { using Lock = std::unique_lock; public: - AudioMixerSlaveThread(AudioMixerSlavePool& pool) : _pool(pool) {} + AudioMixerSlaveThread(AudioMixerSlavePool& pool, AudioMixerSlave::SharedData& sharedData) + : AudioMixerSlave(sharedData), _pool(pool) {} void run() override final; @@ -58,7 +59,8 @@ class AudioMixerSlavePool { public: using ConstIter = NodeList::const_iterator; - AudioMixerSlavePool(int numThreads = QThread::idealThreadCount()) { setNumThreads(numThreads); } + AudioMixerSlavePool(AudioMixerSlave::SharedData& sharedData, int numThreads = QThread::idealThreadCount()) + : _workerSharedData(sharedData) { setNumThreads(numThreads); } ~AudioMixerSlavePool() { resize(0); } // process packets on slave threads @@ -100,6 +102,8 @@ private: float _throttlingRatio { 0.0f }; ConstIter _begin; ConstIter _end; + + AudioMixerSlave::SharedData& _workerSharedData; }; #endif // hifi_AudioMixerSlavePool_h diff --git a/assignment-client/src/avatars/AvatarMixer.cpp b/assignment-client/src/avatars/AvatarMixer.cpp index 561afee296..00cdabe70e 100644 --- a/assignment-client/src/avatars/AvatarMixer.cpp +++ b/assignment-client/src/avatars/AvatarMixer.cpp @@ -673,7 +673,13 @@ void AvatarMixer::handleNodeIgnoreRequestPacket(QSharedPointer void AvatarMixer::handleRadiusIgnoreRequestPacket(QSharedPointer packet, SharedNodePointer sendingNode) { auto start = usecTimestampNow(); - sendingNode->parseIgnoreRadiusRequestMessage(packet); + + bool enabled; + packet->readPrimitive(&enabled); + + auto avatarData = getOrCreateClientData(sendingNode); + avatarData->setIsIgnoreRadiusEnabled(enabled); + auto end = usecTimestampNow(); _handleRadiusIgnoreRequestPacketElapsedTime += (end - start); } diff --git a/assignment-client/src/avatars/AvatarMixerClientData.cpp b/assignment-client/src/avatars/AvatarMixerClientData.cpp index 6c01e6e02b..187c9ed0f2 100644 --- a/assignment-client/src/avatars/AvatarMixerClientData.cpp +++ b/assignment-client/src/avatars/AvatarMixerClientData.cpp @@ -227,7 +227,7 @@ void AvatarMixerClientData::ignoreOther(const Node* self, const Node* other) { addToRadiusIgnoringSet(other->getUUID()); auto killPacket = NLPacket::create(PacketType::KillAvatar, NUM_BYTES_RFC4122_UUID + sizeof(KillAvatarReason), true); killPacket->write(other->getUUID().toRfc4122()); - if (self->isIgnoreRadiusEnabled()) { + if (_isIgnoreRadiusEnabled) { killPacket->writePrimitive(KillAvatarReason::TheirAvatarEnteredYourBubble); } else { killPacket->writePrimitive(KillAvatarReason::YourAvatarEnteredTheirBubble); diff --git a/assignment-client/src/avatars/AvatarMixerClientData.h b/assignment-client/src/avatars/AvatarMixerClientData.h index d38a90ef1f..09d11359c3 100644 --- a/assignment-client/src/avatars/AvatarMixerClientData.h +++ b/assignment-client/src/avatars/AvatarMixerClientData.h @@ -49,6 +49,9 @@ public: const AvatarData* getConstAvatarData() const { return _avatar.get(); } AvatarSharedPointer getAvatarSharedPointer() const { return _avatar; } + bool isIgnoreRadiusEnabled() const { return _isIgnoreRadiusEnabled; } + void setIsIgnoreRadiusEnabled(bool enabled) { _isIgnoreRadiusEnabled = enabled; } + uint16_t getLastBroadcastSequenceNumber(const QUuid& nodeUUID) const; void setLastBroadcastSequenceNumber(const QUuid& nodeUUID, uint16_t sequenceNumber) { _lastBroadcastSequenceNumbers[nodeUUID] = sequenceNumber; } @@ -180,6 +183,8 @@ private: std::unordered_map _lastSentTraitsTimestamps; std::unordered_map _sentTraitVersions; + + std::atomic_bool _isIgnoreRadiusEnabled { false }; }; #endif // hifi_AvatarMixerClientData_h diff --git a/assignment-client/src/avatars/AvatarMixerSlave.cpp b/assignment-client/src/avatars/AvatarMixerSlave.cpp index 7368db0c31..9c88580b99 100644 --- a/assignment-client/src/avatars/AvatarMixerSlave.cpp +++ b/assignment-client/src/avatars/AvatarMixerSlave.cpp @@ -345,7 +345,7 @@ void AvatarMixerSlave::broadcastAvatarDataToAgent(const SharedNodePointer& node) } else { // Check to see if the space bubble is enabled // Don't bother with these checks if the other avatar has their bubble enabled and we're gettingAnyIgnored - if (destinationNode->isIgnoreRadiusEnabled() || (avatarNode->isIgnoreRadiusEnabled() && !getsAnyIgnored)) { + if (nodeData->isIgnoreRadiusEnabled() || (avatarClientNodeData->isIgnoreRadiusEnabled() && !getsAnyIgnored)) { // Perform the collision check between the two bounding boxes const float OTHER_AVATAR_BUBBLE_EXPANSION_FACTOR = 2.4f; // magic number determined empirically AABox otherNodeBox = computeBubbleBox(avatarClientNodeData->getAvatar(), OTHER_AVATAR_BUBBLE_EXPANSION_FACTOR); diff --git a/libraries/audio/src/PositionalAudioStream.cpp b/libraries/audio/src/PositionalAudioStream.cpp index a6bbc71a65..956d2bc105 100644 --- a/libraries/audio/src/PositionalAudioStream.cpp +++ b/libraries/audio/src/PositionalAudioStream.cpp @@ -92,6 +92,11 @@ int PositionalAudioStream::parsePositionalData(const QByteArray& positionalByteA packetStream.readRawData(reinterpret_cast(&_avatarBoundingBoxCorner), sizeof(_avatarBoundingBoxCorner)); packetStream.readRawData(reinterpret_cast(&_avatarBoundingBoxScale), sizeof(_avatarBoundingBoxScale)); + if (_avatarBoundingBoxCorner != _ignoreBox.getCorner()) { + // if the ignore box corner changes, we need to re-calculate the ignore box + calculateIgnoreBox(); + } + // if this node sent us a NaN for first float in orientation then don't consider this good audio and bail if (glm::isnan(_orientation.x)) { // NOTE: why would we reset the ring buffer here? @@ -107,3 +112,29 @@ AudioStreamStats PositionalAudioStream::getAudioStreamStats() const { streamStats._streamType = _type; return streamStats; } + +void PositionalAudioStream::calculateIgnoreBox() { + if (_avatarBoundingBoxScale != glm::vec3(0)) { + auto scale = _avatarBoundingBoxScale; + + // enforce a minimum scale + static const glm::vec3 MIN_IGNORE_BOX_SCALE = glm::vec3(0.3f, 1.3f, 0.3f); + if (glm::any(glm::lessThan(scale, MIN_IGNORE_BOX_SCALE))) { + scale = MIN_IGNORE_BOX_SCALE; + } + + // (this is arbitrary number determined empirically for comfort) + const float IGNORE_BOX_SCALE_FACTOR = 2.4f; + scale *= IGNORE_BOX_SCALE_FACTOR; + + // create the box (we use a box for the zone for convenience) + _ignoreBox.setBox(_avatarBoundingBoxCorner, scale); + } +} + +void PositionalAudioStream::enableIgnoreBox() { + // re-calculate the ignore box using the latest values + calculateIgnoreBox(); + + _isIgnoreBoxEnabled = true; +} diff --git a/libraries/audio/src/PositionalAudioStream.h b/libraries/audio/src/PositionalAudioStream.h index 4c48ea3386..f11a43f93f 100644 --- a/libraries/audio/src/PositionalAudioStream.h +++ b/libraries/audio/src/PositionalAudioStream.h @@ -19,6 +19,21 @@ const int AUDIOMIXER_INBOUND_RING_BUFFER_FRAME_CAPACITY = 100; +using StreamID = QUuid; + +struct NodeIDStreamID { + QUuid nodeID; + Node::LocalID nodeLocalID; + StreamID streamID; + + NodeIDStreamID(QUuid nodeID, Node::LocalID nodeLocalID, StreamID streamID) + : nodeID(nodeID), nodeLocalID(nodeLocalID), streamID(streamID) {}; + + bool operator==(const NodeIDStreamID& other) const { + return (nodeLocalID == other.nodeLocalID || nodeID == other.nodeID) && streamID == other.streamID; + } +}; + class PositionalAudioStream : public InboundAudioStream { Q_OBJECT public: @@ -30,7 +45,7 @@ public: PositionalAudioStream(PositionalAudioStream::Type type, bool isStereo, int numStaticJitterFrames = -1); const QUuid DEFAULT_STREAM_IDENTIFIER = QUuid(); - virtual const QUuid& getStreamIdentifier() const { return DEFAULT_STREAM_IDENTIFIER; } + virtual const StreamID& getStreamIdentifier() const { return DEFAULT_STREAM_IDENTIFIER; } virtual void resetStats() override; @@ -53,6 +68,16 @@ public: bool hasValidPosition() const { return _hasValidPosition; } + using IgnoreBox = AABox; + + // called from single AudioMixerSlave while processing packets for node + void enableIgnoreBox(); + void disableIgnoreBox() { _isIgnoreBoxEnabled = false; } + + // thread-safe, called from AudioMixerSlave(s) while preparing mixes + bool isIgnoreBoxEnabled() const { return _isIgnoreBoxEnabled; } + const IgnoreBox& getIgnoreBox() const { return _ignoreBox; } + protected: // disallow copying of PositionalAudioStream objects PositionalAudioStream(const PositionalAudioStream&); @@ -61,6 +86,8 @@ protected: int parsePositionalData(const QByteArray& positionalByteArray); protected: + void calculateIgnoreBox(); + Type _type; glm::vec3 _position; glm::quat _orientation; @@ -80,6 +107,9 @@ protected: int _frameCounter; bool _hasValidPosition { false }; + + bool _isIgnoreBoxEnabled { false }; + IgnoreBox _ignoreBox; }; #endif // hifi_PositionalAudioStream_h diff --git a/libraries/networking/src/Node.cpp b/libraries/networking/src/Node.cpp index ec72ae9bf6..9421e1da44 100644 --- a/libraries/networking/src/Node.cpp +++ b/libraries/networking/src/Node.cpp @@ -96,7 +96,6 @@ Node::Node(const QUuid& uuid, NodeType_t type, const HifiSockAddr& publicSocket, { // Update socket's object name setType(_type); - _ignoreRadiusEnabled = false; } void Node::setType(char type) { @@ -114,9 +113,12 @@ void Node::updateClockSkewUsec(qint64 clockSkewSample) { _clockSkewUsec = (quint64)_clockSkewMovingPercentile.getValueAtPercentile(); } -void Node::parseIgnoreRequestMessage(QSharedPointer message) { +Node::NodesIgnoredPair Node::parseIgnoreRequestMessage(QSharedPointer message) { bool addToIgnore; message->readPrimitive(&addToIgnore); + + std::vector nodesIgnored; + while (message->getBytesLeftToRead()) { // parse out the UUID being ignored from the packet QUuid ignoredUUID = QUuid::fromRfc4122(message->readWithoutCopy(NUM_BYTES_RFC4122_UUID)); @@ -126,7 +128,11 @@ void Node::parseIgnoreRequestMessage(QSharedPointer message) { } else { removeIgnoredNode(ignoredUUID); } + + nodesIgnored.push_back(ignoredUUID); } + + return { nodesIgnored, addToIgnore }; } void Node::addIgnoredNode(const QUuid& otherNodeID) { @@ -167,12 +173,6 @@ bool Node::isIgnoringNodeWithID(const QUuid& nodeID) const { return std::find(_ignoredNodeIDs.begin(), _ignoredNodeIDs.end(), nodeID) != _ignoredNodeIDs.end(); } -void Node::parseIgnoreRadiusRequestMessage(QSharedPointer message) { - bool enabled; - message->readPrimitive(&enabled); - _ignoreRadiusEnabled = enabled; -} - QDataStream& operator<<(QDataStream& out, const Node& node) { out << node._type; out << node._uuid; diff --git a/libraries/networking/src/Node.h b/libraries/networking/src/Node.h index 72a32d9d18..6c5a56c94e 100644 --- a/libraries/networking/src/Node.h +++ b/libraries/networking/src/Node.h @@ -81,17 +81,19 @@ public: bool getCanKick() const { return _permissions.can(NodePermissions::Permission::canKick); } bool getCanReplaceContent() const { return _permissions.can(NodePermissions::Permission::canReplaceDomainContent); } - void parseIgnoreRequestMessage(QSharedPointer message); + using NodesIgnoredPair = std::pair, bool>; + + NodesIgnoredPair parseIgnoreRequestMessage(QSharedPointer message); void addIgnoredNode(const QUuid& otherNodeID); void removeIgnoredNode(const QUuid& otherNodeID); bool isIgnoringNodeWithID(const QUuid& nodeID) const; - void parseIgnoreRadiusRequestMessage(QSharedPointer message); + + using IgnoredNodeIDs = std::vector; + const IgnoredNodeIDs& getIgnoredNodeIDs() const { return _ignoredNodeIDs; } friend QDataStream& operator<<(QDataStream& out, const Node& node); friend QDataStream& operator>>(QDataStream& in, Node& node); - bool isIgnoreRadiusEnabled() const { return _ignoreRadiusEnabled; } - private: // privatize copy and assignment operator to disallow Node copying Node(const Node &otherNode); @@ -109,11 +111,10 @@ private: MovingPercentile _clockSkewMovingPercentile; NodePermissions _permissions; bool _isUpstream { false }; - std::vector _ignoredNodeIDs; + + IgnoredNodeIDs _ignoredNodeIDs; mutable QReadWriteLock _ignoredNodeIDSetLock; std::vector _replicatedUsernames { }; - - std::atomic_bool _ignoreRadiusEnabled; }; Q_DECLARE_METATYPE(Node*) From 7d8b15ed75b18f37149ee18c142fd4cd1fda0c11 Mon Sep 17 00:00:00 2001 From: Stephen Birarda Date: Tue, 28 Aug 2018 14:47:01 -0700 Subject: [PATCH 07/21] move valid position check to packet processing --- .../src/audio/AudioMixerClientData.cpp | 78 +++++++++++++++---- .../src/audio/AudioMixerClientData.h | 2 + .../src/audio/AudioMixerSlave.cpp | 9 +-- .../src/audio/AvatarAudioStream.cpp | 10 +-- .../src/audio/AvatarAudioStream.h | 2 + libraries/audio/src/InboundAudioStream.h | 2 + libraries/audio/src/InjectedAudioStream.cpp | 2 +- libraries/audio/src/InjectedAudioStream.h | 2 + libraries/audio/src/PositionalAudioStream.cpp | 11 --- libraries/audio/src/PositionalAudioStream.h | 7 +- 10 files changed, 83 insertions(+), 42 deletions(-) diff --git a/assignment-client/src/audio/AudioMixerClientData.cpp b/assignment-client/src/audio/AudioMixerClientData.cpp index e0b156473a..40d5cbfcc5 100644 --- a/assignment-client/src/audio/AudioMixerClientData.cpp +++ b/assignment-client/src/audio/AudioMixerClientData.cpp @@ -13,6 +13,8 @@ #include +#include + #include #include @@ -332,18 +334,65 @@ int AudioMixerClientData::parseData(ReceivedMessage& message) { return 0; } +bool AudioMixerClientData::containsValidPosition(ReceivedMessage& message) const { + static const int SEQUENCE_NUMBER_BYTES = sizeof(quint16); + + auto posBefore = message.getPosition(); + + message.seek(SEQUENCE_NUMBER_BYTES); + + // skip over the codec string + message.readString(); + + switch (message.getType()) { + case PacketType::MicrophoneAudioNoEcho: + case PacketType::MicrophoneAudioWithEcho: { + // skip over the stereo flag + message.seek(message.getPosition() + sizeof(ChannelFlag)); + break; + } + case PacketType::SilentAudioFrame: { + // skip the number of silent samples + message.seek(message.getPosition() + sizeof(SilentSamplesBytes)); + break; + } + case PacketType::InjectAudio: { + // skip the stream ID, stereo flag, and loopback flag + message.seek(message.getPosition() + NUM_STREAM_ID_BYTES + sizeof(ChannelFlag) + sizeof(LoopbackFlag)); + } + default: + Q_UNREACHABLE(); + break; + } + + glm::vec3 peekPosition; + message.readPrimitive(&peekPosition); + + // reset the position the message was at before we were called + message.seek(posBefore); + + if (glm::any(glm::isnan(peekPosition))) { + return false; + } + + return true; +} + void AudioMixerClientData::processStreamPacket(ReceivedMessage& message, ConcurrentAddedStreams &addedStreams) { + + if (!containsValidPosition(message)) { + qDebug() << "Refusing to process audio stream from" << message.getSourceID() << "with invalid position"; + return; + } + SharedStreamPointer matchingStream; auto packetType = message.getType(); bool newStream = false; if (packetType == PacketType::MicrophoneAudioWithEcho - || packetType == PacketType::ReplicatedMicrophoneAudioWithEcho || packetType == PacketType::MicrophoneAudioNoEcho - || packetType == PacketType::ReplicatedMicrophoneAudioNoEcho - || packetType == PacketType::SilentAudioFrame - || packetType == PacketType::ReplicatedSilentAudioFrame) { + || packetType == PacketType::SilentAudioFrame) { QWriteLocker writeLocker { &_streamsLock }; @@ -355,7 +404,7 @@ void AudioMixerClientData::processStreamPacket(ReceivedMessage& message, Concurr // we don't have a mic stream yet, so add it // hop past the sequence number that leads the packet - message.seek(sizeof(quint16)); + message.seek(sizeof(StreamSequenceNumber)); // pull the codec string from the packet auto codecString = message.readString(); @@ -363,11 +412,11 @@ void AudioMixerClientData::processStreamPacket(ReceivedMessage& message, Concurr // determine if the stream is stereo or not bool isStereo; if (packetType == PacketType::SilentAudioFrame || packetType == PacketType::ReplicatedSilentAudioFrame) { - quint16 numSilentSamples; + SilentSamplesBytes numSilentSamples; message.readPrimitive(&numSilentSamples); isStereo = numSilentSamples == AudioConstants::NETWORK_FRAME_SAMPLES_STEREO; } else { - quint8 channelFlag; + ChannelFlag channelFlag; message.readPrimitive(&channelFlag); isStereo = channelFlag == 1; } @@ -395,17 +444,15 @@ void AudioMixerClientData::processStreamPacket(ReceivedMessage& message, Concurr } writeLocker.unlock(); - } else if (packetType == PacketType::InjectAudio - || packetType == PacketType::ReplicatedInjectAudio) { + } else if (packetType == PacketType::InjectAudio) { + // this is injected audio - // grab the stream identifier for this injected audio - message.seek(sizeof(quint16)); + // skip the sequence number and codec string and grab the stream identifier for this injected audio + message.seek(sizeof(StreamSequenceNumber)); + message.readString(); QUuid streamIdentifier = QUuid::fromRfc4122(message.readWithoutCopy(NUM_BYTES_RFC4122_UUID)); - bool isStereo; - message.readPrimitive(&isStereo); - QWriteLocker writeLock { &_streamsLock }; auto streamIt = std::find_if(_audioStreams.begin(), _audioStreams.end(), [&streamIdentifier](const SharedStreamPointer& stream) { @@ -413,6 +460,9 @@ void AudioMixerClientData::processStreamPacket(ReceivedMessage& message, Concurr }); if (streamIt == _audioStreams.end()) { + bool isStereo; + message.readPrimitive(&isStereo); + // we don't have this injected stream yet, so add it auto injectorStream = new InjectedAudioStream(streamIdentifier, isStereo, AudioMixer::getStaticJitterFrames()); diff --git a/assignment-client/src/audio/AudioMixerClientData.h b/assignment-client/src/audio/AudioMixerClientData.h index 7de5309717..96971cb0ba 100644 --- a/assignment-client/src/audio/AudioMixerClientData.h +++ b/assignment-client/src/audio/AudioMixerClientData.h @@ -172,6 +172,8 @@ private: void setGainForAvatar(QUuid nodeID, uint8_t gain); + bool containsValidPosition(ReceivedMessage& message) const; + MixableStreamsVector _mixableStreams; quint16 _outgoingMixedAudioSequenceNumber; diff --git a/assignment-client/src/audio/AudioMixerSlave.cpp b/assignment-client/src/audio/AudioMixerSlave.cpp index 547043c0f0..94d7d06b8c 100644 --- a/assignment-client/src/audio/AudioMixerSlave.cpp +++ b/assignment-client/src/audio/AudioMixerSlave.cpp @@ -135,12 +135,6 @@ bool AudioMixerSlave::prepareMix(const SharedNodePointer& listener) { AvatarAudioStream* listenerAudioStream = static_cast(listener->getLinkedData())->getAvatarAudioStream(); AudioMixerClientData* listenerData = static_cast(listener->getLinkedData()); - // if we received an invalid position from this listener, then refuse to make them a mix - // because we don't know how to do it properly - if (!listenerAudioStream->hasValidPosition()) { - return false; - } - // zero out the mix for this listener memset(_mixSamples, 0, sizeof(_mixSamples)); @@ -195,6 +189,7 @@ bool AudioMixerSlave::prepareMix(const SharedNodePointer& listener) { auto it = mixableStreams.begin(); auto end = mixableStreams.end(); while (it != end) { + // check if this node (and therefore all of the node's streams) has been removed auto& nodeIDStreamID = it->nodeStreamID; auto matchedRemovedNode = std::find(_sharedData.removedNodes.cbegin(), _sharedData.removedNodes.cend(), @@ -279,7 +274,7 @@ bool AudioMixerSlave::prepareMix(const SharedNodePointer& listener) { } else { // we're throttling, so we need to update the approximate volume for any un-skipped streams // unless this is simply for an echo (in which case the approx volume is 1.0) - if (!it->skippedStream) { + if (!it->skippedStream && it->positionalStream->getLastPopOutputTrailingLoudness() > 0.0f) { if (it->positionalStream != listenerAudioStream) { // approximate the gain float gain = approximateGain(*listenerAudioStream, *(it->positionalStream)); diff --git a/assignment-client/src/audio/AvatarAudioStream.cpp b/assignment-client/src/audio/AvatarAudioStream.cpp index 22ea8c0617..1b3ca9a8b1 100644 --- a/assignment-client/src/audio/AvatarAudioStream.cpp +++ b/assignment-client/src/audio/AvatarAudioStream.cpp @@ -23,9 +23,9 @@ int AvatarAudioStream::parseStreamProperties(PacketType type, const QByteArray& if (type == PacketType::SilentAudioFrame) { const char* dataAt = packetAfterSeqNum.constData(); - quint16 numSilentSamples = *(reinterpret_cast(dataAt)); - readBytes += sizeof(quint16); - numAudioSamples = (int)numSilentSamples; + SilentSamplesBytes numSilentSamples = *(reinterpret_cast(dataAt)); + readBytes += sizeof(SilentSamplesBytes); + numAudioSamples = (int) numSilentSamples; // read the positional data readBytes += parsePositionalData(packetAfterSeqNum.mid(readBytes)); @@ -34,9 +34,9 @@ int AvatarAudioStream::parseStreamProperties(PacketType type, const QByteArray& _shouldLoopbackForNode = (type == PacketType::MicrophoneAudioWithEcho); // read the channel flag - quint8 channelFlag = packetAfterSeqNum.at(readBytes); + ChannelFlag channelFlag = packetAfterSeqNum.at(readBytes); bool isStereo = channelFlag == 1; - readBytes += sizeof(quint8); + readBytes += sizeof(ChannelFlag); // if isStereo value has changed, restart the ring buffer with new frame size if (isStereo != _isStereo) { diff --git a/assignment-client/src/audio/AvatarAudioStream.h b/assignment-client/src/audio/AvatarAudioStream.h index 497e522922..de9577099e 100644 --- a/assignment-client/src/audio/AvatarAudioStream.h +++ b/assignment-client/src/audio/AvatarAudioStream.h @@ -16,6 +16,8 @@ #include "PositionalAudioStream.h" +using SilentSamplesBytes = quint16; + class AvatarAudioStream : public PositionalAudioStream { public: AvatarAudioStream(bool isStereo, int numStaticJitterFrames = -1); diff --git a/libraries/audio/src/InboundAudioStream.h b/libraries/audio/src/InboundAudioStream.h index ecd1a118f9..5ff9e2c84c 100644 --- a/libraries/audio/src/InboundAudioStream.h +++ b/libraries/audio/src/InboundAudioStream.h @@ -30,6 +30,8 @@ // Audio Env bitset const int HAS_REVERB_BIT = 0; // 1st bit +using StreamSequenceNumber = quint16; + class InboundAudioStream : public NodeData { Q_OBJECT diff --git a/libraries/audio/src/InjectedAudioStream.cpp b/libraries/audio/src/InjectedAudioStream.cpp index 2f357416f2..4c598fe6d1 100644 --- a/libraries/audio/src/InjectedAudioStream.cpp +++ b/libraries/audio/src/InjectedAudioStream.cpp @@ -50,7 +50,7 @@ int InjectedAudioStream::parseStreamProperties(PacketType type, } // pull the loopback flag and set our boolean - uchar shouldLoopback; + LoopbackFlag shouldLoopback; packetStream >> shouldLoopback; _shouldLoopbackForNode = (shouldLoopback == 1); diff --git a/libraries/audio/src/InjectedAudioStream.h b/libraries/audio/src/InjectedAudioStream.h index 75b1c0b236..990ea81272 100644 --- a/libraries/audio/src/InjectedAudioStream.h +++ b/libraries/audio/src/InjectedAudioStream.h @@ -16,6 +16,8 @@ #include "PositionalAudioStream.h" +using LoopbackFlag = uchar; + class InjectedAudioStream : public PositionalAudioStream { public: InjectedAudioStream(const QUuid& streamIdentifier, bool isStereo, int numStaticJitterFrames = -1); diff --git a/libraries/audio/src/PositionalAudioStream.cpp b/libraries/audio/src/PositionalAudioStream.cpp index 956d2bc105..4161b66060 100644 --- a/libraries/audio/src/PositionalAudioStream.cpp +++ b/libraries/audio/src/PositionalAudioStream.cpp @@ -14,7 +14,6 @@ #include -#include #include #include @@ -78,16 +77,6 @@ int PositionalAudioStream::parsePositionalData(const QByteArray& positionalByteA QDataStream packetStream(positionalByteArray); packetStream.readRawData(reinterpret_cast(&_position), sizeof(_position)); - - // if the client sends us a bad position, flag it so that we don't consider this stream for mixing - if (glm::isnan(_position.x) || glm::isnan(_position.y) || glm::isnan(_position.z)) { - HIFI_FDEBUG("PositionalAudioStream unpacked invalid position for node" << uuidStringWithoutCurlyBraces(getNodeID()) ); - - _hasValidPosition = false; - } else { - _hasValidPosition = true; - } - packetStream.readRawData(reinterpret_cast(&_orientation), sizeof(_orientation)); packetStream.readRawData(reinterpret_cast(&_avatarBoundingBoxCorner), sizeof(_avatarBoundingBoxCorner)); packetStream.readRawData(reinterpret_cast(&_avatarBoundingBoxScale), sizeof(_avatarBoundingBoxScale)); diff --git a/libraries/audio/src/PositionalAudioStream.h b/libraries/audio/src/PositionalAudioStream.h index f11a43f93f..01a714aeb4 100644 --- a/libraries/audio/src/PositionalAudioStream.h +++ b/libraries/audio/src/PositionalAudioStream.h @@ -20,6 +20,7 @@ const int AUDIOMIXER_INBOUND_RING_BUFFER_FRAME_CAPACITY = 100; using StreamID = QUuid; +const int NUM_STREAM_ID_BYTES = NUM_BYTES_RFC4122_UUID; struct NodeIDStreamID { QUuid nodeID; @@ -34,6 +35,8 @@ struct NodeIDStreamID { } }; +using ChannelFlag = quint8; + class PositionalAudioStream : public InboundAudioStream { Q_OBJECT public: @@ -66,8 +69,6 @@ public: const glm::vec3& getAvatarBoundingBoxCorner() const { return _avatarBoundingBoxCorner; } const glm::vec3& getAvatarBoundingBoxScale() const { return _avatarBoundingBoxScale; } - bool hasValidPosition() const { return _hasValidPosition; } - using IgnoreBox = AABox; // called from single AudioMixerSlave while processing packets for node @@ -106,8 +107,6 @@ protected: float _quietestFrameLoudness; int _frameCounter; - bool _hasValidPosition { false }; - bool _isIgnoreBoxEnabled { false }; IgnoreBox _ignoreBox; }; From 6750d4a37050af703f16873fe61907ca101c133c Mon Sep 17 00:00:00 2001 From: Stephen Birarda Date: Tue, 28 Aug 2018 16:58:04 -0700 Subject: [PATCH 08/21] move buffer popping to packet processing --- assignment-client/src/audio/AudioMixer.cpp | 20 +------------------ assignment-client/src/audio/AudioMixer.h | 4 ---- .../src/audio/AudioMixerClientData.cpp | 20 +++++-------------- .../src/audio/AudioMixerClientData.h | 5 ++--- .../src/audio/AudioMixerSlave.cpp | 3 ++- 5 files changed, 10 insertions(+), 42 deletions(-) diff --git a/assignment-client/src/audio/AudioMixer.cpp b/assignment-client/src/audio/AudioMixer.cpp index a7a5ac95bc..ce0ecd8e37 100644 --- a/assignment-client/src/audio/AudioMixer.cpp +++ b/assignment-client/src/audio/AudioMixer.cpp @@ -285,10 +285,9 @@ void AudioMixer::sendStatsPacket() { addTiming(_ticTiming, "tic"); addTiming(_sleepTiming, "sleep"); addTiming(_frameTiming, "frame"); - addTiming(_prepareTiming, "prepare"); + addTiming(_packetsTiming, "packets"); addTiming(_mixTiming, "mix"); addTiming(_eventsTiming, "events"); - addTiming(_packetsTiming, "packets"); #ifdef HIFI_AUDIO_MIXER_DEBUG timingStats["ns_per_mix"] = (_stats.totalMixes > 0) ? (float)(_stats.mixTime / _stats.totalMixes) : 0; @@ -421,14 +420,6 @@ void AudioMixer::start() { } nodeList->nestedEach([&](NodeList::const_iterator cbegin, NodeList::const_iterator cend) { - // prepare frames; pop off any new audio from their streams - { - auto prepareTimer = _prepareTiming.timer(); - for_each(cbegin, cend, [&](const SharedNodePointer& node) { - _stats.sumStreams += prepareFrame(node, frame); - }); - } - // mix across slave threads { auto mixTimer = _mixTiming.timer(); @@ -520,15 +511,6 @@ void AudioMixer::throttle(chrono::microseconds duration, int frame) { } } -int AudioMixer::prepareFrame(const SharedNodePointer& node, unsigned int frame) { - AudioMixerClientData* data = (AudioMixerClientData*)node->getLinkedData(); - if (data == nullptr) { - return 0; - } - - return data->checkBuffersBeforeFrameSend(); -} - void AudioMixer::clearDomainSettings() { _numStaticJitterFrames = DISABLE_STATIC_JITTER_FRAMES; _attenuationPerDoublingInDistance = DEFAULT_ATTENUATION_PER_DOUBLING_IN_DISTANCE; diff --git a/assignment-client/src/audio/AudioMixer.h b/assignment-client/src/audio/AudioMixer.h index 6cddd539da..baea856b41 100644 --- a/assignment-client/src/audio/AudioMixer.h +++ b/assignment-client/src/audio/AudioMixer.h @@ -87,10 +87,6 @@ private: std::chrono::microseconds timeFrame(p_high_resolution_clock::time_point& timestamp); void throttle(std::chrono::microseconds frameDuration, int frame); - // pop a frame from any streams on the node - // returns the number of available streams - int prepareFrame(const SharedNodePointer& node, unsigned int frame); - AudioMixerClientData* getOrCreateClientData(Node* node); QString percentageForMixStats(int counter); diff --git a/assignment-client/src/audio/AudioMixerClientData.cpp b/assignment-client/src/audio/AudioMixerClientData.cpp index 40d5cbfcc5..2af1a0fc08 100644 --- a/assignment-client/src/audio/AudioMixerClientData.cpp +++ b/assignment-client/src/audio/AudioMixerClientData.cpp @@ -57,7 +57,7 @@ void AudioMixerClientData::queuePacket(QSharedPointer message, _packetQueue.push(message); } -void AudioMixerClientData::processPackets(ConcurrentAddedStreams& addedStreams) { +int AudioMixerClientData::processPackets(ConcurrentAddedStreams& addedStreams) { SharedNodePointer node = _packetQueue.node; assert(_packetQueue.empty() || node); _packetQueue.node.clear(); @@ -105,6 +105,10 @@ void AudioMixerClientData::processPackets(ConcurrentAddedStreams& addedStreams) _packetQueue.pop(); } assert(_packetQueue.empty()); + + // now that we have processed all packets for this frame + // we can prepare the sources from this client to be ready for mixing + return checkBuffersBeforeFrameSend(); } bool isReplicatedPacket(PacketType packetType) { @@ -292,8 +296,6 @@ void AudioMixerClientData::parseRadiusIgnoreRequest(QSharedPointergetStreamIdentifier().isNull(); }); @@ -307,8 +309,6 @@ AvatarAudioStream* AudioMixerClientData::getAvatarAudioStream() { } void AudioMixerClientData::removeAgentAvatarAudioStream() { - QWriteLocker writeLocker { &_streamsLock }; - auto it = std::remove_if(_audioStreams.begin(), _audioStreams.end(), [](const SharedStreamPointer& stream){ return stream->getStreamIdentifier().isNull(); }); @@ -394,8 +394,6 @@ void AudioMixerClientData::processStreamPacket(ReceivedMessage& message, Concurr || packetType == PacketType::MicrophoneAudioNoEcho || packetType == PacketType::SilentAudioFrame) { - QWriteLocker writeLocker { &_streamsLock }; - auto micStreamIt = std::find_if(_audioStreams.begin(), _audioStreams.end(), [](const SharedStreamPointer& stream){ return stream->getStreamIdentifier().isNull(); }); @@ -442,8 +440,6 @@ void AudioMixerClientData::processStreamPacket(ReceivedMessage& message, Concurr } else { matchingStream = *micStreamIt; } - - writeLocker.unlock(); } else if (packetType == PacketType::InjectAudio) { // this is injected audio @@ -453,8 +449,6 @@ void AudioMixerClientData::processStreamPacket(ReceivedMessage& message, Concurr QUuid streamIdentifier = QUuid::fromRfc4122(message.readWithoutCopy(NUM_BYTES_RFC4122_UUID)); - QWriteLocker writeLock { &_streamsLock }; - auto streamIt = std::find_if(_audioStreams.begin(), _audioStreams.end(), [&streamIdentifier](const SharedStreamPointer& stream) { return stream->getStreamIdentifier() == streamIdentifier; }); @@ -478,8 +472,6 @@ void AudioMixerClientData::processStreamPacket(ReceivedMessage& message, Concurr } else { matchingStream = *streamIt; } - - writeLock.unlock(); } // seek to the beginning of the packet so that the next reader is in the right spot @@ -501,8 +493,6 @@ void AudioMixerClientData::processStreamPacket(ReceivedMessage& message, Concurr } int AudioMixerClientData::checkBuffersBeforeFrameSend() { - QWriteLocker writeLocker { &_streamsLock }; - auto it = _audioStreams.begin(); while (it != _audioStreams.end()) { SharedStreamPointer stream = *it; diff --git a/assignment-client/src/audio/AudioMixerClientData.h b/assignment-client/src/audio/AudioMixerClientData.h index 96971cb0ba..332c6bfa51 100644 --- a/assignment-client/src/audio/AudioMixerClientData.h +++ b/assignment-client/src/audio/AudioMixerClientData.h @@ -50,7 +50,7 @@ public: using AudioStreamVector = std::vector; void queuePacket(QSharedPointer packet, SharedNodePointer node); - void processPackets(ConcurrentAddedStreams& addedStreams); + int processPackets(ConcurrentAddedStreams& addedStreams); // returns the number of available streams this frame AudioStreamVector& getAudioStreams() { return _audioStreams; } AvatarAudioStream* getAvatarAudioStream(); @@ -165,8 +165,7 @@ private: }; PacketQueue _packetQueue; - QReadWriteLock _streamsLock; - AudioStreamVector _audioStreams; // microphone stream from avatar is stored under key of null UUID + AudioStreamVector _audioStreams; // microphone stream from avatar has a null stream ID void optionallyReplicatePacket(ReceivedMessage& packet, const Node& node); diff --git a/assignment-client/src/audio/AudioMixerSlave.cpp b/assignment-client/src/audio/AudioMixerSlave.cpp index 94d7d06b8c..f2afe000d9 100644 --- a/assignment-client/src/audio/AudioMixerSlave.cpp +++ b/assignment-client/src/audio/AudioMixerSlave.cpp @@ -55,7 +55,8 @@ inline float computeAzimuth(const AvatarAudioStream& listeningNodeStream, const void AudioMixerSlave::processPackets(const SharedNodePointer& node) { AudioMixerClientData* data = (AudioMixerClientData*)node->getLinkedData(); if (data) { - data->processPackets(_sharedData.addedStreams); + // process packets and collect the number of streams available for this frame + stats.sumStreams += data->processPackets(_sharedData.addedStreams); } } From dacf343e9a8d3fff48cb4f462da8d061cb2bf66c Mon Sep 17 00:00:00 2001 From: Stephen Birarda Date: Wed, 5 Sep 2018 13:53:29 -0700 Subject: [PATCH 09/21] reduce codec mismatch and starve log spam --- libraries/audio/src/InboundAudioStream.cpp | 9 ++------- 1 file changed, 2 insertions(+), 7 deletions(-) diff --git a/libraries/audio/src/InboundAudioStream.cpp b/libraries/audio/src/InboundAudioStream.cpp index 7645a674e4..8c5388e222 100644 --- a/libraries/audio/src/InboundAudioStream.cpp +++ b/libraries/audio/src/InboundAudioStream.cpp @@ -171,7 +171,6 @@ int InboundAudioStream::parseData(ReceivedMessage& message) { } else { _mismatchedAudioCodecCount++; - qDebug(audio) << "Codec mismatch: expected" << _selectedCodecName << "got" << codecInPacket; if (packetPCM) { // If there are PCM packets in-flight after the codec is changed, use them. @@ -191,7 +190,8 @@ int InboundAudioStream::parseData(ReceivedMessage& message) { auto sendingNode = DependencyManager::get()->nodeWithLocalID(message.getSourceID()); if (sendingNode) { emit mismatchedAudioCodec(sendingNode, _selectedCodecName, codecInPacket); - qDebug(audio) << "Codec mismatch threshold exceeded, SelectedAudioFormat(" << _selectedCodecName << " ) sent"; + qDebug(audio) << "Codec mismatch threshold exceeded, sent selected codec" + << _selectedCodecName << "to" << message.getSenderSockAddr(); } } } @@ -208,7 +208,6 @@ int InboundAudioStream::parseData(ReceivedMessage& message) { int framesAvailable = _ringBuffer.framesAvailable(); // if this stream was starved, check if we're still starved. if (_isStarved && framesAvailable >= _desiredJitterBufferFrames) { - qCInfo(audiostream, "Starve ended"); _isStarved = false; } // if the ringbuffer exceeds the desired size by more than the threshold specified, @@ -378,10 +377,6 @@ void InboundAudioStream::framesAvailableChanged() { } void InboundAudioStream::setToStarved() { - if (!_isStarved) { - qCInfo(audiostream, "Starved"); - } - _consecutiveNotMixedCount = 0; _starveCount++; // if we have more than the desired frames when setToStarved() is called, then we'll immediately From 996e033deeb25d43f2c76ab61358420a266995e0 Mon Sep 17 00:00:00 2001 From: Stephen Birarda Date: Wed, 5 Sep 2018 12:06:58 -0700 Subject: [PATCH 10/21] add stats for skipped streams --- assignment-client/src/audio/AudioMixer.cpp | 3 +++ assignment-client/src/audio/AudioMixerSlave.cpp | 11 +++++++---- assignment-client/src/audio/AudioMixerStats.cpp | 7 +++++++ assignment-client/src/audio/AudioMixerStats.h | 4 ++++ 4 files changed, 21 insertions(+), 4 deletions(-) diff --git a/assignment-client/src/audio/AudioMixer.cpp b/assignment-client/src/audio/AudioMixer.cpp index ce0ecd8e37..8c9a202844 100644 --- a/assignment-client/src/audio/AudioMixer.cpp +++ b/assignment-client/src/audio/AudioMixer.cpp @@ -302,6 +302,9 @@ void AudioMixer::sendStatsPacket() { mixStats["%_hrtf_mixes"] = percentageForMixStats(_stats.hrtfRenders); mixStats["%_hrtf_silent_mixes"] = percentageForMixStats(_stats.hrtfSilentRenders); mixStats["%_hrtf_throttle_mixes"] = percentageForMixStats(_stats.hrtfThrottleRenders); + mixStats["%_skipped_throttle_mixes"] = percentageForMixStats(_stats.skippedThrottle); + mixStats["%_skipped_silent_mixes"] = percentageForMixStats(_stats.skippedSilent); + mixStats["%_skipped_other_mixes"] = percentageForMixStats(_stats.skippedOther); mixStats["%_manual_stereo_mixes"] = percentageForMixStats(_stats.manualStereoMixes); mixStats["%_manual_echo_mixes"] = percentageForMixStats(_stats.manualEchoMixes); diff --git a/assignment-client/src/audio/AudioMixerSlave.cpp b/assignment-client/src/audio/AudioMixerSlave.cpp index f2afe000d9..9d2f3728a5 100644 --- a/assignment-client/src/audio/AudioMixerSlave.cpp +++ b/assignment-client/src/audio/AudioMixerSlave.cpp @@ -347,6 +347,7 @@ void AudioMixerSlave::addStream(AudioMixerClientData::MixableStream& mixableStre if (mixableStream.skippedStream) { // any skipped stream gets no processing and no silent render - early return + ++stats.skippedOther; return; } @@ -357,11 +358,13 @@ void AudioMixerSlave::addStream(AudioMixerClientData::MixableStream& mixableStre // to reduce artifacts we still call the HRTF functor for every silent or throttled source // for the first frame where the source becomes throttled or silent // this ensures the correct tail from last mixed block and the correct spatialization of next first block - if (throttle || mixableStream.skippedStream || streamToAdd->getLastPopOutputLoudness() == 0.0f) { + if (throttle || streamToAdd->getLastPopOutputLoudness() == 0.0f) { if (mixableStream.completedSilentRender) { - if (throttle) { - ++stats.hrtfThrottleRenders; + if (streamToAdd->getLastPopOutputLoudness() == 0.0f) { + ++stats.skippedSilent; + } else { + ++stats.skippedThrottle; } return; @@ -456,7 +459,7 @@ void AudioMixerSlave::addStream(AudioMixerClientData::MixableStream& mixableStre streamPopOutput.readSamples(_bufferSamples, AudioConstants::NETWORK_FRAME_SAMPLES_PER_CHANNEL); - if (streamToAdd->getLastPopOutputLoudness() == 0.0f || mixableStream.skippedStream) { + if (streamToAdd->getLastPopOutputLoudness() == 0.0f) { // call renderSilent to reduce artifacts mixableStream.hrtf->renderSilent(_bufferSamples, _mixSamples, HRTF_DATASET_INDEX, azimuth, distance, gain, AudioConstants::NETWORK_FRAME_SAMPLES_PER_CHANNEL); diff --git a/assignment-client/src/audio/AudioMixerStats.cpp b/assignment-client/src/audio/AudioMixerStats.cpp index 4cfdd55167..213c0ba5e6 100644 --- a/assignment-client/src/audio/AudioMixerStats.cpp +++ b/assignment-client/src/audio/AudioMixerStats.cpp @@ -21,6 +21,9 @@ void AudioMixerStats::reset() { hrtfThrottleRenders = 0; manualStereoMixes = 0; manualEchoMixes = 0; + skippedThrottle = 0; + skippedSilent = 0; + skippedOther = 0; #ifdef HIFI_AUDIO_MIXER_DEBUG mixTime = 0; #endif @@ -36,6 +39,10 @@ void AudioMixerStats::accumulate(const AudioMixerStats& otherStats) { hrtfThrottleRenders += otherStats.hrtfThrottleRenders; manualStereoMixes += otherStats.manualStereoMixes; manualEchoMixes += otherStats.manualEchoMixes; + skippedThrottle += otherStats.skippedThrottle; + skippedSilent += otherStats.skippedSilent; + skippedOther += otherStats.skippedOther; + #ifdef HIFI_AUDIO_MIXER_DEBUG mixTime += otherStats.mixTime; #endif diff --git a/assignment-client/src/audio/AudioMixerStats.h b/assignment-client/src/audio/AudioMixerStats.h index f4ba9db769..41eb295214 100644 --- a/assignment-client/src/audio/AudioMixerStats.h +++ b/assignment-client/src/audio/AudioMixerStats.h @@ -30,6 +30,10 @@ struct AudioMixerStats { int manualStereoMixes { 0 }; int manualEchoMixes { 0 }; + int skippedThrottle { 0 }; + int skippedSilent { 0 }; + int skippedOther { 0 }; + #ifdef HIFI_AUDIO_MIXER_DEBUG uint64_t mixTime { 0 }; #endif From 925c39cbd3acb65db39ac6acdd36f5f9dda97825 Mon Sep 17 00:00:00 2001 From: Stephen Birarda Date: Wed, 5 Sep 2018 16:29:11 -0700 Subject: [PATCH 11/21] split sleep timing from check time timing --- assignment-client/src/audio/AudioMixer.cpp | 15 ++++++++++----- assignment-client/src/audio/AudioMixer.h | 2 ++ 2 files changed, 12 insertions(+), 5 deletions(-) diff --git a/assignment-client/src/audio/AudioMixer.cpp b/assignment-client/src/audio/AudioMixer.cpp index 8c9a202844..bba3398cbd 100644 --- a/assignment-client/src/audio/AudioMixer.cpp +++ b/assignment-client/src/audio/AudioMixer.cpp @@ -283,6 +283,7 @@ void AudioMixer::sendStatsPacket() { }; addTiming(_ticTiming, "tic"); + addTiming(_checkTimeTiming, "check_time"); addTiming(_sleepTiming, "sleep"); addTiming(_frameTiming, "frame"); addTiming(_packetsTiming, "packets"); @@ -391,7 +392,7 @@ void AudioMixer::start() { auto ticTimer = _ticTiming.timer(); { - auto timer = _sleepTiming.timer(); + auto timer = _checkTimeTiming.timer(); auto frameDuration = timeFrame(frameTimestamp); throttle(frameDuration, frame); } @@ -459,10 +460,14 @@ chrono::microseconds AudioMixer::timeFrame(p_high_resolution_clock::time_point& // set the new frame timestamp timestamp = max(now, nextTimestamp); - // sleep until the next frame should start - // WIN32 sleep_until is broken until VS2015 Update 2 - // instead, max (above) guarantees that timestamp >= now, so we can sleep_for - this_thread::sleep_for(timestamp - now); + { + auto timer = _sleepTiming.timer(); + + // sleep until the next frame should start + // WIN32 sleep_until is broken until VS2015 Update 2 + // instead, max (above) guarantees that timestamp >= now, so we can sleep_for + this_thread::sleep_for(timestamp - now); + } return duration; } diff --git a/assignment-client/src/audio/AudioMixer.h b/assignment-client/src/audio/AudioMixer.h index baea856b41..99b01683bf 100644 --- a/assignment-client/src/audio/AudioMixer.h +++ b/assignment-client/src/audio/AudioMixer.h @@ -125,7 +125,9 @@ private: uint64_t _history[TIMER_TRAILING_SECONDS] {}; int _index { 0 }; }; + Timer _ticTiming; + Timer _checkTimeTiming; Timer _sleepTiming; Timer _frameTiming; Timer _prepareTiming; From aea9775e4dba802a6ac766b9757020ea43a5f94e Mon Sep 17 00:00:00 2001 From: Stephen Birarda Date: Wed, 5 Sep 2018 16:55:46 -0700 Subject: [PATCH 12/21] get back to ideal frame time when slow --- assignment-client/src/audio/AudioMixer.cpp | 29 +++++++++++----------- assignment-client/src/audio/AudioMixer.h | 5 +++- 2 files changed, 18 insertions(+), 16 deletions(-) diff --git a/assignment-client/src/audio/AudioMixer.cpp b/assignment-client/src/audio/AudioMixer.cpp index bba3398cbd..b5318a72d9 100644 --- a/assignment-client/src/audio/AudioMixer.cpp +++ b/assignment-client/src/audio/AudioMixer.cpp @@ -386,15 +386,18 @@ void AudioMixer::start() { // mix state unsigned int frame = 1; - auto frameTimestamp = p_high_resolution_clock::now(); while (!_isFinished) { auto ticTimer = _ticTiming.timer(); - { - auto timer = _checkTimeTiming.timer(); - auto frameDuration = timeFrame(frameTimestamp); - throttle(frameDuration, frame); + if (_startFrameTimestamp.time_since_epoch().count() == 0) { + _startFrameTimestamp = _idealFrameTimestamp = p_high_resolution_clock::now(); + } else { + { + auto timer = _checkTimeTiming.timer(); + auto frameDuration = timeFrame(); + throttle(frameDuration, frame); + } } auto frameTimer = _frameTiming.timer(); @@ -449,26 +452,22 @@ void AudioMixer::start() { } } -chrono::microseconds AudioMixer::timeFrame(p_high_resolution_clock::time_point& timestamp) { +chrono::microseconds AudioMixer::timeFrame() { // advance the next frame - auto nextTimestamp = timestamp + chrono::microseconds(AudioConstants::NETWORK_FRAME_USECS); auto now = p_high_resolution_clock::now(); // compute how long the last frame took - auto duration = chrono::duration_cast(now - timestamp); + auto duration = chrono::duration_cast(now - _startFrameTimestamp); - // set the new frame timestamp - timestamp = max(now, nextTimestamp); + _idealFrameTimestamp += chrono::microseconds(AudioConstants::NETWORK_FRAME_USECS); { auto timer = _sleepTiming.timer(); - - // sleep until the next frame should start - // WIN32 sleep_until is broken until VS2015 Update 2 - // instead, max (above) guarantees that timestamp >= now, so we can sleep_for - this_thread::sleep_for(timestamp - now); + this_thread::sleep_until(_idealFrameTimestamp); } + _startFrameTimestamp = p_high_resolution_clock::now(); + return duration; } diff --git a/assignment-client/src/audio/AudioMixer.h b/assignment-client/src/audio/AudioMixer.h index 99b01683bf..b8ea0d5c58 100644 --- a/assignment-client/src/audio/AudioMixer.h +++ b/assignment-client/src/audio/AudioMixer.h @@ -84,7 +84,7 @@ private slots: private: // mixing helpers - std::chrono::microseconds timeFrame(p_high_resolution_clock::time_point& timestamp); + std::chrono::microseconds timeFrame(); void throttle(std::chrono::microseconds frameDuration, int frame); AudioMixerClientData* getOrCreateClientData(Node* node); @@ -94,6 +94,9 @@ private: void parseSettingsObject(const QJsonObject& settingsObject); void clearDomainSettings(); + p_high_resolution_clock::time_point _idealFrameTimestamp; + p_high_resolution_clock::time_point _startFrameTimestamp; + float _trailingMixRatio { 0.0f }; float _throttlingRatio { 0.0f }; From 88fd42010d3d7dabe80efbfc604c2b99f77a050c Mon Sep 17 00:00:00 2001 From: Stephen Birarda Date: Fri, 21 Sep 2018 17:18:09 -0700 Subject: [PATCH 13/21] make sure the audio stream stats size is right --- libraries/audio/src/AudioStreamStats.h | 2 ++ 1 file changed, 2 insertions(+) diff --git a/libraries/audio/src/AudioStreamStats.h b/libraries/audio/src/AudioStreamStats.h index 1230c14706..021abd5199 100644 --- a/libraries/audio/src/AudioStreamStats.h +++ b/libraries/audio/src/AudioStreamStats.h @@ -66,4 +66,6 @@ public: PacketStreamStats _packetStreamWindowStats; }; +static_assert(sizeof(AudioStreamStats) == 152, "AudioStreamStats size isn't right"); + #endif // hifi_AudioStreamStats_h From 2ac1445683f00c677e944d7f005ba6737e543177 Mon Sep 17 00:00:00 2001 From: Stephen Birarda Date: Mon, 24 Sep 2018 11:48:34 -0700 Subject: [PATCH 14/21] remove double scoping of the frame check timer --- assignment-client/src/audio/AudioMixer.cpp | 8 +++----- 1 file changed, 3 insertions(+), 5 deletions(-) diff --git a/assignment-client/src/audio/AudioMixer.cpp b/assignment-client/src/audio/AudioMixer.cpp index b5318a72d9..44579ba790 100644 --- a/assignment-client/src/audio/AudioMixer.cpp +++ b/assignment-client/src/audio/AudioMixer.cpp @@ -393,11 +393,9 @@ void AudioMixer::start() { if (_startFrameTimestamp.time_since_epoch().count() == 0) { _startFrameTimestamp = _idealFrameTimestamp = p_high_resolution_clock::now(); } else { - { - auto timer = _checkTimeTiming.timer(); - auto frameDuration = timeFrame(); - throttle(frameDuration, frame); - } + auto timer = _checkTimeTiming.timer(); + auto frameDuration = timeFrame(); + throttle(frameDuration, frame); } auto frameTimer = _frameTiming.timer(); From b7c6fa003dee1931f363094f6a7beeef0ead5b76 Mon Sep 17 00:00:00 2001 From: Stephen Birarda Date: Mon, 24 Sep 2018 15:54:10 -0700 Subject: [PATCH 15/21] guard against incorrectly sized AudioStreamStats from packet --- assignment-client/src/audio/AudioMixerClientData.cpp | 8 ++++++++ 1 file changed, 8 insertions(+) diff --git a/assignment-client/src/audio/AudioMixerClientData.cpp b/assignment-client/src/audio/AudioMixerClientData.cpp index 2af1a0fc08..667b0ee955 100644 --- a/assignment-client/src/audio/AudioMixerClientData.cpp +++ b/assignment-client/src/audio/AudioMixerClientData.cpp @@ -325,6 +325,14 @@ int AudioMixerClientData::parseData(ReceivedMessage& message) { // skip over header, appendFlag, and num stats packed message.seek(sizeof(quint8) + sizeof(quint16)); + if (message.getBytesLeftToRead() != sizeof(AudioStreamStats)) { + qWarning() << "Received AudioStreamStats of wrong size" << message.getBytesLeftToRead() + << "instead of" << sizeof(AudioStreamStats) << "from" + << message.getSourceID() << "at" << message.getSenderSockAddr(); + + return message.getPosition(); + } + // read the downstream audio stream stats message.readPrimitive(&_downstreamAudioStreamStats); From 7e4cfd3c78d5c0c6c91f79bb9afaa9de229d7378 Mon Sep 17 00:00:00 2001 From: Stephen Birarda Date: Fri, 7 Sep 2018 12:01:09 -0700 Subject: [PATCH 16/21] fix for injected audio going to unreachable --- assignment-client/src/audio/AudioMixerClientData.cpp | 1 + 1 file changed, 1 insertion(+) diff --git a/assignment-client/src/audio/AudioMixerClientData.cpp b/assignment-client/src/audio/AudioMixerClientData.cpp index 667b0ee955..ee98e94d41 100644 --- a/assignment-client/src/audio/AudioMixerClientData.cpp +++ b/assignment-client/src/audio/AudioMixerClientData.cpp @@ -367,6 +367,7 @@ bool AudioMixerClientData::containsValidPosition(ReceivedMessage& message) const case PacketType::InjectAudio: { // skip the stream ID, stereo flag, and loopback flag message.seek(message.getPosition() + NUM_STREAM_ID_BYTES + sizeof(ChannelFlag) + sizeof(LoopbackFlag)); + break; } default: Q_UNREACHABLE(); From b18d8e22540504dc400cea0d4dee77d1add49bbb Mon Sep 17 00:00:00 2001 From: Stephen Birarda Date: Tue, 25 Sep 2018 10:41:21 -0700 Subject: [PATCH 17/21] pull out removal and adding of streams to sep functions --- .../src/audio/AudioMixerSlave.cpp | 135 ++++++++++-------- assignment-client/src/audio/AudioMixerSlave.h | 3 + 2 files changed, 78 insertions(+), 60 deletions(-) diff --git a/assignment-client/src/audio/AudioMixerSlave.cpp b/assignment-client/src/audio/AudioMixerSlave.cpp index 9d2f3728a5..7c0566ee44 100644 --- a/assignment-client/src/audio/AudioMixerSlave.cpp +++ b/assignment-client/src/audio/AudioMixerSlave.cpp @@ -132,6 +132,77 @@ bool containsNodeID(const V& vector, QUuid nodeID) { }); } +void AudioMixerSlave::addStreams(Node& listener, AudioMixerClientData& listenerData) { + auto& ignoredNodeIDs = listener.getIgnoredNodeIDs(); + auto& ignoringNodeIDs = listenerData.getIgnoringNodeIDs(); + + auto& mixableStreams = listenerData.getMixableStreams(); + + // add data for newly created streams to our vector + if (!listenerData.getHasReceivedFirstMix()) { + // when this listener is new, we need to fill its added streams object with all available streams + std::for_each(_begin, _end, [&](const SharedNodePointer& node) { + AudioMixerClientData* nodeData = static_cast(node->getLinkedData()); + if (nodeData) { + for (auto& stream : nodeData->getAudioStreams()) { + mixableStreams.emplace_back(node->getUUID(), node->getLocalID(), + stream->getStreamIdentifier(), &(*stream)); + + // pre-populate ignored and ignoring flags for this stream + mixableStreams.back().ignoredByListener = containsNodeID(ignoredNodeIDs, node->getUUID()); + mixableStreams.back().ignoringListener = containsNodeID(ignoringNodeIDs, node->getUUID()); + } + } + }); + + // flag this listener as having received their first mix so we know we don't need to enumerate all nodes again + listenerData.setHasReceivedFirstMix(true); + } else { + for (const auto& newStream : _sharedData.addedStreams) { + mixableStreams.emplace_back(newStream.nodeIDStreamID, newStream.positionalStream); + + // pre-populate ignored and ignoring flags for this stream + mixableStreams.back().ignoredByListener = containsNodeID(ignoredNodeIDs, newStream.nodeIDStreamID.nodeID); + mixableStreams.back().ignoringListener = containsNodeID(ignoringNodeIDs, newStream.nodeIDStreamID.nodeID); + } + } +} + +void AudioMixerSlave::removeStreams(AudioMixerClientData::MixableStreamsVector& mixableStreams) { + if (_sharedData.removedNodes.size() > 0) { + // enumerate the available streams + auto it = mixableStreams.begin(); + auto end = mixableStreams.end(); + + while (it != end) { + // check if this node (and therefore all of the node's streams) has been removed + auto& nodeIDStreamID = it->nodeStreamID; + auto matchedRemovedNode = std::find(_sharedData.removedNodes.cbegin(), _sharedData.removedNodes.cend(), + nodeIDStreamID.nodeLocalID); + bool streamRemoved = matchedRemovedNode != _sharedData.removedNodes.cend(); + + // if the node wasn't removed, check if this stream was specifically removed + if (!streamRemoved) { + auto matchedRemovedStream = std::find(_sharedData.removedStreams.cbegin(), _sharedData.removedStreams.cend(), + nodeIDStreamID); + streamRemoved = matchedRemovedStream != _sharedData.removedStreams.cend(); + } + + if (streamRemoved) { + // this stream was removed, so swap it with the last item and decrease the end iterator + --end; + std::swap(*it, *end); + + // process the it element (which is now the element that was the last item before the swap) + continue; + } + } + + // erase any removed streams that were swapped to the end + mixableStreams.erase(end, mixableStreams.end()); + } +} + bool AudioMixerSlave::prepareMix(const SharedNodePointer& listener) { AvatarAudioStream* listenerAudioStream = static_cast(listener->getLinkedData())->getAvatarAudioStream(); AudioMixerClientData* listenerData = static_cast(listener->getLinkedData()); @@ -143,42 +214,7 @@ bool AudioMixerSlave::prepareMix(const SharedNodePointer& listener) { auto nodeList = DependencyManager::get(); -#ifdef HIFI_AUDIO_MIXER_DEBUG - auto mixStart = p_high_resolution_clock::now(); -#endif - auto& mixableStreams = listenerData->getMixableStreams(); - auto& ignoredNodeIDs = listener->getIgnoredNodeIDs(); - auto& ignoringNodeIDs = listenerData->getIgnoringNodeIDs(); - - // add data for newly created streams to our vector - if (!listenerData->getHasReceivedFirstMix()) { - // when this listener is new, we need to fill its added streams object with all available streams - std::for_each(_begin, _end, [&](const SharedNodePointer& node) { - AudioMixerClientData* nodeData = static_cast(node->getLinkedData()); - if (nodeData) { - for (auto& stream : nodeData->getAudioStreams()) { - mixableStreams.emplace_back(node->getUUID(), node->getLocalID(), - stream->getStreamIdentifier(), &(*stream)); - - // pre-populate ignored and ignoring flags for this stream - mixableStreams.back().ignoredByListener = containsNodeID(ignoredNodeIDs, node->getUUID()); - mixableStreams.back().ignoringListener = containsNodeID(ignoringNodeIDs, node->getUUID()); - } - } - }); - - // flag this listener as having received their first mix so we know we don't need to enumerate all nodes again - listenerData->setHasReceivedFirstMix(true); - } else { - for (const auto& newStream : _sharedData.addedStreams) { - mixableStreams.emplace_back(newStream.nodeIDStreamID, newStream.positionalStream); - - // pre-populate ignored and ignoring flags for this stream - mixableStreams.back().ignoredByListener = containsNodeID(ignoredNodeIDs, newStream.nodeIDStreamID.nodeID); - mixableStreams.back().ignoringListener = containsNodeID(ignoringNodeIDs, newStream.nodeIDStreamID.nodeID); - } - } // grab the unprocessed ignores and unignores from and for this listener const auto& nodesIgnoredByListener = listenerData->getNewIgnoredNodeIDs(); @@ -186,32 +222,14 @@ bool AudioMixerSlave::prepareMix(const SharedNodePointer& listener) { const auto& nodesIgnoringListener = listenerData->getNewIgnoringNodeIDs(); const auto& nodesUnignoringListener = listenerData->getNewUnignoringNodeIDs(); + removeStreams(mixableStreams); + addStreams(*listener, *listenerData); + // enumerate the available streams auto it = mixableStreams.begin(); - auto end = mixableStreams.end(); - while (it != end) { + while (it != mixableStreams.end()) { - // check if this node (and therefore all of the node's streams) has been removed auto& nodeIDStreamID = it->nodeStreamID; - auto matchedRemovedNode = std::find(_sharedData.removedNodes.cbegin(), _sharedData.removedNodes.cend(), - nodeIDStreamID.nodeLocalID); - bool streamRemoved = matchedRemovedNode != _sharedData.removedNodes.cend(); - - // if the node wasn't removed, check if this stream was specifically removed - if (!streamRemoved) { - auto matchedRemovedStream = std::find(_sharedData.removedStreams.cbegin(), _sharedData.removedStreams.cend(), - nodeIDStreamID); - streamRemoved = matchedRemovedStream != _sharedData.removedStreams.cend(); - } - - if (streamRemoved) { - // this stream was removed, so swap it with the last item and decrease the end iterator - --end; - std::swap(*it, *end); - - // process the it element (which is now the element that was the last item before the swap) - continue; - } if (it->nodeStreamID.nodeLocalID == listener->getLocalID()) { // streams from this node should be skipped unless loopback is specifically requested @@ -297,9 +315,6 @@ bool AudioMixerSlave::prepareMix(const SharedNodePointer& listener) { ++it; } - // erase any removed streams that were swapped to the end - mixableStreams.erase(end, mixableStreams.end()); - if (isThrottling) { // since we're throttling, we need to partition the mixable into throttled and unthrottled streams auto numToRetain = std::distance(_begin, _end) * (1 - _throttlingRatio); diff --git a/assignment-client/src/audio/AudioMixerSlave.h b/assignment-client/src/audio/AudioMixerSlave.h index da960cb1fd..a2860f3f75 100644 --- a/assignment-client/src/audio/AudioMixerSlave.h +++ b/assignment-client/src/audio/AudioMixerSlave.h @@ -58,6 +58,9 @@ private: void addStream(AudioMixerClientData::MixableStream& mixableStream, AvatarAudioStream& listeningNodeStream, float masterListenerGain, bool throttle); + void removeStreams(AudioMixerClientData::MixableStreamsVector& mixableStreams); + void addStreams(Node& listener, AudioMixerClientData& listenerData); + // mixing buffers float _mixSamples[AudioConstants::NETWORK_FRAME_SAMPLES_STEREO]; int16_t _bufferSamples[AudioConstants::NETWORK_FRAME_SAMPLES_STEREO]; From 2dc89f81a50272749a624a56bf67bd2a029641d7 Mon Sep 17 00:00:00 2001 From: Ken Cooke Date: Wed, 26 Sep 2018 18:18:20 -0700 Subject: [PATCH 18/21] Add HRTF reset() --- libraries/audio/src/AudioHRTF.h | 22 ++++++++++++++++++++++ 1 file changed, 22 insertions(+) diff --git a/libraries/audio/src/AudioHRTF.h b/libraries/audio/src/AudioHRTF.h index 8993842d6e..c50b4dfc0b 100644 --- a/libraries/audio/src/AudioHRTF.h +++ b/libraries/audio/src/AudioHRTF.h @@ -13,6 +13,7 @@ #define hifi_AudioHRTF_h #include +#include static const int HRTF_AZIMUTHS = 72; // 360 / 5-degree steps static const int HRTF_TAPS = 64; // minimum-phase FIR coefficients @@ -56,6 +57,27 @@ public: void setGainAdjustment(float gain) { _gainAdjust = HRTF_GAIN * gain; }; float getGainAdjustment() { return _gainAdjust; } + // clear internal state, but retain settings + void reset() { + // FIR history + memset(_firState, 0, sizeof(_firState)); + + // integer delay history + memset(_delayState, 0, sizeof(_delayState)); + + // biquad history + memset(_bqState, 0, sizeof(_bqState)); + + // parameter history + _azimuthState = 0.0f; + _distanceState = 0.0f; + _gainState = 0.0f; + + // _gainAdjust is retained + + _silentState = false; + } + private: AudioHRTF(const AudioHRTF&) = delete; AudioHRTF& operator=(const AudioHRTF&) = delete; From ac6dd57a30a2f3e33d85cb111d1d2aebeccd5282 Mon Sep 17 00:00:00 2001 From: Ken Cooke Date: Wed, 26 Sep 2018 18:23:26 -0700 Subject: [PATCH 19/21] HRTF starts in silent state --- libraries/audio/src/AudioHRTF.h | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/libraries/audio/src/AudioHRTF.h b/libraries/audio/src/AudioHRTF.h index c50b4dfc0b..65b28bc5f8 100644 --- a/libraries/audio/src/AudioHRTF.h +++ b/libraries/audio/src/AudioHRTF.h @@ -75,7 +75,7 @@ public: // _gainAdjust is retained - _silentState = false; + _silentState = true; } private: @@ -110,7 +110,7 @@ private: // global and local gain adjustment float _gainAdjust = HRTF_GAIN; - bool _silentState = false; + bool _silentState = true; }; #endif // AudioHRTF_h From 585c2784000a5b3cbd29de2709d8ffbb34852a1a Mon Sep 17 00:00:00 2001 From: Clement Date: Wed, 26 Sep 2018 10:26:19 -0700 Subject: [PATCH 20/21] streamline mix operation --- assignment-client/src/audio/AudioMixer.cpp | 27 +- .../src/audio/AudioMixerClientData.cpp | 4 +- .../src/audio/AudioMixerClientData.h | 13 +- .../src/audio/AudioMixerSlave.cpp | 503 +++++++++++------- assignment-client/src/audio/AudioMixerSlave.h | 14 +- .../src/audio/AudioMixerSlavePool.cpp | 6 +- .../src/audio/AudioMixerSlavePool.h | 4 +- .../src/audio/AudioMixerStats.cpp | 43 +- assignment-client/src/audio/AudioMixerStats.h | 17 +- libraries/audio/src/AudioHRTF.cpp | 17 +- libraries/audio/src/AudioHRTF.h | 37 +- 11 files changed, 409 insertions(+), 276 deletions(-) diff --git a/assignment-client/src/audio/AudioMixer.cpp b/assignment-client/src/audio/AudioMixer.cpp index 44579ba790..afd4047c68 100644 --- a/assignment-client/src/audio/AudioMixer.cpp +++ b/assignment-client/src/audio/AudioMixer.cpp @@ -301,14 +301,24 @@ void AudioMixer::sendStatsPacket() { QJsonObject mixStats; mixStats["%_hrtf_mixes"] = percentageForMixStats(_stats.hrtfRenders); - mixStats["%_hrtf_silent_mixes"] = percentageForMixStats(_stats.hrtfSilentRenders); - mixStats["%_hrtf_throttle_mixes"] = percentageForMixStats(_stats.hrtfThrottleRenders); - mixStats["%_skipped_throttle_mixes"] = percentageForMixStats(_stats.skippedThrottle); - mixStats["%_skipped_silent_mixes"] = percentageForMixStats(_stats.skippedSilent); - mixStats["%_skipped_other_mixes"] = percentageForMixStats(_stats.skippedOther); mixStats["%_manual_stereo_mixes"] = percentageForMixStats(_stats.manualStereoMixes); mixStats["%_manual_echo_mixes"] = percentageForMixStats(_stats.manualEchoMixes); + mixStats["1_hrtf_renders"] = (int)(_stats.hrtfRenders / (float)_numStatFrames); + mixStats["1_hrtf_resets"] = (int)(_stats.hrtfResets / (float)_numStatFrames); + mixStats["1_hrtf_updates"] = (int)(_stats.hrtfUpdates / (float)_numStatFrames); + + mixStats["2_skipped_streams"] = (int)(_stats.skipped / (float)_numStatFrames); + mixStats["2_inactive_streams"] = (int)(_stats.inactive / (float)_numStatFrames); + mixStats["2_active_streams"] = (int)(_stats.active / (float)_numStatFrames); + + mixStats["3_skippped_to_active"] = (int)(_stats.skippedToActive / (float)_numStatFrames); + mixStats["3_skippped_to_inactive"] = (int)(_stats.skippedToInactive / (float)_numStatFrames); + mixStats["3_inactive_to_skippped"] = (int)(_stats.inactiveToSkipped / (float)_numStatFrames); + mixStats["3_inactive_to_active"] = (int)(_stats.inactiveToActive / (float)_numStatFrames); + mixStats["3_active_to_skippped"] = (int)(_stats.activeToSkipped / (float)_numStatFrames); + mixStats["3_active_to_inactive"] = (int)(_stats.activeToInactive / (float)_numStatFrames); + mixStats["total_mixes"] = _stats.totalMixes; mixStats["avg_mixes_per_block"] = _stats.totalMixes / _numStatFrames; @@ -424,12 +434,11 @@ void AudioMixer::start() { QCoreApplication::processEvents(); } + int numToRetain = nodeList->size() * (1 - _throttlingRatio); nodeList->nestedEach([&](NodeList::const_iterator cbegin, NodeList::const_iterator cend) { // mix across slave threads - { - auto mixTimer = _mixTiming.timer(); - _slavePool.mix(cbegin, cend, frame, _throttlingRatio); - } + auto mixTimer = _mixTiming.timer(); + _slavePool.mix(cbegin, cend, frame, numToRetain); }); // gather stats diff --git a/assignment-client/src/audio/AudioMixerClientData.cpp b/assignment-client/src/audio/AudioMixerClientData.cpp index ee98e94d41..4545f48c41 100644 --- a/assignment-client/src/audio/AudioMixerClientData.cpp +++ b/assignment-client/src/audio/AudioMixerClientData.cpp @@ -203,11 +203,11 @@ void AudioMixerClientData::parsePerAvatarGainSet(ReceivedMessage& message, const } void AudioMixerClientData::setGainForAvatar(QUuid nodeID, uint8_t gain) { - auto it = std::find_if(_mixableStreams.cbegin(), _mixableStreams.cend(), [nodeID](const MixableStream& mixableStream){ + auto it = std::find_if(_streams.active.cbegin(), _streams.active.cend(), [nodeID](const MixableStream& mixableStream){ return mixableStream.nodeStreamID.nodeID == nodeID && mixableStream.nodeStreamID.streamID.isNull(); }); - if (it != _mixableStreams.cend()) { + if (it != _streams.active.cend()) { it->hrtf->setGainAdjustment(gain); } } diff --git a/assignment-client/src/audio/AudioMixerClientData.h b/assignment-client/src/audio/AudioMixerClientData.h index 332c6bfa51..610b258789 100644 --- a/assignment-client/src/audio/AudioMixerClientData.h +++ b/assignment-client/src/audio/AudioMixerClientData.h @@ -105,7 +105,7 @@ public: bool shouldMuteClient() { return _shouldMuteClient; } void setShouldMuteClient(bool shouldMuteClient) { _shouldMuteClient = shouldMuteClient; } glm::vec3 getPosition() { return getAvatarAudioStream() ? getAvatarAudioStream()->getPosition() : glm::vec3(0); } - bool getRequestsDomainListData() { return _requestsDomainListData; } + bool getRequestsDomainListData() const { return _requestsDomainListData; } void setRequestsDomainListData(bool requesting) { _requestsDomainListData = requesting; } void setupCodecForReplicatedAgent(QSharedPointer message); @@ -117,8 +117,6 @@ public: PositionalAudioStream* positionalStream; bool ignoredByListener { false }; bool ignoringListener { false }; - bool completedSilentRender { false }; - bool skippedStream { false }; MixableStream(NodeIDStreamID nodeIDStreamID, PositionalAudioStream* positionalStream) : nodeStreamID(nodeIDStreamID), hrtf(new AudioHRTF), positionalStream(positionalStream) {}; @@ -127,8 +125,13 @@ public: }; using MixableStreamsVector = std::vector; + struct Streams { + MixableStreamsVector active; + MixableStreamsVector inactive; + MixableStreamsVector skipped; + }; - MixableStreamsVector& getMixableStreams() { return _mixableStreams; } + Streams& getStreams() { return _streams; } // thread-safe, called from AudioMixerSlave(s) while processing ignore packets for other nodes void ignoredByNode(QUuid nodeID); @@ -173,7 +176,7 @@ private: bool containsValidPosition(ReceivedMessage& message) const; - MixableStreamsVector _mixableStreams; + Streams _streams; quint16 _outgoingMixedAudioSequenceNumber; diff --git a/assignment-client/src/audio/AudioMixerSlave.cpp b/assignment-client/src/audio/AudioMixerSlave.cpp index 7c0566ee44..8675eabe62 100644 --- a/assignment-client/src/audio/AudioMixerSlave.cpp +++ b/assignment-client/src/audio/AudioMixerSlave.cpp @@ -36,7 +36,10 @@ #include "InjectedAudioStream.h" #include "AudioHelpers.h" +using namespace std; using AudioStreamVector = AudioMixerClientData::AudioStreamVector; +using MixableStream = AudioMixerClientData::MixableStream; +using MixableStreamsVector = AudioMixerClientData::MixableStreamsVector; // packet helpers std::unique_ptr createAudioPacket(PacketType type, int size, quint16 sequence, QString codec); @@ -60,11 +63,11 @@ void AudioMixerSlave::processPackets(const SharedNodePointer& node) { } } -void AudioMixerSlave::configureMix(ConstIter begin, ConstIter end, unsigned int frame, float throttlingRatio) { +void AudioMixerSlave::configureMix(ConstIter begin, ConstIter end, unsigned int frame, int numToRetain) { _begin = begin; _end = end; _frame = frame; - _throttlingRatio = throttlingRatio; + _numToRetain = numToRetain; } void AudioMixerSlave::mix(const SharedNodePointer& node) { @@ -125,18 +128,61 @@ void AudioMixerSlave::mix(const SharedNodePointer& node) { } } -template -bool containsNodeID(const V& vector, QUuid nodeID) { - return std::any_of(std::begin(vector), std::end(vector), [&nodeID](const QUuid& vectorID){ - return vectorID == nodeID; + +template +void erase_if(Container& cont, Predicate&& pred) { + auto it = remove_if(begin(cont), end(cont), std::forward(pred)); + cont.erase(it, end(cont)); +} + +template +bool contains(const Container& cont, typename Container::value_type value) { + return std::any_of(begin(cont), end(cont), [&value](const auto& element) { + return value == element; }); } +// This class lets you do an erase if in several segments +// that use different predicates +template +class SegmentedEraseIf { +public: + using iterator = typename Container::iterator; + + SegmentedEraseIf(Container& cont) : _cont(cont) { + _first = begin(_cont); + _it = _first; + } + ~SegmentedEraseIf() { + assert(_it == end(_cont)); + _cont.erase(_first, _it); + } + + template + void iterateTo(iterator last, Predicate pred) { + while (_it != last) { + if (!pred(*_it)) { + if (_first != _it) { + *_first = move(*_it); + } + ++_first; + } + ++_it; + } + } + +private: + iterator _first; + iterator _it; + Container& _cont; +}; + + void AudioMixerSlave::addStreams(Node& listener, AudioMixerClientData& listenerData) { auto& ignoredNodeIDs = listener.getIgnoredNodeIDs(); auto& ignoringNodeIDs = listenerData.getIgnoringNodeIDs(); - auto& mixableStreams = listenerData.getMixableStreams(); + auto& streams = listenerData.getStreams(); // add data for newly created streams to our vector if (!listenerData.getHasReceivedFirstMix()) { @@ -145,12 +191,20 @@ void AudioMixerSlave::addStreams(Node& listener, AudioMixerClientData& listenerD AudioMixerClientData* nodeData = static_cast(node->getLinkedData()); if (nodeData) { for (auto& stream : nodeData->getAudioStreams()) { - mixableStreams.emplace_back(node->getUUID(), node->getLocalID(), - stream->getStreamIdentifier(), &(*stream)); + bool ignoredByListener = contains(ignoredNodeIDs, node->getUUID()); + bool ignoringListener = contains(ignoringNodeIDs, node->getUUID()); - // pre-populate ignored and ignoring flags for this stream - mixableStreams.back().ignoredByListener = containsNodeID(ignoredNodeIDs, node->getUUID()); - mixableStreams.back().ignoringListener = containsNodeID(ignoringNodeIDs, node->getUUID()); + if (ignoredByListener || ignoringListener) { + streams.skipped.emplace_back(node->getUUID(), node->getLocalID(), + stream->getStreamIdentifier(), stream.get()); + + // pre-populate ignored and ignoring flags for this stream + streams.skipped.back().ignoredByListener = ignoredByListener; + streams.skipped.back().ignoringListener = ignoringListener; + } else { + streams.active.emplace_back(node->getUUID(), node->getLocalID(), + stream->getStreamIdentifier(), stream.get()); + } } } }); @@ -159,49 +213,94 @@ void AudioMixerSlave::addStreams(Node& listener, AudioMixerClientData& listenerD listenerData.setHasReceivedFirstMix(true); } else { for (const auto& newStream : _sharedData.addedStreams) { - mixableStreams.emplace_back(newStream.nodeIDStreamID, newStream.positionalStream); + bool ignoredByListener = contains(ignoredNodeIDs, newStream.nodeIDStreamID.nodeID); + bool ignoringListener = contains(ignoringNodeIDs, newStream.nodeIDStreamID.nodeID); - // pre-populate ignored and ignoring flags for this stream - mixableStreams.back().ignoredByListener = containsNodeID(ignoredNodeIDs, newStream.nodeIDStreamID.nodeID); - mixableStreams.back().ignoringListener = containsNodeID(ignoringNodeIDs, newStream.nodeIDStreamID.nodeID); + if (ignoredByListener || ignoringListener) { + streams.skipped.emplace_back(newStream.nodeIDStreamID, newStream.positionalStream); + + // pre-populate ignored and ignoring flags for this stream + streams.skipped.back().ignoredByListener = ignoredByListener; + streams.skipped.back().ignoringListener = ignoringListener; + } else { + streams.active.emplace_back(newStream.nodeIDStreamID, newStream.positionalStream); + } } } } -void AudioMixerSlave::removeStreams(AudioMixerClientData::MixableStreamsVector& mixableStreams) { - if (_sharedData.removedNodes.size() > 0) { - // enumerate the available streams - auto it = mixableStreams.begin(); - auto end = mixableStreams.end(); +bool shouldBeRemoved(const MixableStream& stream, const AudioMixerSlave::SharedData& sharedData) { + return (contains(sharedData.removedNodes, stream.nodeStreamID.nodeLocalID) || + contains(sharedData.removedStreams, stream.nodeStreamID)); +}; - while (it != end) { - // check if this node (and therefore all of the node's streams) has been removed - auto& nodeIDStreamID = it->nodeStreamID; - auto matchedRemovedNode = std::find(_sharedData.removedNodes.cbegin(), _sharedData.removedNodes.cend(), - nodeIDStreamID.nodeLocalID); - bool streamRemoved = matchedRemovedNode != _sharedData.removedNodes.cend(); +bool shouldBeInactive(MixableStream& stream) { + return (!stream.positionalStream->lastPopSucceeded() || + stream.positionalStream->getLastPopOutputLoudness() == 0.0f); +}; - // if the node wasn't removed, check if this stream was specifically removed - if (!streamRemoved) { - auto matchedRemovedStream = std::find(_sharedData.removedStreams.cbegin(), _sharedData.removedStreams.cend(), - nodeIDStreamID); - streamRemoved = matchedRemovedStream != _sharedData.removedStreams.cend(); - } +bool shouldBeSkipped(MixableStream& stream, const Node& listener, + const AvatarAudioStream& listenerAudioStream, + const AudioMixerClientData& listenerData) { - if (streamRemoved) { - // this stream was removed, so swap it with the last item and decrease the end iterator - --end; - std::swap(*it, *end); - - // process the it element (which is now the element that was the last item before the swap) - continue; - } - } - - // erase any removed streams that were swapped to the end - mixableStreams.erase(end, mixableStreams.end()); + if (stream.nodeStreamID.nodeLocalID == listener.getLocalID()) { + return !stream.positionalStream->shouldLoopbackForNode(); } -} + + // grab the unprocessed ignores and unignores from and for this listener + const auto& nodesIgnoredByListener = listenerData.getNewIgnoredNodeIDs(); + const auto& nodesUnignoredByListener = listenerData.getNewUnignoredNodeIDs(); + const auto& nodesIgnoringListener = listenerData.getNewIgnoringNodeIDs(); + const auto& nodesUnignoringListener = listenerData.getNewUnignoringNodeIDs(); + + // this stream was previously not ignored by the listener and we have some newly ignored streams + // check now if it is one of the ignored streams and flag it as such + if (stream.ignoredByListener) { + stream.ignoredByListener = !contains(nodesUnignoredByListener, stream.nodeStreamID.nodeID); + } else { + stream.ignoredByListener = contains(nodesIgnoredByListener, stream.nodeStreamID.nodeID); + } + + if (stream.ignoringListener) { + stream.ignoringListener = !contains(nodesUnignoringListener, stream.nodeStreamID.nodeID); + } else { + stream.ignoringListener = contains(nodesIgnoringListener, stream.nodeStreamID.nodeID); + } + + bool listenerIsAdmin = listenerData.getRequestsDomainListData() && listener.getCanKick(); + if (stream.ignoredByListener || (stream.ignoringListener && !listenerIsAdmin)) { + return true; + } + + bool shouldCheckIgnoreBox = (listenerAudioStream.isIgnoreBoxEnabled() || + stream.positionalStream->isIgnoreBoxEnabled()); + if (shouldCheckIgnoreBox && + listenerAudioStream.getIgnoreBox().touches(stream.positionalStream->getIgnoreBox())) { + return true; + } + + return false; +}; + +float approximateVolume(const MixableStream& stream, const AvatarAudioStream* listenerAudioStream) { + if (stream.positionalStream->getLastPopOutputTrailingLoudness() == 0.0f) { + return 0.0f; + } + + if (stream.positionalStream == listenerAudioStream) { + return 1.0f; + } + + // approximate the gain + float gain = approximateGain(*listenerAudioStream, *(stream.positionalStream)); + + // for avatar streams, modify by the set gain adjustment + if (stream.nodeStreamID.streamID.isNull()) { + gain *= stream.hrtf->getGainAdjustment(); + } + + return stream.positionalStream->getLastPopOutputTrailingLoudness() * gain; +}; bool AudioMixerSlave::prepareMix(const SharedNodePointer& listener) { AvatarAudioStream* listenerAudioStream = static_cast(listener->getLinkedData())->getAvatarAudioStream(); @@ -210,128 +309,154 @@ bool AudioMixerSlave::prepareMix(const SharedNodePointer& listener) { // zero out the mix for this listener memset(_mixSamples, 0, sizeof(_mixSamples)); - bool isThrottling = _throttlingRatio > 0.0f; + bool isThrottling = _numToRetain != -1; - auto nodeList = DependencyManager::get(); + auto& streams = listenerData->getStreams(); - auto& mixableStreams = listenerData->getMixableStreams(); - - // grab the unprocessed ignores and unignores from and for this listener - const auto& nodesIgnoredByListener = listenerData->getNewIgnoredNodeIDs(); - const auto& nodesUnignoredByListener = listenerData->getNewUnignoredNodeIDs(); - const auto& nodesIgnoringListener = listenerData->getNewIgnoringNodeIDs(); - const auto& nodesUnignoringListener = listenerData->getNewUnignoringNodeIDs(); - - removeStreams(mixableStreams); addStreams(*listener, *listenerData); - // enumerate the available streams - auto it = mixableStreams.begin(); - while (it != mixableStreams.end()) { + // Process skipped streams + erase_if(streams.skipped, [&](MixableStream& stream) { + if (shouldBeRemoved(stream, _sharedData)) { + return true; + } - auto& nodeIDStreamID = it->nodeStreamID; - - if (it->nodeStreamID.nodeLocalID == listener->getLocalID()) { - // streams from this node should be skipped unless loopback is specifically requested - if (it->positionalStream->shouldLoopbackForNode()) { - it->skippedStream = false; + if (!shouldBeSkipped(stream, *listener, *listenerAudioStream, *listenerData)) { + if (shouldBeInactive(stream)) { + streams.inactive.push_back(move(stream)); + ++stats.skippedToInactive; } else { - it->approximateVolume = 0.0f; - it->skippedStream = true; - it->completedSilentRender = true; - - // if we know we're skipping this stream, no more processing is required - // since we don't do silent HRTF renders for echo streams - ++it; - continue; - } - } else { - if (it->ignoredByListener && nodesUnignoredByListener.size() > 0) { - // this stream was previously ignored by the listener and we have some unignored streams - // check now if it is one of the unignored streams and flag it as such - it->ignoredByListener = !containsNodeID(nodesUnignoredByListener, nodeIDStreamID.nodeID); - - } else if (!it->ignoredByListener && nodesIgnoredByListener.size() > 0) { - // this stream was previously not ignored by the listener and we have some newly ignored streams - // check now if it is one of the ignored streams and flag it as such - it->ignoredByListener = containsNodeID(nodesIgnoredByListener, nodeIDStreamID.nodeID); - } - - if (it->ignoringListener && nodesUnignoringListener.size() > 0) { - // this stream was previously ignoring the listener and we have some new un-ignoring nodes - // check now if it is one of the unignoring streams and flag it as such - it->ignoringListener = !containsNodeID(nodesUnignoringListener, nodeIDStreamID.nodeID); - } else if (!it->ignoringListener && nodesIgnoringListener.size() > 0) { - it->ignoringListener = containsNodeID(nodesIgnoringListener, nodeIDStreamID.nodeID); - } - - if (it->ignoredByListener - || (it->ignoringListener && !(listenerData->getRequestsDomainListData() && listener->getCanKick()))) { - // this is a stream ignoring by the listener - // or ignoring the listener (and the listener is not an admin asking for (the poorly named) "domain list" data) - // mark it skipped and move on - it->skippedStream = true; - } else { - it->skippedStream = false; - } - - if (!it->skippedStream) { - if ((listenerAudioStream->isIgnoreBoxEnabled() || it->positionalStream->isIgnoreBoxEnabled()) - && listenerAudioStream->getIgnoreBox().touches(it->positionalStream->getIgnoreBox())) { - // the listener is ignoring audio sources within a radius, and this source is in that radius - // so we mark it skipped - it->skippedStream = true; - } else { - it->skippedStream = false; - } + streams.active.push_back(move(stream)); + ++stats.skippedToActive; } + return true; } if (!isThrottling) { - // we aren't throttling, so we already know that we can add this stream to the mix - addStream(*it, *listenerAudioStream, listenerData->getMasterAvatarGain(), false); - } else { + updateHRTFParameters(stream, *listenerAudioStream, + listenerData->getMasterAvatarGain()); + } + return false; + }); + + // Process inactive streams + erase_if(streams.inactive, [&](MixableStream& stream) { + if (shouldBeRemoved(stream, _sharedData)) { + return true; + } + + if (shouldBeSkipped(stream, *listener, *listenerAudioStream, *listenerData)) { + streams.skipped.push_back(move(stream)); + ++stats.inactiveToSkipped; + return true; + } + + if (!shouldBeInactive(stream)) { + streams.active.push_back(move(stream)); + ++stats.inactiveToActive; + return true; + } + + if (!isThrottling) { + updateHRTFParameters(stream, *listenerAudioStream, + listenerData->getMasterAvatarGain()); + } + return false; + }); + + // Process active streams + erase_if(streams.active, [&](MixableStream& stream) { + if (shouldBeRemoved(stream, _sharedData)) { + return true; + } + + if (isThrottling) { // we're throttling, so we need to update the approximate volume for any un-skipped streams // unless this is simply for an echo (in which case the approx volume is 1.0) - if (!it->skippedStream && it->positionalStream->getLastPopOutputTrailingLoudness() > 0.0f) { - if (it->positionalStream != listenerAudioStream) { - // approximate the gain - float gain = approximateGain(*listenerAudioStream, *(it->positionalStream)); + stream.approximateVolume = approximateVolume(stream, listenerAudioStream); + } else { + if (shouldBeSkipped(stream, *listener, *listenerAudioStream, *listenerData)) { + addStream(stream, *listenerAudioStream, 0.0f); + streams.skipped.push_back(move(stream)); + ++stats.activeToSkipped; + return true; + } - // for avatar streams, modify by the set gain adjustment - if (nodeIDStreamID.streamID.isNull()) { - gain *= it->hrtf->getGainAdjustment(); - } + addStream(stream, *listenerAudioStream, listenerData->getMasterAvatarGain()); - it->approximateVolume = it->positionalStream->getLastPopOutputTrailingLoudness() * gain; - } else { - it->approximateVolume = 1.0f; - } - } else { - it->approximateVolume = 0.0f; + if (shouldBeInactive(stream)) { + // To reduce artifacts we still call render to flush the HRTF for every silent + // sources on the first frame where the source becomes silent + // this ensures the correct tail from last mixed block + streams.inactive.push_back(move(stream)); + ++stats.activeToInactive; + return true; } } - ++it; - } + return false; + }); if (isThrottling) { // since we're throttling, we need to partition the mixable into throttled and unthrottled streams - auto numToRetain = std::distance(_begin, _end) * (1 - _throttlingRatio); - auto throttlePoint = mixableStreams.begin() + numToRetain; + int numToRetain = min(_numToRetain, (int)streams.active.size()); // Make sure we don't overflow + auto throttlePoint = begin(streams.active) + numToRetain; - std::nth_element(mixableStreams.begin(), throttlePoint, mixableStreams.end(), + std::nth_element(streams.active.begin(), throttlePoint, streams.active.end(), [](const auto& a, const auto& b) - { - return a.approximateVolume > b.approximateVolume; - }); + { + return a.approximateVolume > b.approximateVolume; + }); - for (auto it = mixableStreams.begin(); it != mixableStreams.end(); ++it) { - // add this stream, it is throttled if it is at or past the throttle iterator in the vector - addStream(*it, *listenerAudioStream, listenerData->getMasterAvatarGain(), it >= throttlePoint); - } + SegmentedEraseIf erase(streams.active); + erase.iterateTo(throttlePoint, [&](MixableStream& stream) { + if (shouldBeSkipped(stream, *listener, *listenerAudioStream, *listenerData)) { + resetHRTFState(stream); + streams.skipped.push_back(move(stream)); + ++stats.activeToSkipped; + return true; + } + + addStream(stream, *listenerAudioStream, listenerData->getMasterAvatarGain()); + + if (shouldBeInactive(stream)) { + // To reduce artifacts we still call render to flush the HRTF for every silent + // sources on the first frame where the source becomes silent + // this ensures the correct tail from last mixed block + streams.inactive.push_back(move(stream)); + ++stats.activeToInactive; + return true; + } + + return false; + }); + erase.iterateTo(end(streams.active), [&](MixableStream& stream) { + // To reduce artifacts we reset the HRTF state for every throttled + // sources on the first frame where the source becomes throttled + // this ensures at least remove the tail from last mixed block + // preventing excessive artifacts on the next first block + resetHRTFState(stream); + + if (shouldBeSkipped(stream, *listener, *listenerAudioStream, *listenerData)) { + streams.skipped.push_back(move(stream)); + ++stats.activeToSkipped; + return true; + } + + if (shouldBeInactive(stream)) { + streams.inactive.push_back(move(stream)); + ++stats.activeToInactive; + return true; + } + + return false; + }); } + stats.skipped += streams.skipped.size(); + stats.inactive += streams.inactive.size(); + stats.active += streams.active.size(); + // clear the newly ignored, un-ignored, ignoring, and un-ignoring streams now that we've processed them listenerData->clearStagedIgnoreChanges(); @@ -357,41 +482,13 @@ bool AudioMixerSlave::prepareMix(const SharedNodePointer& listener) { return hasAudio; } -void AudioMixerSlave::addStream(AudioMixerClientData::MixableStream& mixableStream, AvatarAudioStream& listeningNodeStream, - float masterListenerGain, bool throttle) { - - if (mixableStream.skippedStream) { - // any skipped stream gets no processing and no silent render - early return - ++stats.skippedOther; - return; - } - +void AudioMixerSlave::addStream(AudioMixerClientData::MixableStream& mixableStream, + AvatarAudioStream& listeningNodeStream, + float masterListenerGain) { ++stats.totalMixes; auto streamToAdd = mixableStream.positionalStream; - // to reduce artifacts we still call the HRTF functor for every silent or throttled source - // for the first frame where the source becomes throttled or silent - // this ensures the correct tail from last mixed block and the correct spatialization of next first block - if (throttle || streamToAdd->getLastPopOutputLoudness() == 0.0f) { - if (mixableStream.completedSilentRender) { - - if (streamToAdd->getLastPopOutputLoudness() == 0.0f) { - ++stats.skippedSilent; - } else { - ++stats.skippedThrottle; - } - - return; - } else { - mixableStream.completedSilentRender = true; - } - } else if (mixableStream.completedSilentRender) { - // a stream that is no longer throttled or silent should have its silent render flag reset to false - // so that we complete a silent render for the stream next time it is throttled or otherwise goes silent - mixableStream.completedSilentRender = false; - } - // check if this is a server echo of a source back to itself bool isEcho = (streamToAdd == &listeningNodeStream); @@ -400,6 +497,7 @@ void AudioMixerSlave::addStream(AudioMixerClientData::MixableStream& mixableStre float distance = glm::max(glm::length(relativePosition), EPSILON); float gain = computeGain(masterListenerGain, listeningNodeStream, *streamToAdd, relativePosition, distance, isEcho); float azimuth = isEcho ? 0.0f : computeAzimuth(listeningNodeStream, listeningNodeStream, relativePosition); + const int HRTF_DATASET_INDEX = 1; if (!streamToAdd->lastPopSucceeded()) { @@ -426,10 +524,10 @@ void AudioMixerSlave::addStream(AudioMixerClientData::MixableStream& mixableStre // (this is not done for stereo streams since they do not go through the HRTF) if (!streamToAdd->isStereo() && !isEcho) { static int16_t silentMonoBlock[AudioConstants::NETWORK_FRAME_SAMPLES_PER_CHANNEL] = {}; - mixableStream.hrtf->renderSilent(silentMonoBlock, _mixSamples, HRTF_DATASET_INDEX, azimuth, distance, gain, - AudioConstants::NETWORK_FRAME_SAMPLES_PER_CHANNEL); + mixableStream.hrtf->render(silentMonoBlock, _mixSamples, HRTF_DATASET_INDEX, azimuth, distance, gain, + AudioConstants::NETWORK_FRAME_SAMPLES_PER_CHANNEL); - ++stats.hrtfSilentRenders; + ++stats.hrtfRenders; } return; @@ -453,11 +551,8 @@ void AudioMixerSlave::addStream(AudioMixerClientData::MixableStream& mixableStre } ++stats.manualStereoMixes; - return; - } - - // echo sources are not passed through HRTF - if (isEcho) { + } else if (isEcho) { + // echo sources are not passed through HRTF const float scale = 1/32768.0f; // int16_t to float @@ -468,34 +563,38 @@ void AudioMixerSlave::addStream(AudioMixerClientData::MixableStream& mixableStre } ++stats.manualEchoMixes; - return; + } else { + streamPopOutput.readSamples(_bufferSamples, AudioConstants::NETWORK_FRAME_SAMPLES_PER_CHANNEL); + + mixableStream.hrtf->render(_bufferSamples, _mixSamples, HRTF_DATASET_INDEX, azimuth, distance, gain, + AudioConstants::NETWORK_FRAME_SAMPLES_PER_CHANNEL); + + ++stats.hrtfRenders; } +} +void AudioMixerSlave::updateHRTFParameters(AudioMixerClientData::MixableStream& mixableStream, + AvatarAudioStream& listeningNodeStream, + float masterListenerGain) { + auto streamToAdd = mixableStream.positionalStream; - streamPopOutput.readSamples(_bufferSamples, AudioConstants::NETWORK_FRAME_SAMPLES_PER_CHANNEL); + // check if this is a server echo of a source back to itself + bool isEcho = (streamToAdd == &listeningNodeStream); - if (streamToAdd->getLastPopOutputLoudness() == 0.0f) { - // call renderSilent to reduce artifacts - mixableStream.hrtf->renderSilent(_bufferSamples, _mixSamples, HRTF_DATASET_INDEX, azimuth, distance, gain, - AudioConstants::NETWORK_FRAME_SAMPLES_PER_CHANNEL); + glm::vec3 relativePosition = streamToAdd->getPosition() - listeningNodeStream.getPosition(); - ++stats.hrtfSilentRenders; - return; - } + float distance = glm::max(glm::length(relativePosition), EPSILON); + float gain = computeGain(masterListenerGain, listeningNodeStream, *streamToAdd, relativePosition, distance, isEcho); + float azimuth = isEcho ? 0.0f : computeAzimuth(listeningNodeStream, listeningNodeStream, relativePosition); - if (throttle) { - // call renderSilent with actual frame data and a gain of 0.0f to reduce artifacts - mixableStream.hrtf->renderSilent(_bufferSamples, _mixSamples, HRTF_DATASET_INDEX, azimuth, distance, 0.0f, - AudioConstants::NETWORK_FRAME_SAMPLES_PER_CHANNEL); + mixableStream.hrtf->setParameterHistory(azimuth, distance, gain); - ++stats.hrtfThrottleRenders; - return; - } + ++stats.hrtfUpdates; +} - mixableStream.hrtf->render(_bufferSamples, _mixSamples, HRTF_DATASET_INDEX, azimuth, distance, gain, - AudioConstants::NETWORK_FRAME_SAMPLES_PER_CHANNEL); - - ++stats.hrtfRenders; +void AudioMixerSlave::resetHRTFState(AudioMixerClientData::MixableStream& mixableStream) { + mixableStream.hrtf->reset(); + ++stats.hrtfResets; } std::unique_ptr createAudioPacket(PacketType type, int size, quint16 sequence, QString codec) { diff --git a/assignment-client/src/audio/AudioMixerSlave.h b/assignment-client/src/audio/AudioMixerSlave.h index a2860f3f75..6566c839b8 100644 --- a/assignment-client/src/audio/AudioMixerSlave.h +++ b/assignment-client/src/audio/AudioMixerSlave.h @@ -44,7 +44,7 @@ public: void processPackets(const SharedNodePointer& node); // configure a round of mixing - void configureMix(ConstIter begin, ConstIter end, unsigned int frame, float throttlingRatio); + void configureMix(ConstIter begin, ConstIter end, unsigned int frame, int numToRetain); // mix and broadcast non-ignored streams to the node (requires configuration using configureMix, above) // returns true if a mixed packet was sent to the node @@ -55,10 +55,14 @@ public: private: // create mix, returns true if mix has audio bool prepareMix(const SharedNodePointer& listener); - void addStream(AudioMixerClientData::MixableStream& mixableStream, AvatarAudioStream& listeningNodeStream, - float masterListenerGain, bool throttle); + void addStream(AudioMixerClientData::MixableStream& mixableStream, + AvatarAudioStream& listeningNodeStream, + float masterListenerGain); + void updateHRTFParameters(AudioMixerClientData::MixableStream& mixableStream, + AvatarAudioStream& listeningNodeStream, + float masterListenerGain); + void resetHRTFState(AudioMixerClientData::MixableStream& mixableStream); - void removeStreams(AudioMixerClientData::MixableStreamsVector& mixableStreams); void addStreams(Node& listener, AudioMixerClientData& listenerData); // mixing buffers @@ -69,7 +73,7 @@ private: ConstIter _begin; ConstIter _end; unsigned int _frame { 0 }; - float _throttlingRatio { 0.0f }; + int _numToRetain { -1 }; SharedData& _sharedData; }; diff --git a/assignment-client/src/audio/AudioMixerSlavePool.cpp b/assignment-client/src/audio/AudioMixerSlavePool.cpp index d51ba010d7..7cc7ac9f93 100644 --- a/assignment-client/src/audio/AudioMixerSlavePool.cpp +++ b/assignment-client/src/audio/AudioMixerSlavePool.cpp @@ -74,13 +74,11 @@ void AudioMixerSlavePool::processPackets(ConstIter begin, ConstIter end) { run(begin, end); } -void AudioMixerSlavePool::mix(ConstIter begin, ConstIter end, unsigned int frame, float throttlingRatio) { +void AudioMixerSlavePool::mix(ConstIter begin, ConstIter end, unsigned int frame, int numToRetain) { _function = &AudioMixerSlave::mix; _configure = [=](AudioMixerSlave& slave) { - slave.configureMix(_begin, _end, _frame, _throttlingRatio); + slave.configureMix(_begin, _end, frame, numToRetain); }; - _frame = frame; - _throttlingRatio = throttlingRatio; run(begin, end); } diff --git a/assignment-client/src/audio/AudioMixerSlavePool.h b/assignment-client/src/audio/AudioMixerSlavePool.h index c9487686b5..82b892123c 100644 --- a/assignment-client/src/audio/AudioMixerSlavePool.h +++ b/assignment-client/src/audio/AudioMixerSlavePool.h @@ -67,7 +67,7 @@ public: void processPackets(ConstIter begin, ConstIter end); // mix on slave threads - void mix(ConstIter begin, ConstIter end, unsigned int frame, float throttlingRatio); + void mix(ConstIter begin, ConstIter end, unsigned int frame, int numToRetain); // iterate over all slaves void each(std::function functor); @@ -98,8 +98,6 @@ private: // frame state Queue _queue; - unsigned int _frame { 0 }; - float _throttlingRatio { 0.0f }; ConstIter _begin; ConstIter _end; diff --git a/assignment-client/src/audio/AudioMixerStats.cpp b/assignment-client/src/audio/AudioMixerStats.cpp index 213c0ba5e6..bb2daa1d2d 100644 --- a/assignment-client/src/audio/AudioMixerStats.cpp +++ b/assignment-client/src/audio/AudioMixerStats.cpp @@ -15,15 +15,27 @@ void AudioMixerStats::reset() { sumStreams = 0; sumListeners = 0; sumListenersSilent = 0; + totalMixes = 0; + hrtfRenders = 0; - hrtfSilentRenders = 0; - hrtfThrottleRenders = 0; + hrtfResets = 0; + hrtfUpdates = 0; + manualStereoMixes = 0; manualEchoMixes = 0; - skippedThrottle = 0; - skippedSilent = 0; - skippedOther = 0; + + skippedToActive = 0; + skippedToInactive = 0; + inactiveToSkipped = 0; + inactiveToActive = 0; + activeToSkipped = 0; + activeToInactive = 0; + + skipped = 0; + inactive = 0; + active = 0; + #ifdef HIFI_AUDIO_MIXER_DEBUG mixTime = 0; #endif @@ -33,15 +45,26 @@ void AudioMixerStats::accumulate(const AudioMixerStats& otherStats) { sumStreams += otherStats.sumStreams; sumListeners += otherStats.sumListeners; sumListenersSilent += otherStats.sumListenersSilent; + totalMixes += otherStats.totalMixes; + hrtfRenders += otherStats.hrtfRenders; - hrtfSilentRenders += otherStats.hrtfSilentRenders; - hrtfThrottleRenders += otherStats.hrtfThrottleRenders; + hrtfResets += otherStats.hrtfResets; + hrtfUpdates += otherStats.hrtfUpdates; + manualStereoMixes += otherStats.manualStereoMixes; manualEchoMixes += otherStats.manualEchoMixes; - skippedThrottle += otherStats.skippedThrottle; - skippedSilent += otherStats.skippedSilent; - skippedOther += otherStats.skippedOther; + + skippedToActive += otherStats.skippedToActive; + skippedToInactive += otherStats.skippedToInactive; + inactiveToSkipped += otherStats.inactiveToSkipped; + inactiveToActive += otherStats.inactiveToActive; + activeToSkipped += otherStats.activeToSkipped; + activeToInactive += otherStats.activeToInactive; + + skipped += otherStats.skipped; + inactive += otherStats.inactive; + active += otherStats.active; #ifdef HIFI_AUDIO_MIXER_DEBUG mixTime += otherStats.mixTime; diff --git a/assignment-client/src/audio/AudioMixerStats.h b/assignment-client/src/audio/AudioMixerStats.h index 41eb295214..459cbfc970 100644 --- a/assignment-client/src/audio/AudioMixerStats.h +++ b/assignment-client/src/audio/AudioMixerStats.h @@ -24,15 +24,22 @@ struct AudioMixerStats { int totalMixes { 0 }; int hrtfRenders { 0 }; - int hrtfSilentRenders { 0 }; - int hrtfThrottleRenders { 0 }; + int hrtfResets { 0 }; + int hrtfUpdates { 0 }; int manualStereoMixes { 0 }; int manualEchoMixes { 0 }; - int skippedThrottle { 0 }; - int skippedSilent { 0 }; - int skippedOther { 0 }; + int skippedToActive { 0 }; + int skippedToInactive { 0 }; + int inactiveToSkipped { 0 }; + int inactiveToActive { 0 }; + int activeToSkipped { 0 }; + int activeToInactive { 0 }; + + int skipped { 0 }; + int inactive { 0 }; + int active { 0 }; #ifdef HIFI_AUDIO_MIXER_DEBUG uint64_t mixTime { 0 }; diff --git a/libraries/audio/src/AudioHRTF.cpp b/libraries/audio/src/AudioHRTF.cpp index b639301404..9de6440d67 100644 --- a/libraries/audio/src/AudioHRTF.cpp +++ b/libraries/audio/src/AudioHRTF.cpp @@ -1173,20 +1173,5 @@ void AudioHRTF::render(int16_t* input, float* output, int index, float azimuth, // crossfade old/new output and accumulate crossfade_4x2(bqBuffer, output, crossfadeTable, HRTF_BLOCK); - _silentState = false; -} - -void AudioHRTF::renderSilent(int16_t* input, float* output, int index, float azimuth, float distance, float gain, int numFrames) { - - // process the first silent block, to flush internal state - if (!_silentState) { - render(input, output, index, azimuth, distance, gain, numFrames); - } - - // new parameters become old - _azimuthState = azimuth; - _distanceState = distance; - _gainState = gain; - - _silentState = true; + _resetState = false; } diff --git a/libraries/audio/src/AudioHRTF.h b/libraries/audio/src/AudioHRTF.h index 65b28bc5f8..eeef66e10c 100644 --- a/libraries/audio/src/AudioHRTF.h +++ b/libraries/audio/src/AudioHRTF.h @@ -47,9 +47,14 @@ public: void render(int16_t* input, float* output, int index, float azimuth, float distance, float gain, int numFrames); // - // Fast path when input is known to be silent + // Fast path when input is known to be silent and state as been flushed // - void renderSilent(int16_t* input, float* output, int index, float azimuth, float distance, float gain, int numFrames); + void setParameterHistory(float azimuth, float distance, float gain) { + // new parameters become old + _azimuthState = azimuth; + _distanceState = distance; + _gainState = gain; + } // // HRTF local gain adjustment in amplitude (1.0 == unity) @@ -59,23 +64,25 @@ public: // clear internal state, but retain settings void reset() { - // FIR history - memset(_firState, 0, sizeof(_firState)); + if (!_resetState) { + // FIR history + memset(_firState, 0, sizeof(_firState)); - // integer delay history - memset(_delayState, 0, sizeof(_delayState)); + // integer delay history + memset(_delayState, 0, sizeof(_delayState)); - // biquad history - memset(_bqState, 0, sizeof(_bqState)); + // biquad history + memset(_bqState, 0, sizeof(_bqState)); - // parameter history - _azimuthState = 0.0f; - _distanceState = 0.0f; - _gainState = 0.0f; + // parameter history + _azimuthState = 0.0f; + _distanceState = 0.0f; + _gainState = 0.0f; - // _gainAdjust is retained + // _gainAdjust is retained - _silentState = true; + _resetState = true; + } } private: @@ -110,7 +117,7 @@ private: // global and local gain adjustment float _gainAdjust = HRTF_GAIN; - bool _silentState = true; + bool _resetState = true; }; #endif // AudioHRTF_h From 1997191e777716a5d3641f909d86ad3733a9b724 Mon Sep 17 00:00:00 2001 From: Clement Date: Mon, 1 Oct 2018 10:49:04 -0700 Subject: [PATCH 21/21] Fix MSVC warnings --- assignment-client/src/audio/AudioMixerSlave.cpp | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/assignment-client/src/audio/AudioMixerSlave.cpp b/assignment-client/src/audio/AudioMixerSlave.cpp index 8675eabe62..57bada47f1 100644 --- a/assignment-client/src/audio/AudioMixerSlave.cpp +++ b/assignment-client/src/audio/AudioMixerSlave.cpp @@ -453,9 +453,9 @@ bool AudioMixerSlave::prepareMix(const SharedNodePointer& listener) { }); } - stats.skipped += streams.skipped.size(); - stats.inactive += streams.inactive.size(); - stats.active += streams.active.size(); + stats.skipped += (int)streams.skipped.size(); + stats.inactive += (int)streams.inactive.size(); + stats.active += (int)streams.active.size(); // clear the newly ignored, un-ignored, ignoring, and un-ignoring streams now that we've processed them listenerData->clearStagedIgnoreChanges();