Mirrors/overte
3
0
Fork 0
mirror of https://github.com/overte-org/overte.git synced 2025-04-19 13:23:36 +02:00
Code Issues Projects Releases Packages Wiki Activity Actions 2

Merge pull request #14113 from Atlante45/feat/audio-mixer-throttle-equation

Browse source 
Audio Mixer Scaling
This commit is contained in:
John Conklin II 2018-10-03 13:17:28 -07:00 committed by GitHub
commit 4d7b2bb429
No known key found for this signature in database
GPG key ID: 4AEE18F83AFDEB23
27 changed files with 1109 additions and 698 deletions
Show stats Download patch file Download diff file Expand all files Collapse all files

230
assignment-client/src/audio/AudioMixer.cpp
View file

@ -38,6 +38,8 @@
#include "AvatarAudioStream.h"
#include "InjectedAudioStream.h"
using namespace std;
static const float DEFAULT_ATTENUATION_PER_DOUBLING_IN_DISTANCE = 0.5f; // attenuation = -6dB * log2(distance)
static const int DISABLE_STATIC_JITTER_FRAMES = -1;
static const float DEFAULT_NOISE_MUTING_THRESHOLD = 1.0f;
@ -49,11 +51,11 @@ static const QString AUDIO_THREADING_GROUP_KEY = "audio_threading";
int AudioMixer::_numStaticJitterFrames{ DISABLE_STATIC_JITTER_FRAMES };
float AudioMixer::_noiseMutingThreshold{ DEFAULT_NOISE_MUTING_THRESHOLD };
float AudioMixer::_attenuationPerDoublingInDistance{ DEFAULT_ATTENUATION_PER_DOUBLING_IN_DISTANCE };
std::map<QString, std::shared_ptr<CodecPlugin>> AudioMixer::_availableCodecs{ };
map<QString, shared_ptr<CodecPlugin>> AudioMixer::_availableCodecs{ };
QStringList AudioMixer::_codecPreferenceOrder{};
QHash<QString, AABox> AudioMixer::_audioZones;
QVector<AudioMixer::ZoneSettings> AudioMixer::_zoneSettings;
QVector<AudioMixer::ReverbSettings> AudioMixer::_zoneReverbSettings;
vector<AudioMixer::ZoneDescription> AudioMixer::_audioZones;
vector<AudioMixer::ZoneSettings> AudioMixer::_zoneSettings;
vector<AudioMixer::ReverbSettings> AudioMixer::_zoneReverbSettings;
AudioMixer::AudioMixer(ReceivedMessage& message) :
ThreadedAssignment(message)
@ -67,7 +69,7 @@ AudioMixer::AudioMixer(ReceivedMessage& message) :
_availableCodecs.clear(); // Make sure struct is clean
auto pluginManager = DependencyManager::set<PluginManager>();
auto codecPlugins = pluginManager->getCodecPlugins();
std::for_each(codecPlugins.cbegin(), codecPlugins.cend(),
for_each(codecPlugins.cbegin(), codecPlugins.cend(),
[&](const CodecPluginPointer& codec) {
_availableCodecs[codec->getName()] = codec;
});
@ -122,7 +124,7 @@ void AudioMixer::queueAudioPacket(QSharedPointer<ReceivedMessage> message, Share
void AudioMixer::queueReplicatedAudioPacket(QSharedPointer<ReceivedMessage> message) {
// make sure we have a replicated node for the original sender of the packet
auto nodeList = DependencyManager::get<NodeList>();
// Node ID is now part of user data, since replicated audio packets are non-sourced.
QUuid nodeID = QUuid::fromRfc4122(message->readWithoutCopy(NUM_BYTES_RFC4122_UUID));
@ -173,12 +175,12 @@ void AudioMixer::handleMuteEnvironmentPacket(QSharedPointer<ReceivedMessage> mes
}
}
const std::pair<QString, CodecPluginPointer> AudioMixer::negotiateCodec(std::vector<QString> codecs) {
const pair<QString, CodecPluginPointer> AudioMixer::negotiateCodec(vector<QString> codecs) {
QString selectedCodecName;
CodecPluginPointer selectedCodec;
// read the codecs requested (by the client)
int minPreference = std::numeric_limits<int>::max();
int minPreference = numeric_limits<int>::max();
for (auto& codec : codecs) {
if (_availableCodecs.count(codec) > 0) {
int preference = _codecPreferenceOrder.indexOf(codec);
@ -191,20 +193,9 @@ const std::pair<QString, CodecPluginPointer> AudioMixer::negotiateCodec(std::vec
}
}
return std::make_pair(selectedCodecName, _availableCodecs[selectedCodecName]);
return make_pair(selectedCodecName, _availableCodecs[selectedCodecName]);
}
void AudioMixer::handleNodeKilled(SharedNodePointer killedNode) {
// enumerate the connected listeners to remove HRTF objects for the disconnected node
auto nodeList = DependencyManager::get<NodeList>();
nodeList->eachNode([&killedNode](const SharedNodePointer& node) {
auto clientData = dynamic_cast<AudioMixerClientData*>(node->getLinkedData());
if (clientData) {
clientData->removeNode(killedNode->getUUID());
}
});
}
void AudioMixer::handleNodeMuteRequestPacket(QSharedPointer<ReceivedMessage> packet, SharedNodePointer sendingNode) {
auto nodeList = DependencyManager::get<NodeList>();
@ -223,32 +214,31 @@ void AudioMixer::handleNodeMuteRequestPacket(QSharedPointer<ReceivedMessage> pac
}
}
void AudioMixer::handleNodeKilled(SharedNodePointer killedNode) {
auto clientData = dynamic_cast<AudioMixerClientData*>(killedNode->getLinkedData());
if (clientData) {
// stage the removal of all streams from this node, workers handle when preparing mixes for listeners
_workerSharedData.removedNodes.emplace_back(killedNode->getLocalID());
}
}
void AudioMixer::handleKillAvatarPacket(QSharedPointer<ReceivedMessage> packet, SharedNodePointer sendingNode) {
auto clientData = dynamic_cast<AudioMixerClientData*>(sendingNode->getLinkedData());
if (clientData) {
clientData->removeAgentAvatarAudioStream();
auto nodeList = DependencyManager::get<NodeList>();
nodeList->eachNode([sendingNode](const SharedNodePointer& node){
auto listenerClientData = dynamic_cast<AudioMixerClientData*>(node->getLinkedData());
if (listenerClientData) {
listenerClientData->removeHRTFForStream(sendingNode->getUUID());
}
});
// stage a removal of the avatar audio stream from this Agent, workers handle when preparing mixes for listeners
_workerSharedData.removedStreams.emplace_back(sendingNode->getUUID(), sendingNode->getLocalID(), QUuid());
}
}
void AudioMixer::removeHRTFsForFinishedInjector(const QUuid& streamID) {
auto injectorClientData = qobject_cast<AudioMixerClientData*>(sender());
if (injectorClientData) {
// enumerate the connected listeners to remove HRTF objects for the disconnected injector
auto nodeList = DependencyManager::get<NodeList>();
nodeList->eachNode([injectorClientData, &streamID](const SharedNodePointer& node){
auto listenerClientData = dynamic_cast<AudioMixerClientData*>(node->getLinkedData());
if (listenerClientData) {
listenerClientData->removeHRTFForStream(injectorClientData->getNodeID(), streamID);
}
});
if (injectorClientData) {
// stage the removal of this stream, workers handle when preparing mixes for listeners
_workerSharedData.removedStreams.emplace_back(injectorClientData->getNodeID(), injectorClientData->getNodeLocalID(),
streamID);
}
}
@ -285,7 +275,7 @@ void AudioMixer::sendStatsPacket() {
// timing stats
QJsonObject timingStats;
auto addTiming = [&](Timer& timer, std::string name) {
auto addTiming = [&](Timer& timer, string name) {
uint64_t timing, trailing;
timer.get(timing, trailing);
timingStats[("us_per_" + name).c_str()] = (qint64)(timing / _numStatFrames);
@ -293,12 +283,12 @@ void AudioMixer::sendStatsPacket() {
};
addTiming(_ticTiming, "tic");
addTiming(_checkTimeTiming, "check_time");
addTiming(_sleepTiming, "sleep");
addTiming(_frameTiming, "frame");
addTiming(_prepareTiming, "prepare");
addTiming(_packetsTiming, "packets");
addTiming(_mixTiming, "mix");
addTiming(_eventsTiming, "events");
addTiming(_packetsTiming, "packets");
#ifdef HIFI_AUDIO_MIXER_DEBUG
timingStats["ns_per_mix"] = (_stats.totalMixes > 0) ? (float)(_stats.mixTime / _stats.totalMixes) : 0;
@ -311,11 +301,24 @@ void AudioMixer::sendStatsPacket() {
QJsonObject mixStats;
mixStats["%_hrtf_mixes"] = percentageForMixStats(_stats.hrtfRenders);
mixStats["%_hrtf_silent_mixes"] = percentageForMixStats(_stats.hrtfSilentRenders);
mixStats["%_hrtf_throttle_mixes"] = percentageForMixStats(_stats.hrtfThrottleRenders);
mixStats["%_manual_stereo_mixes"] = percentageForMixStats(_stats.manualStereoMixes);
mixStats["%_manual_echo_mixes"] = percentageForMixStats(_stats.manualEchoMixes);
mixStats["1_hrtf_renders"] = (int)(_stats.hrtfRenders / (float)_numStatFrames);
mixStats["1_hrtf_resets"] = (int)(_stats.hrtfResets / (float)_numStatFrames);
mixStats["1_hrtf_updates"] = (int)(_stats.hrtfUpdates / (float)_numStatFrames);
mixStats["2_skipped_streams"] = (int)(_stats.skipped / (float)_numStatFrames);
mixStats["2_inactive_streams"] = (int)(_stats.inactive / (float)_numStatFrames);
mixStats["2_active_streams"] = (int)(_stats.active / (float)_numStatFrames);
mixStats["3_skippped_to_active"] = (int)(_stats.skippedToActive / (float)_numStatFrames);
mixStats["3_skippped_to_inactive"] = (int)(_stats.skippedToInactive / (float)_numStatFrames);
mixStats["3_inactive_to_skippped"] = (int)(_stats.inactiveToSkipped / (float)_numStatFrames);
mixStats["3_inactive_to_active"] = (int)(_stats.inactiveToActive / (float)_numStatFrames);
mixStats["3_active_to_skippped"] = (int)(_stats.activeToSkipped / (float)_numStatFrames);
mixStats["3_active_to_inactive"] = (int)(_stats.activeToInactive / (float)_numStatFrames);
mixStats["total_mixes"] = _stats.totalMixes;
mixStats["avg_mixes_per_block"] = _stats.totalMixes / _numStatFrames;
@ -366,7 +369,7 @@ AudioMixerClientData* AudioMixer::getOrCreateClientData(Node* node) {
auto clientData = dynamic_cast<AudioMixerClientData*>(node->getLinkedData());
if (!clientData) {
node->setLinkedData(std::unique_ptr<NodeData> { new AudioMixerClientData(node->getUUID(), node->getLocalID()) });
node->setLinkedData(unique_ptr<NodeData> { new AudioMixerClientData(node->getUUID(), node->getLocalID()) });
clientData = dynamic_cast<AudioMixerClientData*>(node->getLinkedData());
connect(clientData, &AudioMixerClientData::injectorStreamFinished, this, &AudioMixer::removeHRTFsForFinishedInjector);
}
@ -393,33 +396,49 @@ void AudioMixer::start() {
// mix state
unsigned int frame = 1;
auto frameTimestamp = p_high_resolution_clock::now();
while (!_isFinished) {
auto ticTimer = _ticTiming.timer();
{
auto timer = _sleepTiming.timer();
auto frameDuration = timeFrame(frameTimestamp);
if (_startFrameTimestamp.time_since_epoch().count() == 0) {
_startFrameTimestamp = _idealFrameTimestamp = p_high_resolution_clock::now();
} else {
auto timer = _checkTimeTiming.timer();
auto frameDuration = timeFrame();
throttle(frameDuration, frame);
}
auto frameTimer = _frameTiming.timer();
nodeList->nestedEach([&](NodeList::const_iterator cbegin, NodeList::const_iterator cend) {
// prepare frames; pop off any new audio from their streams
{
auto prepareTimer = _prepareTiming.timer();
std::for_each(cbegin, cend, [&](const SharedNodePointer& node) {
_stats.sumStreams += prepareFrame(node, frame);
});
}
// process (node-isolated) audio packets across slave threads
{
auto packetsTimer = _packetsTiming.timer();
// first clear the concurrent vector of added streams that the slaves will add to when they process packets
_workerSharedData.addedStreams.clear();
nodeList->nestedEach([&](NodeList::const_iterator cbegin, NodeList::const_iterator cend) {
_slavePool.processPackets(cbegin, cend);
});
}
// process queued events (networking, global audio packets, &c.)
{
auto eventsTimer = _eventsTiming.timer();
// clear removed nodes and removed streams before we process events that will setup the new set
_workerSharedData.removedNodes.clear();
_workerSharedData.removedStreams.clear();
// since we're a while loop we need to yield to qt's event processing
QCoreApplication::processEvents();
}
int numToRetain = nodeList->size() * (1 - _throttlingRatio);
nodeList->nestedEach([&](NodeList::const_iterator cbegin, NodeList::const_iterator cend) {
// mix across slave threads
{
auto mixTimer = _mixTiming.timer();
_slavePool.mix(cbegin, cend, frame, _throttlingRatio);
}
auto mixTimer = _mixTiming.timer();
_slavePool.mix(cbegin, cend, frame, numToRetain);
});
// gather stats
@ -431,21 +450,6 @@ void AudioMixer::start() {
++frame;
++_numStatFrames;
// process queued events (networking, global audio packets, &c.)
{
auto eventsTimer = _eventsTiming.timer();
// since we're a while loop we need to yield to qt's event processing
QCoreApplication::processEvents();
// process (node-isolated) audio packets across slave threads
{
nodeList->nestedEach([&](NodeList::const_iterator cbegin, NodeList::const_iterator cend) {
auto packetsTimer = _packetsTiming.timer();
_slavePool.processPackets(cbegin, cend);
});
}
}
if (_isFinished) {
// alert qt eventing that this is finished
@ -455,26 +459,26 @@ void AudioMixer::start() {
}
}
std::chrono::microseconds AudioMixer::timeFrame(p_high_resolution_clock::time_point& timestamp) {
chrono::microseconds AudioMixer::timeFrame() {
// advance the next frame
auto nextTimestamp = timestamp + std::chrono::microseconds(AudioConstants::NETWORK_FRAME_USECS);
auto now = p_high_resolution_clock::now();
// compute how long the last frame took
auto duration = std::chrono::duration_cast<std::chrono::microseconds>(now - timestamp);
auto duration = chrono::duration_cast<chrono::microseconds>(now - _startFrameTimestamp);
// set the new frame timestamp
timestamp = std::max(now, nextTimestamp);
_idealFrameTimestamp += chrono::microseconds(AudioConstants::NETWORK_FRAME_USECS);
// sleep until the next frame should start
// WIN32 sleep_until is broken until VS2015 Update 2
// instead, std::max (above) guarantees that timestamp >= now, so we can sleep_for
std::this_thread::sleep_for(timestamp - now);
{
auto timer = _sleepTiming.timer();
this_thread::sleep_until(_idealFrameTimestamp);
}
_startFrameTimestamp = p_high_resolution_clock::now();
return duration;
}
void AudioMixer::throttle(std::chrono::microseconds duration, int frame) {
void AudioMixer::throttle(chrono::microseconds duration, int frame) {
// throttle using a modified proportional-integral controller
const float FRAME_TIME = 10000.0f;
float mixRatio = duration.count() / FRAME_TIME;
@ -508,28 +512,19 @@ void AudioMixer::throttle(std::chrono::microseconds duration, int frame) {
if (_trailingMixRatio > TARGET) {
int proportionalTerm = 1 + (_trailingMixRatio - TARGET) / 0.1f;
_throttlingRatio += THROTTLE_RATE * proportionalTerm;
_throttlingRatio = std::min(_throttlingRatio, 1.0f);
_throttlingRatio = min(_throttlingRatio, 1.0f);
qCDebug(audio) << "audio-mixer is struggling (" << _trailingMixRatio << "mix/sleep) - throttling"
<< _throttlingRatio << "of streams";
} else if (_throttlingRatio > 0.0f && _trailingMixRatio <= BACKOFF_TARGET) {
int proportionalTerm = 1 + (TARGET - _trailingMixRatio) / 0.2f;
_throttlingRatio -= BACKOFF_RATE * proportionalTerm;
_throttlingRatio = std::max(_throttlingRatio, 0.0f);
_throttlingRatio = max(_throttlingRatio, 0.0f);
qCDebug(audio) << "audio-mixer is recovering (" << _trailingMixRatio << "mix/sleep) - throttling"
<< _throttlingRatio << "of streams";
}
}
}
int AudioMixer::prepareFrame(const SharedNodePointer& node, unsigned int frame) {
AudioMixerClientData* data = (AudioMixerClientData*)node->getLinkedData();
if (data == nullptr) {
return 0;
}
return data->checkBuffersBeforeFrameSend();
}
void AudioMixer::clearDomainSettings() {
_numStaticJitterFrames = DISABLE_STATIC_JITTER_FRAMES;
_attenuationPerDoublingInDistance = DEFAULT_ATTENUATION_PER_DOUBLING_IN_DISTANCE;
@ -661,8 +656,11 @@ void AudioMixer::parseSettingsObject(const QJsonObject& settingsObject) {
const QString Y_MAX = "y_max";
const QString Z_MIN = "z_min";
const QString Z_MAX = "z_max";
foreach (const QString& zone, zones.keys()) {
QJsonObject zoneObject = zones[zone].toObject();
auto zoneNames = zones.keys();
_audioZones.reserve(zoneNames.length());
foreach (const QString& zoneName, zoneNames) {
QJsonObject zoneObject = zones[zoneName].toObject();
if (zoneObject.contains(X_MIN) && zoneObject.contains(X_MAX) && zoneObject.contains(Y_MIN) &&
zoneObject.contains(Y_MAX) && zoneObject.contains(Z_MIN) && zoneObject.contains(Z_MAX)) {
@ -686,8 +684,8 @@ void AudioMixer::parseSettingsObject(const QJsonObject& settingsObject) {
glm::vec3 corner(xMin, yMin, zMin);
glm::vec3 dimensions(xMax - xMin, yMax - yMin, zMax - zMin);
AABox zoneAABox(corner, dimensions);
_audioZones.insert(zone, zoneAABox);
qCDebug(audio) << "Added zone:" << zone << "(corner:" << corner << ", dimensions:" << dimensions << ")";
_audioZones.push_back({ zoneName, zoneAABox });
qCDebug(audio) << "Added zone:" << zoneName << "(corner:" << corner << ", dimensions:" << dimensions << ")";
}
}
}
@ -707,18 +705,28 @@ void AudioMixer::parseSettingsObject(const QJsonObject& settingsObject) {
coefficientObject.contains(LISTENER) &&
coefficientObject.contains(COEFFICIENT)) {
ZoneSettings settings;
auto itSource = find_if(begin(_audioZones), end(_audioZones), [&](const ZoneDescription& description) {
return description.name == coefficientObject.value(SOURCE).toString();
});
auto itListener = find_if(begin(_audioZones), end(_audioZones), [&](const ZoneDescription& description) {
return description.name == coefficientObject.value(LISTENER).toString();
});
bool ok;
settings.source = coefficientObject.value(SOURCE).toString();
settings.listener = coefficientObject.value(LISTENER).toString();
settings.coefficient = coefficientObject.value(COEFFICIENT).toString().toFloat(&ok);
float coefficient = coefficientObject.value(COEFFICIENT).toString().toFloat(&ok);
if (ok && settings.coefficient >= 0.0f && settings.coefficient <= 1.0f &&
_audioZones.contains(settings.source) && _audioZones.contains(settings.listener)) {
if (ok && coefficient >= 0.0f && coefficient <= 1.0f &&
itSource != end(_audioZones) &&
itListener != end(_audioZones)) {
ZoneSettings settings;
settings.source = itSource - begin(_audioZones);
settings.listener = itListener - begin(_audioZones);
settings.coefficient = coefficient;
_zoneSettings.push_back(settings);
qCDebug(audio) << "Added Coefficient:" << settings.source << settings.listener << settings.coefficient;
qCDebug(audio) << "Added Coefficient:" << itSource->name << itListener->name << settings.coefficient;
}
}
}
@ -739,19 +747,21 @@ void AudioMixer::parseSettingsObject(const QJsonObject& settingsObject) {
reverbObject.contains(WET_LEVEL)) {
bool okReverbTime, okWetLevel;
QString zone = reverbObject.value(ZONE).toString();
auto itZone = find_if(begin(_audioZones), end(_audioZones), [&](const ZoneDescription& description) {
return description.name == reverbObject.value(ZONE).toString();
});
float reverbTime = reverbObject.value(REVERB_TIME).toString().toFloat(&okReverbTime);
float wetLevel = reverbObject.value(WET_LEVEL).toString().toFloat(&okWetLevel);
if (okReverbTime && okWetLevel && _audioZones.contains(zone)) {
if (okReverbTime && okWetLevel && itZone != end(_audioZones)) {
ReverbSettings settings;
settings.zone = zone;
settings.zone = itZone - begin(_audioZones);
settings.reverbTime = reverbTime;
settings.wetLevel = wetLevel;
_zoneReverbSettings.push_back(settings);
qCDebug(audio) << "Added Reverb:" << zone << reverbTime << wetLevel;
qCDebug(audio) << "Added Reverb:" << itZone->name << reverbTime << wetLevel;
}
}
}
@ -764,7 +774,7 @@ AudioMixer::Timer::Timing::Timing(uint64_t& sum) : _sum(sum) {
}
AudioMixer::Timer::Timing::~Timing() {
_sum += std::chrono::duration_cast<std::chrono::microseconds>(p_high_resolution_clock::now() - _timing).count();
_sum += chrono::duration_cast<chrono::microseconds>(p_high_resolution_clock::now() - _timing).count();
}
void AudioMixer::Timer::get(uint64_t& timing, uint64_t& trailing) {

38
assignment-client/src/audio/AudioMixer.h
View file

@ -34,13 +34,18 @@ class AudioMixer : public ThreadedAssignment {
public:
AudioMixer(ReceivedMessage& message);
struct ZoneDescription {
QString name;
AABox area;
};
struct ZoneSettings {
QString source;
QString listener;
int source;
int listener;
float coefficient;
};
struct ReverbSettings {
QString zone;
int zone;
float reverbTime;
float wetLevel;
};
@ -48,9 +53,9 @@ public:
static int getStaticJitterFrames() { return _numStaticJitterFrames; }
static bool shouldMute(float quietestFrame) { return quietestFrame > _noiseMutingThreshold; }
static float getAttenuationPerDoublingInDistance() { return _attenuationPerDoublingInDistance; }
static const QHash<QString, AABox>& getAudioZones() { return _audioZones; }
static const QVector<ZoneSettings>& getZoneSettings() { return _zoneSettings; }
static const QVector<ReverbSettings>& getReverbSettings() { return _zoneReverbSettings; }
static const std::vector<ZoneDescription>& getAudioZones() { return _audioZones; }
static const std::vector<ZoneSettings>& getZoneSettings() { return _zoneSettings; }
static const std::vector<ReverbSettings>& getReverbSettings() { return _zoneReverbSettings; }
static const std::pair<QString, CodecPluginPointer> negotiateCodec(std::vector<QString> codecs);
static bool shouldReplicateTo(const Node& from, const Node& to) {
@ -79,11 +84,8 @@ private slots:
private:
// mixing helpers
std::chrono::microseconds timeFrame(p_high_resolution_clock::time_point& timestamp);
std::chrono::microseconds timeFrame();
void throttle(std::chrono::microseconds frameDuration, int frame);
// pop a frame from any streams on the node
// returns the number of available streams
int prepareFrame(const SharedNodePointer& node, unsigned int frame);
AudioMixerClientData* getOrCreateClientData(Node* node);
@ -92,6 +94,9 @@ private:
void parseSettingsObject(const QJsonObject& settingsObject);
void clearDomainSettings();
p_high_resolution_clock::time_point _idealFrameTimestamp;
p_high_resolution_clock::time_point _startFrameTimestamp;
float _trailingMixRatio { 0.0f };
float _throttlingRatio { 0.0f };
@ -100,7 +105,7 @@ private:
int _numStatFrames { 0 };
AudioMixerStats _stats;
AudioMixerSlavePool _slavePool;
AudioMixerSlavePool _slavePool { _workerSharedData };
class Timer {
public:
@ -123,7 +128,9 @@ private:
uint64_t _history[TIMER_TRAILING_SECONDS] {};
int _index { 0 };
};
Timer _ticTiming;
Timer _checkTimeTiming;
Timer _sleepTiming;
Timer _frameTiming;
Timer _prepareTiming;
@ -136,10 +143,13 @@ private:
static float _attenuationPerDoublingInDistance;
static std::map<QString, CodecPluginPointer> _availableCodecs;
static QStringList _codecPreferenceOrder;
static QHash<QString, AABox> _audioZones;
static QVector<ZoneSettings> _zoneSettings;
static QVector<ReverbSettings> _zoneReverbSettings;
static std::vector<ZoneDescription> _audioZones;
static std::vector<ZoneSettings> _zoneSettings;
static std::vector<ReverbSettings> _zoneReverbSettings;
AudioMixerSlave::SharedData _workerSharedData;
};
#endif // hifi_AudioMixer_h

524
assignment-client/src/audio/AudioMixerClientData.cpp
View file

@ -13,6 +13,8 @@
#include <random>
#include <glm/detail/func_common.hpp>
#include <QtCore/QDebug>
#include <QtCore/QJsonArray>
@ -28,7 +30,6 @@
AudioMixerClientData::AudioMixerClientData(const QUuid& nodeID, Node::LocalID nodeLocalID) :
NodeData(nodeID, nodeLocalID),
audioLimiter(AudioConstants::SAMPLE_RATE, AudioConstants::STEREO),
_ignoreZone(*this),
_outgoingMixedAudioSequenceNumber(0),
_downstreamAudioStreamStats()
{
@ -56,7 +57,7 @@ void AudioMixerClientData::queuePacket(QSharedPointer<ReceivedMessage> message,
_packetQueue.push(message);
}
void AudioMixerClientData::processPackets() {
int AudioMixerClientData::processPackets(ConcurrentAddedStreams& addedStreams) {
SharedNodePointer node = _packetQueue.node;
assert(_packetQueue.empty() || node);
_packetQueue.node.clear();
@ -69,22 +70,17 @@ void AudioMixerClientData::processPackets() {
case PacketType::MicrophoneAudioWithEcho:
case PacketType::InjectAudio:
case PacketType::SilentAudioFrame: {
if (node->isUpstream()) {
setupCodecForReplicatedAgent(packet);
}
QMutexLocker lock(&getMutex());
parseData(*packet);
processStreamPacket(*packet, addedStreams);
optionallyReplicatePacket(*packet, *node);
break;
}
case PacketType::AudioStreamStats: {
QMutexLocker lock(&getMutex());
parseData(*packet);
break;
}
case PacketType::NegotiateAudioFormat:
@ -109,6 +105,10 @@ void AudioMixerClientData::processPackets() {
_packetQueue.pop();
}
assert(_packetQueue.empty());
// now that we have processed all packets for this frame
// we can prepare the sources from this client to be ready for mixing
return checkBuffersBeforeFrameSend();
}
bool isReplicatedPacket(PacketType packetType) {
@ -186,63 +186,136 @@ void AudioMixerClientData::parseRequestsDomainListData(ReceivedMessage& message)
void AudioMixerClientData::parsePerAvatarGainSet(ReceivedMessage& message, const SharedNodePointer& node) {
QUuid uuid = node->getUUID();
// parse the UUID from the packet
QUuid avatarUuid = QUuid::fromRfc4122(message.readWithoutCopy(NUM_BYTES_RFC4122_UUID));
QUuid avatarUUID = QUuid::fromRfc4122(message.readWithoutCopy(NUM_BYTES_RFC4122_UUID));
uint8_t packedGain;
message.readPrimitive(&packedGain);
float gain = unpackFloatGainFromByte(packedGain);
if (avatarUuid.isNull()) {
if (avatarUUID.isNull()) {
// set the MASTER avatar gain
setMasterAvatarGain(gain);
qCDebug(audio) << "Setting MASTER avatar gain for " << uuid << " to " << gain;
} else {
// set the per-source avatar gain
hrtfForStream(avatarUuid, QUuid()).setGainAdjustment(gain);
qCDebug(audio) << "Setting avatar gain adjustment for hrtf[" << uuid << "][" << avatarUuid << "] to " << gain;
setGainForAvatar(avatarUUID, gain);
qCDebug(audio) << "Setting avatar gain adjustment for hrtf[" << uuid << "][" << avatarUUID << "] to " << gain;
}
}
void AudioMixerClientData::setGainForAvatar(QUuid nodeID, uint8_t gain) {
auto it = std::find_if(_streams.active.cbegin(), _streams.active.cend(), [nodeID](const MixableStream& mixableStream){
return mixableStream.nodeStreamID.nodeID == nodeID && mixableStream.nodeStreamID.streamID.isNull();
});
if (it != _streams.active.cend()) {
it->hrtf->setGainAdjustment(gain);
}
}
void AudioMixerClientData::parseNodeIgnoreRequest(QSharedPointer<ReceivedMessage> message, const SharedNodePointer& node) {
node->parseIgnoreRequestMessage(message);
auto ignoredNodesPair = node->parseIgnoreRequestMessage(message);
// we have a vector of ignored or unignored node UUIDs - update our internal data structures so that
// streams can be included or excluded next time a mix is being created
if (ignoredNodesPair.second) {
// we have newly ignored nodes, add them to our vector
_newIgnoredNodeIDs.insert(std::end(_newIgnoredNodeIDs),
std::begin(ignoredNodesPair.first), std::end(ignoredNodesPair.first));
} else {
// we have newly unignored nodes, add them to our vector
_newUnignoredNodeIDs.insert(std::end(_newUnignoredNodeIDs),
std::begin(ignoredNodesPair.first), std::end(ignoredNodesPair.first));
}
auto nodeList = DependencyManager::get<NodeList>();
for (auto& nodeID : ignoredNodesPair.first) {
auto otherNode = nodeList->nodeWithUUID(nodeID);
if (otherNode) {
auto otherNodeMixerClientData = static_cast<AudioMixerClientData*>(otherNode->getLinkedData());
if (otherNodeMixerClientData) {
if (ignoredNodesPair.second) {
otherNodeMixerClientData->ignoredByNode(getNodeID());
} else {
otherNodeMixerClientData->unignoredByNode(getNodeID());
}
}
}
}
}
void AudioMixerClientData::ignoredByNode(QUuid nodeID) {
// first add this ID to the concurrent vector for newly ignoring nodes
_newIgnoringNodeIDs.push_back(nodeID);
// now take a lock and on the consistent vector of ignoring nodes and make sure this node is in it
std::lock_guard<std::mutex> lock(_ignoringNodeIDsMutex);
if (std::find(_ignoringNodeIDs.begin(), _ignoringNodeIDs.end(), nodeID) == _ignoringNodeIDs.end()) {
_ignoringNodeIDs.push_back(nodeID);
}
}
void AudioMixerClientData::unignoredByNode(QUuid nodeID) {
// first add this ID to the concurrent vector for newly unignoring nodes
_newUnignoringNodeIDs.push_back(nodeID);
// now take a lock on the consistent vector of ignoring nodes and make sure this node isn't in it
std::lock_guard<std::mutex> lock(_ignoringNodeIDsMutex);
auto it = _ignoringNodeIDs.begin();
while (it != _ignoringNodeIDs.end()) {
if (*it == nodeID) {
it = _ignoringNodeIDs.erase(it);
} else {
++it;
}
}
}
void AudioMixerClientData::clearStagedIgnoreChanges() {
_newIgnoredNodeIDs.clear();
_newUnignoredNodeIDs.clear();
_newIgnoringNodeIDs.clear();
_newUnignoringNodeIDs.clear();
}
void AudioMixerClientData::parseRadiusIgnoreRequest(QSharedPointer<ReceivedMessage> message, const SharedNodePointer& node) {
node->parseIgnoreRadiusRequestMessage(message);
bool enabled;
message->readPrimitive(&enabled);
_isIgnoreRadiusEnabled = enabled;
auto avatarAudioStream = getAvatarAudioStream();
// if we have an avatar audio stream, tell it wether its ignore box should be enabled or disabled
if (avatarAudioStream) {
if (_isIgnoreRadiusEnabled) {
avatarAudioStream->enableIgnoreBox();
} else {
avatarAudioStream->disableIgnoreBox();
}
}
}
AvatarAudioStream* AudioMixerClientData::getAvatarAudioStream() {
QReadLocker readLocker { &_streamsLock };
auto it = std::find_if(_audioStreams.begin(), _audioStreams.end(), [](const SharedStreamPointer& stream){
return stream->getStreamIdentifier().isNull();
});
auto it = _audioStreams.find(QUuid());
if (it != _audioStreams.end()) {
return dynamic_cast<AvatarAudioStream*>(it->second.get());
return dynamic_cast<AvatarAudioStream*>(it->get());
}
// no mic stream found - return NULL
return NULL;
}
void AudioMixerClientData::removeHRTFForStream(const QUuid& nodeID, const QUuid& streamID) {
auto it = _nodeSourcesHRTFMap.find(nodeID);
if (it != _nodeSourcesHRTFMap.end()) {
// erase the stream with the given ID from the given node
it->second.erase(streamID);
// is the map for this node now empty?
// if so we can remove it
if (it->second.size() == 0) {
_nodeSourcesHRTFMap.erase(it);
}
}
}
void AudioMixerClientData::removeAgentAvatarAudioStream() {
QWriteLocker writeLocker { &_streamsLock };
auto it = _audioStreams.find(QUuid());
auto it = std::remove_if(_audioStreams.begin(), _audioStreams.end(), [](const SharedStreamPointer& stream){
return stream->getStreamIdentifier().isNull();
});
if (it != _audioStreams.end()) {
_audioStreams.erase(it);
}
writeLocker.unlock();
}
int AudioMixerClientData::parseData(ReceivedMessage& message) {
@ -252,128 +325,186 @@ int AudioMixerClientData::parseData(ReceivedMessage& message) {
// skip over header, appendFlag, and num stats packed
message.seek(sizeof(quint8) + sizeof(quint16));
if (message.getBytesLeftToRead() != sizeof(AudioStreamStats)) {
qWarning() << "Received AudioStreamStats of wrong size" << message.getBytesLeftToRead()
<< "instead of" << sizeof(AudioStreamStats) << "from"
<< message.getSourceID() << "at" << message.getSenderSockAddr();
return message.getPosition();
}
// read the downstream audio stream stats
message.readPrimitive(&_downstreamAudioStreamStats);
return message.getPosition();
} else {
SharedStreamPointer matchingStream;
bool isMicStream = false;
if (packetType == PacketType::MicrophoneAudioWithEcho
|| packetType == PacketType::ReplicatedMicrophoneAudioWithEcho
|| packetType == PacketType::MicrophoneAudioNoEcho
|| packetType == PacketType::ReplicatedMicrophoneAudioNoEcho
|| packetType == PacketType::SilentAudioFrame
|| packetType == PacketType::ReplicatedSilentAudioFrame) {
QWriteLocker writeLocker { &_streamsLock };
auto micStreamIt = _audioStreams.find(QUuid());
if (micStreamIt == _audioStreams.end()) {
// we don't have a mic stream yet, so add it
// hop past the sequence number that leads the packet
message.seek(sizeof(quint16));
// pull the codec string from the packet
auto codecString = message.readString();
// determine if the stream is stereo or not
bool isStereo;
if (packetType == PacketType::SilentAudioFrame
|| packetType == PacketType::ReplicatedSilentAudioFrame) {
quint16 numSilentSamples;
message.readPrimitive(&numSilentSamples);
isStereo = numSilentSamples == AudioConstants::NETWORK_FRAME_SAMPLES_STEREO;
} else {
quint8 channelFlag;
message.readPrimitive(&channelFlag);
isStereo = channelFlag == 1;
}
auto avatarAudioStream = new AvatarAudioStream(isStereo, AudioMixer::getStaticJitterFrames());
avatarAudioStream->setupCodec(_codec, _selectedCodecName, isStereo ? AudioConstants::STEREO : AudioConstants::MONO);
qCDebug(audio) << "creating new AvatarAudioStream... codec:" << _selectedCodecName << "isStereo:" << isStereo;
connect(avatarAudioStream, &InboundAudioStream::mismatchedAudioCodec,
this, &AudioMixerClientData::handleMismatchAudioFormat);
auto emplaced = _audioStreams.emplace(
QUuid(),
std::unique_ptr<PositionalAudioStream> { avatarAudioStream }
);
micStreamIt = emplaced.first;
}
matchingStream = micStreamIt->second;
writeLocker.unlock();
isMicStream = true;
} else if (packetType == PacketType::InjectAudio
|| packetType == PacketType::ReplicatedInjectAudio) {
// this is injected audio
// grab the stream identifier for this injected audio
message.seek(sizeof(quint16));
QUuid streamIdentifier = QUuid::fromRfc4122(message.readWithoutCopy(NUM_BYTES_RFC4122_UUID));
bool isStereo;
message.readPrimitive(&isStereo);
QWriteLocker writeLock { &_streamsLock };
auto streamIt = _audioStreams.find(streamIdentifier);
if (streamIt == _audioStreams.end()) {
// we don't have this injected stream yet, so add it
auto injectorStream = new InjectedAudioStream(streamIdentifier, isStereo, AudioMixer::getStaticJitterFrames());
#if INJECTORS_SUPPORT_CODECS
injectorStream->setupCodec(_codec, _selectedCodecName, isStereo ? AudioConstants::STEREO : AudioConstants::MONO);
qCDebug(audio) << "creating new injectorStream... codec:" << _selectedCodecName << "isStereo:" << isStereo;
#endif
auto emplaced = _audioStreams.emplace(
streamIdentifier,
std::unique_ptr<InjectedAudioStream> { injectorStream }
);
streamIt = emplaced.first;
}
matchingStream = streamIt->second;
writeLock.unlock();
}
// seek to the beginning of the packet so that the next reader is in the right spot
message.seek(0);
// check the overflow count before we parse data
auto overflowBefore = matchingStream->getOverflowCount();
auto parseResult = matchingStream->parseData(message);
if (matchingStream->getOverflowCount() > overflowBefore) {
qCDebug(audio) << "Just overflowed on stream from" << message.getSourceID() << "at" << message.getSenderSockAddr();
qCDebug(audio) << "This stream is for" << (isMicStream ? "microphone audio" : "injected audio");
}
return parseResult;
}
return 0;
}
int AudioMixerClientData::checkBuffersBeforeFrameSend() {
QWriteLocker writeLocker { &_streamsLock };
bool AudioMixerClientData::containsValidPosition(ReceivedMessage& message) const {
static const int SEQUENCE_NUMBER_BYTES = sizeof(quint16);
auto posBefore = message.getPosition();
message.seek(SEQUENCE_NUMBER_BYTES);
// skip over the codec string
message.readString();
switch (message.getType()) {
case PacketType::MicrophoneAudioNoEcho:
case PacketType::MicrophoneAudioWithEcho: {
// skip over the stereo flag
message.seek(message.getPosition() + sizeof(ChannelFlag));
break;
}
case PacketType::SilentAudioFrame: {
// skip the number of silent samples
message.seek(message.getPosition() + sizeof(SilentSamplesBytes));
break;
}
case PacketType::InjectAudio: {
// skip the stream ID, stereo flag, and loopback flag
message.seek(message.getPosition() + NUM_STREAM_ID_BYTES + sizeof(ChannelFlag) + sizeof(LoopbackFlag));
break;
}
default:
Q_UNREACHABLE();
break;
}
glm::vec3 peekPosition;
message.readPrimitive(&peekPosition);
// reset the position the message was at before we were called
message.seek(posBefore);
if (glm::any(glm::isnan(peekPosition))) {
return false;
}
return true;
}
void AudioMixerClientData::processStreamPacket(ReceivedMessage& message, ConcurrentAddedStreams &addedStreams) {
if (!containsValidPosition(message)) {
qDebug() << "Refusing to process audio stream from" << message.getSourceID() << "with invalid position";
return;
}
SharedStreamPointer matchingStream;
auto packetType = message.getType();
bool newStream = false;
if (packetType == PacketType::MicrophoneAudioWithEcho
|| packetType == PacketType::MicrophoneAudioNoEcho
|| packetType == PacketType::SilentAudioFrame) {
auto micStreamIt = std::find_if(_audioStreams.begin(), _audioStreams.end(), [](const SharedStreamPointer& stream){
return stream->getStreamIdentifier().isNull();
});
if (micStreamIt == _audioStreams.end()) {
// we don't have a mic stream yet, so add it
// hop past the sequence number that leads the packet
message.seek(sizeof(StreamSequenceNumber));
// pull the codec string from the packet
auto codecString = message.readString();
// determine if the stream is stereo or not
bool isStereo;
if (packetType == PacketType::SilentAudioFrame || packetType == PacketType::ReplicatedSilentAudioFrame) {
SilentSamplesBytes numSilentSamples;
message.readPrimitive(&numSilentSamples);
isStereo = numSilentSamples == AudioConstants::NETWORK_FRAME_SAMPLES_STEREO;
} else {
ChannelFlag channelFlag;
message.readPrimitive(&channelFlag);
isStereo = channelFlag == 1;
}
auto avatarAudioStream = new AvatarAudioStream(isStereo, AudioMixer::getStaticJitterFrames());
avatarAudioStream->setupCodec(_codec, _selectedCodecName, isStereo ? AudioConstants::STEREO : AudioConstants::MONO);
if (_isIgnoreRadiusEnabled) {
avatarAudioStream->enableIgnoreBox();
} else {
avatarAudioStream->disableIgnoreBox();
}
qCDebug(audio) << "creating new AvatarAudioStream... codec:" << _selectedCodecName << "isStereo:" << isStereo;
connect(avatarAudioStream, &InboundAudioStream::mismatchedAudioCodec,
this, &AudioMixerClientData::handleMismatchAudioFormat);
matchingStream = SharedStreamPointer(avatarAudioStream);
_audioStreams.push_back(matchingStream);
newStream = true;
} else {
matchingStream = *micStreamIt;
}
} else if (packetType == PacketType::InjectAudio) {
// this is injected audio
// skip the sequence number and codec string and grab the stream identifier for this injected audio
message.seek(sizeof(StreamSequenceNumber));
message.readString();
QUuid streamIdentifier = QUuid::fromRfc4122(message.readWithoutCopy(NUM_BYTES_RFC4122_UUID));
auto streamIt = std::find_if(_audioStreams.begin(), _audioStreams.end(), [&streamIdentifier](const SharedStreamPointer& stream) {
return stream->getStreamIdentifier() == streamIdentifier;
});
if (streamIt == _audioStreams.end()) {
bool isStereo;
message.readPrimitive(&isStereo);
// we don't have this injected stream yet, so add it
auto injectorStream = new InjectedAudioStream(streamIdentifier, isStereo, AudioMixer::getStaticJitterFrames());
#if INJECTORS_SUPPORT_CODECS
injectorStream->setupCodec(_codec, _selectedCodecName, isStereo ? AudioConstants::STEREO : AudioConstants::MONO);
qCDebug(audio) << "creating new injectorStream... codec:" << _selectedCodecName << "isStereo:" << isStereo;
#endif
matchingStream = SharedStreamPointer(injectorStream);
_audioStreams.push_back(matchingStream);
newStream = true;
} else {
matchingStream = *streamIt;
}
}
// seek to the beginning of the packet so that the next reader is in the right spot
message.seek(0);
// check the overflow count before we parse data
auto overflowBefore = matchingStream->getOverflowCount();
matchingStream->parseData(message);
if (matchingStream->getOverflowCount() > overflowBefore) {
qCDebug(audio) << "Just overflowed on stream" << matchingStream->getStreamIdentifier()
<< "from" << message.getSourceID();
}
if (newStream) {
// whenever a stream is added, push it to the concurrent vector of streams added this frame
addedStreams.emplace_back(getNodeID(), getNodeLocalID(), matchingStream->getStreamIdentifier(), matchingStream.get());
}
}
int AudioMixerClientData::checkBuffersBeforeFrameSend() {
auto it = _audioStreams.begin();
while (it != _audioStreams.end()) {
SharedStreamPointer stream = it->second;
SharedStreamPointer stream = *it;
if (stream->popFrames(1, true) > 0) {
stream->updateLastPopOutputLoudnessAndTrailingLoudness();
@ -388,7 +519,7 @@ int AudioMixerClientData::checkBuffersBeforeFrameSend() {
// this is an inactive injector, pull it from our streams
// first emit that it is finished so that the HRTF objects for this source can be cleaned up
emit injectorStreamFinished(it->second->getStreamIdentifier());
emit injectorStreamFinished(stream->getStreamIdentifier());
// erase the stream to drop our ref to the shared pointer and remove it
it = _audioStreams.erase(it);
@ -441,7 +572,7 @@ void AudioMixerClientData::sendAudioStreamStatsPackets(const SharedNodePointer&
// pack the calculated number of stream stats
for (int i = 0; i < numStreamStatsToPack; i++) {
PositionalAudioStream* stream = it->second.get();
PositionalAudioStream* stream = it->get();
stream->perSecondCallbackForUpdatingStats();
@ -513,12 +644,12 @@ QJsonObject AudioMixerClientData::getAudioStreamStats() {
QJsonArray injectorArray;
auto streamsCopy = getAudioStreams();
for (auto& injectorPair : streamsCopy) {
if (injectorPair.second->getType() == PositionalAudioStream::Injector) {
if (injectorPair->getType() == PositionalAudioStream::Injector) {
QJsonObject upstreamStats;
AudioStreamStats streamStats = injectorPair.second->getAudioStreamStats();
AudioStreamStats streamStats = injectorPair->getAudioStreamStats();
upstreamStats["inj.desired"] = streamStats._desiredJitterBufferFrames;
upstreamStats["desired_calc"] = injectorPair.second->getCalculatedJitterBufferFrames();
upstreamStats["desired_calc"] = injectorPair->getCalculatedJitterBufferFrames();
upstreamStats["available_avg_10s"] = streamStats._framesAvailableAverage;
upstreamStats["available"] = (double) streamStats._framesAvailable;
upstreamStats["unplayed"] = (double) streamStats._unplayedMs;
@ -609,99 +740,6 @@ void AudioMixerClientData::cleanupCodec() {
}
}
AudioMixerClientData::IgnoreZone& AudioMixerClientData::IgnoreZoneMemo::get(unsigned int frame) {
// check for a memoized zone
if (frame != _frame.load(std::memory_order_acquire)) {
AvatarAudioStream* stream = _data.getAvatarAudioStream();
// get the initial dimensions from the stream
glm::vec3 corner = stream ? stream->getAvatarBoundingBoxCorner() : glm::vec3(0);
glm::vec3 scale = stream ? stream->getAvatarBoundingBoxScale() : glm::vec3(0);
// enforce a minimum scale
static const glm::vec3 MIN_IGNORE_BOX_SCALE = glm::vec3(0.3f, 1.3f, 0.3f);
if (glm::any(glm::lessThan(scale, MIN_IGNORE_BOX_SCALE))) {
scale = MIN_IGNORE_BOX_SCALE;
}
// (this is arbitrary number determined empirically for comfort)
const float IGNORE_BOX_SCALE_FACTOR = 2.4f;
scale *= IGNORE_BOX_SCALE_FACTOR;
// create the box (we use a box for the zone for convenience)
AABox box(corner, scale);
// update the memoized zone
// This may be called by multiple threads concurrently,
// so take a lock and only update the memo if this call is first.
// This prevents concurrent updates from invalidating the returned reference
// (contingent on the preconditions listed in the header).
std::lock_guard<std::mutex> lock(_mutex);
if (frame != _frame.load(std::memory_order_acquire)) {
_zone = box;
unsigned int oldFrame = _frame.exchange(frame, std::memory_order_release);
Q_UNUSED(oldFrame);
}
}
return _zone;
}
void AudioMixerClientData::IgnoreNodeCache::cache(bool shouldIgnore) {
if (!_isCached) {
_shouldIgnore = shouldIgnore;
_isCached = true;
}
}
bool AudioMixerClientData::IgnoreNodeCache::isCached() {
return _isCached;
}
bool AudioMixerClientData::IgnoreNodeCache::shouldIgnore() {
bool ignore = _shouldIgnore;
_isCached = false;
return ignore;
}
bool AudioMixerClientData::shouldIgnore(const SharedNodePointer self, const SharedNodePointer node, unsigned int frame) {
// this is symmetric over self / node; if computed, it is cached in the other
// check the cache to avoid computation
auto& cache = _nodeSourcesIgnoreMap[node->getUUID()];
if (cache.isCached()) {
return cache.shouldIgnore();
}
AudioMixerClientData* nodeData = static_cast<AudioMixerClientData*>(node->getLinkedData());
if (!nodeData) {
return false;
}
// compute shouldIgnore
bool shouldIgnore = true;
if ( // the nodes are not ignoring each other explicitly (or are but get data regardless)
(!self->isIgnoringNodeWithID(node->getUUID()) ||
(nodeData->getRequestsDomainListData() && node->getCanKick())) &&
(!node->isIgnoringNodeWithID(self->getUUID()) ||
(getRequestsDomainListData() && self->getCanKick()))) {
// if either node is enabling an ignore radius, check their proximity
if ((self->isIgnoreRadiusEnabled() || node->isIgnoreRadiusEnabled())) {
auto& zone = _ignoreZone.get(frame);
auto& nodeZone = nodeData->_ignoreZone.get(frame);
shouldIgnore = zone.touches(nodeZone);
} else {
shouldIgnore = false;
}
}
// cache in node
nodeData->_nodeSourcesIgnoreMap[self->getUUID()].cache(shouldIgnore);
return shouldIgnore;
}
void AudioMixerClientData::setupCodecForReplicatedAgent(QSharedPointer<ReceivedMessage> message) {
// hop past the sequence number that leads the packet
message->seek(sizeof(quint16));

144
assignment-client/src/audio/AudioMixerClientData.h
View file

@ -14,6 +14,8 @@
#include <queue>
#include <tbb/concurrent_vector.h>
#include <QtCore/QJsonObject>
#include <AABox.h>
@ -30,39 +32,34 @@
class AudioMixerClientData : public NodeData {
Q_OBJECT
public:
struct AddedStream {
NodeIDStreamID nodeIDStreamID;
PositionalAudioStream* positionalStream;
AddedStream(QUuid nodeID, Node::LocalID localNodeID,
StreamID streamID, PositionalAudioStream* positionalStream) :
nodeIDStreamID(nodeID, localNodeID, streamID), positionalStream(positionalStream) {};
};
using ConcurrentAddedStreams = tbb::concurrent_vector<AddedStream>;
AudioMixerClientData(const QUuid& nodeID, Node::LocalID nodeLocalID);
~AudioMixerClientData();
using SharedStreamPointer = std::shared_ptr<PositionalAudioStream>;
using AudioStreamMap = std::unordered_map<QUuid, SharedStreamPointer>;
using AudioStreamVector = std::vector<SharedStreamPointer>;
void queuePacket(QSharedPointer<ReceivedMessage> packet, SharedNodePointer node);
void processPackets();
int processPackets(ConcurrentAddedStreams& addedStreams); // returns the number of available streams this frame
// locks the mutex to make a copy
AudioStreamMap getAudioStreams() { QReadLocker readLock { &_streamsLock }; return _audioStreams; }
AudioStreamVector& getAudioStreams() { return _audioStreams; }
AvatarAudioStream* getAvatarAudioStream();
// returns whether self (this data's node) should ignore node, memoized by frame
// precondition: frame is increasing after first call (including overflow wrap)
bool shouldIgnore(SharedNodePointer self, SharedNodePointer node, unsigned int frame);
// the following methods should be called from the AudioMixer assignment thread ONLY
// they are not thread-safe
// returns a new or existing HRTF object for the given stream from the given node
AudioHRTF& hrtfForStream(const QUuid& nodeID, const QUuid& streamID = QUuid()) { return _nodeSourcesHRTFMap[nodeID][streamID]; }
// removes an AudioHRTF object for a given stream
void removeHRTFForStream(const QUuid& nodeID, const QUuid& streamID = QUuid());
// remove all sources and data from this node
void removeNode(const QUuid& nodeID) { _nodeSourcesIgnoreMap.unsafe_erase(nodeID); _nodeSourcesHRTFMap.erase(nodeID); }
void removeAgentAvatarAudioStream();
// packet parsers
int parseData(ReceivedMessage& message) override;
void processStreamPacket(ReceivedMessage& message, ConcurrentAddedStreams& addedStreams);
void negotiateAudioFormat(ReceivedMessage& message, const SharedNodePointer& node);
void parseRequestsDomainListData(ReceivedMessage& message);
void parsePerAvatarGainSet(ReceivedMessage& message, const SharedNodePointer& node);
@ -108,11 +105,56 @@ public:
bool shouldMuteClient() { return _shouldMuteClient; }
void setShouldMuteClient(bool shouldMuteClient) { _shouldMuteClient = shouldMuteClient; }
glm::vec3 getPosition() { return getAvatarAudioStream() ? getAvatarAudioStream()->getPosition() : glm::vec3(0); }
bool getRequestsDomainListData() { return _requestsDomainListData; }
bool getRequestsDomainListData() const { return _requestsDomainListData; }
void setRequestsDomainListData(bool requesting) { _requestsDomainListData = requesting; }
void setupCodecForReplicatedAgent(QSharedPointer<ReceivedMessage> message);
struct MixableStream {
float approximateVolume { 0.0f };
NodeIDStreamID nodeStreamID;
std::unique_ptr<AudioHRTF> hrtf;
PositionalAudioStream* positionalStream;
bool ignoredByListener { false };
bool ignoringListener { false };
MixableStream(NodeIDStreamID nodeIDStreamID, PositionalAudioStream* positionalStream) :
nodeStreamID(nodeIDStreamID), hrtf(new AudioHRTF), positionalStream(positionalStream) {};
MixableStream(QUuid nodeID, Node::LocalID localNodeID, StreamID streamID, PositionalAudioStream* positionalStream) :
nodeStreamID(nodeID, localNodeID, streamID), hrtf(new AudioHRTF), positionalStream(positionalStream) {};
};
using MixableStreamsVector = std::vector<MixableStream>;
struct Streams {
MixableStreamsVector active;
MixableStreamsVector inactive;
MixableStreamsVector skipped;
};
Streams& getStreams() { return _streams; }
// thread-safe, called from AudioMixerSlave(s) while processing ignore packets for other nodes
void ignoredByNode(QUuid nodeID);
void unignoredByNode(QUuid nodeID);
// start of methods called non-concurrently from single AudioMixerSlave mixing for the owning node
const Node::IgnoredNodeIDs& getNewIgnoredNodeIDs() const { return _newIgnoredNodeIDs; }
const Node::IgnoredNodeIDs& getNewUnignoredNodeIDs() const { return _newUnignoredNodeIDs; }
using ConcurrentIgnoreNodeIDs = tbb::concurrent_vector<QUuid>;
const ConcurrentIgnoreNodeIDs& getNewIgnoringNodeIDs() const { return _newIgnoringNodeIDs; }
const ConcurrentIgnoreNodeIDs& getNewUnignoringNodeIDs() const { return _newUnignoringNodeIDs; }
void clearStagedIgnoreChanges();
const Node::IgnoredNodeIDs& getIgnoringNodeIDs() const { return _ignoringNodeIDs; }
bool getHasReceivedFirstMix() const { return _hasReceivedFirstMix; }
void setHasReceivedFirstMix(bool hasReceivedFirstMix) { _hasReceivedFirstMix = hasReceivedFirstMix; }
// end of methods called non-concurrently from single AudioMixerSlave
signals:
void injectorStreamFinished(const QUuid& streamIdentifier);
@ -126,52 +168,15 @@ private:
};
PacketQueue _packetQueue;
QReadWriteLock _streamsLock;
AudioStreamMap _audioStreams; // microphone stream from avatar is stored under key of null UUID
AudioStreamVector _audioStreams; // microphone stream from avatar has a null stream ID
void optionallyReplicatePacket(ReceivedMessage& packet, const Node& node);
using IgnoreZone = AABox;
class IgnoreZoneMemo {
public:
IgnoreZoneMemo(AudioMixerClientData& data) : _data(data) {}
void setGainForAvatar(QUuid nodeID, uint8_t gain);
// returns an ignore zone, memoized by frame (lockless if the zone is already memoized)
// preconditions:
// - frame is increasing after first call (including overflow wrap)
// - there are no references left from calls to getIgnoreZone(frame - 1)
IgnoreZone& get(unsigned int frame);
bool containsValidPosition(ReceivedMessage& message) const;
private:
AudioMixerClientData& _data;
IgnoreZone _zone;
std::atomic<unsigned int> _frame { 0 };
std::mutex _mutex;
};
IgnoreZoneMemo _ignoreZone;
class IgnoreNodeCache {
public:
// std::atomic is not copyable - always initialize uncached
IgnoreNodeCache() {}
IgnoreNodeCache(const IgnoreNodeCache& other) {}
void cache(bool shouldIgnore);
bool isCached();
bool shouldIgnore();
private:
std::atomic<bool> _isCached { false };
bool _shouldIgnore { false };
};
struct IgnoreNodeCacheHasher { std::size_t operator()(const QUuid& key) const { return qHash(key); } };
using NodeSourcesIgnoreMap = tbb::concurrent_unordered_map<QUuid, IgnoreNodeCache, IgnoreNodeCacheHasher>;
NodeSourcesIgnoreMap _nodeSourcesIgnoreMap;
using HRTFMap = std::unordered_map<QUuid, AudioHRTF>;
using NodeSourcesHRTFMap = std::unordered_map<QUuid, HRTFMap>;
NodeSourcesHRTFMap _nodeSourcesHRTFMap;
Streams _streams;
quint16 _outgoingMixedAudioSequenceNumber;
@ -190,6 +195,21 @@ private:
bool _shouldMuteClient { false };
bool _requestsDomainListData { false };
std::vector<AddedStream> _newAddedStreams;
Node::IgnoredNodeIDs _newIgnoredNodeIDs;
Node::IgnoredNodeIDs _newUnignoredNodeIDs;
tbb::concurrent_vector<QUuid> _newIgnoringNodeIDs;
tbb::concurrent_vector<QUuid> _newUnignoringNodeIDs;
std::mutex _ignoringNodeIDsMutex;
Node::IgnoredNodeIDs _ignoringNodeIDs;
std::atomic_bool _isIgnoreRadiusEnabled { false };
bool _hasReceivedFirstMix { false };
};
#endif // hifi_AudioMixerClientData_h

530
assignment-client/src/audio/AudioMixerSlave.cpp
View file

@ -36,7 +36,10 @@
#include "InjectedAudioStream.h"
#include "AudioHelpers.h"
using AudioStreamMap = AudioMixerClientData::AudioStreamMap;
using namespace std;
using AudioStreamVector = AudioMixerClientData::AudioStreamVector;
using MixableStream = AudioMixerClientData::MixableStream;
using MixableStreamsVector = AudioMixerClientData::MixableStreamsVector;
// packet helpers
std::unique_ptr<NLPacket> createAudioPacket(PacketType type, int size, quint16 sequence, QString codec);
@ -46,9 +49,8 @@ void sendMutePacket(const SharedNodePointer& node, AudioMixerClientData&);
void sendEnvironmentPacket(const SharedNodePointer& node, AudioMixerClientData& data);
// mix helpers
inline float approximateGain(const AvatarAudioStream& listeningNodeStream, const PositionalAudioStream& streamToAdd,
const glm::vec3& relativePosition);
inline float computeGain(const AudioMixerClientData& listenerNodeData, const AvatarAudioStream& listeningNodeStream,
inline float approximateGain(const AvatarAudioStream& listeningNodeStream, const PositionalAudioStream& streamToAdd);
inline float computeGain(float masterListenerGain, const AvatarAudioStream& listeningNodeStream,
const PositionalAudioStream& streamToAdd, const glm::vec3& relativePosition, float distance, bool isEcho);
inline float computeAzimuth(const AvatarAudioStream& listeningNodeStream, const PositionalAudioStream& streamToAdd,
const glm::vec3& relativePosition);
@ -56,15 +58,16 @@ inline float computeAzimuth(const AvatarAudioStream& listeningNodeStream, const
void AudioMixerSlave::processPackets(const SharedNodePointer& node) {
AudioMixerClientData* data = (AudioMixerClientData*)node->getLinkedData();
if (data) {
data->processPackets();
// process packets and collect the number of streams available for this frame
stats.sumStreams += data->processPackets(_sharedData.addedStreams);
}
}
void AudioMixerSlave::configureMix(ConstIter begin, ConstIter end, unsigned int frame, float throttlingRatio) {
void AudioMixerSlave::configureMix(ConstIter begin, ConstIter end, unsigned int frame, int numToRetain) {
_begin = begin;
_end = end;
_frame = frame;
_throttlingRatio = throttlingRatio;
_numToRetain = numToRetain;
}
void AudioMixerSlave::mix(const SharedNodePointer& node) {
@ -125,105 +128,338 @@ void AudioMixerSlave::mix(const SharedNodePointer& node) {
}
}
template <class Container, class Predicate>
void erase_if(Container& cont, Predicate&& pred) {
auto it = remove_if(begin(cont), end(cont), std::forward<Predicate>(pred));
cont.erase(it, end(cont));
}
template <class Container>
bool contains(const Container& cont, typename Container::value_type value) {
return std::any_of(begin(cont), end(cont), [&value](const auto& element) {
return value == element;
});
}
// This class lets you do an erase if in several segments
// that use different predicates
template <class Container>
class SegmentedEraseIf {
public:
using iterator = typename Container::iterator;
SegmentedEraseIf(Container& cont) : _cont(cont) {
_first = begin(_cont);
_it = _first;
}
~SegmentedEraseIf() {
assert(_it == end(_cont));
_cont.erase(_first, _it);
}
template <class Predicate>
void iterateTo(iterator last, Predicate pred) {
while (_it != last) {
if (!pred(*_it)) {
if (_first != _it) {
*_first = move(*_it);
}
++_first;
}
++_it;
}
}
private:
iterator _first;
iterator _it;
Container& _cont;
};
void AudioMixerSlave::addStreams(Node& listener, AudioMixerClientData& listenerData) {
auto& ignoredNodeIDs = listener.getIgnoredNodeIDs();
auto& ignoringNodeIDs = listenerData.getIgnoringNodeIDs();
auto& streams = listenerData.getStreams();
// add data for newly created streams to our vector
if (!listenerData.getHasReceivedFirstMix()) {
// when this listener is new, we need to fill its added streams object with all available streams
std::for_each(_begin, _end, [&](const SharedNodePointer& node) {
AudioMixerClientData* nodeData = static_cast<AudioMixerClientData*>(node->getLinkedData());
if (nodeData) {
for (auto& stream : nodeData->getAudioStreams()) {
bool ignoredByListener = contains(ignoredNodeIDs, node->getUUID());
bool ignoringListener = contains(ignoringNodeIDs, node->getUUID());
if (ignoredByListener || ignoringListener) {
streams.skipped.emplace_back(node->getUUID(), node->getLocalID(),
stream->getStreamIdentifier(), stream.get());
// pre-populate ignored and ignoring flags for this stream
streams.skipped.back().ignoredByListener = ignoredByListener;
streams.skipped.back().ignoringListener = ignoringListener;
} else {
streams.active.emplace_back(node->getUUID(), node->getLocalID(),
stream->getStreamIdentifier(), stream.get());
}
}
}
});
// flag this listener as having received their first mix so we know we don't need to enumerate all nodes again
listenerData.setHasReceivedFirstMix(true);
} else {
for (const auto& newStream : _sharedData.addedStreams) {
bool ignoredByListener = contains(ignoredNodeIDs, newStream.nodeIDStreamID.nodeID);
bool ignoringListener = contains(ignoringNodeIDs, newStream.nodeIDStreamID.nodeID);
if (ignoredByListener || ignoringListener) {
streams.skipped.emplace_back(newStream.nodeIDStreamID, newStream.positionalStream);
// pre-populate ignored and ignoring flags for this stream
streams.skipped.back().ignoredByListener = ignoredByListener;
streams.skipped.back().ignoringListener = ignoringListener;
} else {
streams.active.emplace_back(newStream.nodeIDStreamID, newStream.positionalStream);
}
}
}
}
bool shouldBeRemoved(const MixableStream& stream, const AudioMixerSlave::SharedData& sharedData) {
return (contains(sharedData.removedNodes, stream.nodeStreamID.nodeLocalID) ||
contains(sharedData.removedStreams, stream.nodeStreamID));
};
bool shouldBeInactive(MixableStream& stream) {
return (!stream.positionalStream->lastPopSucceeded() ||
stream.positionalStream->getLastPopOutputLoudness() == 0.0f);
};
bool shouldBeSkipped(MixableStream& stream, const Node& listener,
const AvatarAudioStream& listenerAudioStream,
const AudioMixerClientData& listenerData) {
if (stream.nodeStreamID.nodeLocalID == listener.getLocalID()) {
return !stream.positionalStream->shouldLoopbackForNode();
}
// grab the unprocessed ignores and unignores from and for this listener
const auto& nodesIgnoredByListener = listenerData.getNewIgnoredNodeIDs();
const auto& nodesUnignoredByListener = listenerData.getNewUnignoredNodeIDs();
const auto& nodesIgnoringListener = listenerData.getNewIgnoringNodeIDs();
const auto& nodesUnignoringListener = listenerData.getNewUnignoringNodeIDs();
// this stream was previously not ignored by the listener and we have some newly ignored streams
// check now if it is one of the ignored streams and flag it as such
if (stream.ignoredByListener) {
stream.ignoredByListener = !contains(nodesUnignoredByListener, stream.nodeStreamID.nodeID);
} else {
stream.ignoredByListener = contains(nodesIgnoredByListener, stream.nodeStreamID.nodeID);
}
if (stream.ignoringListener) {
stream.ignoringListener = !contains(nodesUnignoringListener, stream.nodeStreamID.nodeID);
} else {
stream.ignoringListener = contains(nodesIgnoringListener, stream.nodeStreamID.nodeID);
}
bool listenerIsAdmin = listenerData.getRequestsDomainListData() && listener.getCanKick();
if (stream.ignoredByListener || (stream.ignoringListener && !listenerIsAdmin)) {
return true;
}
bool shouldCheckIgnoreBox = (listenerAudioStream.isIgnoreBoxEnabled() ||
stream.positionalStream->isIgnoreBoxEnabled());
if (shouldCheckIgnoreBox &&
listenerAudioStream.getIgnoreBox().touches(stream.positionalStream->getIgnoreBox())) {
return true;
}
return false;
};
float approximateVolume(const MixableStream& stream, const AvatarAudioStream* listenerAudioStream) {
if (stream.positionalStream->getLastPopOutputTrailingLoudness() == 0.0f) {
return 0.0f;
}
if (stream.positionalStream == listenerAudioStream) {
return 1.0f;
}
// approximate the gain
float gain = approximateGain(*listenerAudioStream, *(stream.positionalStream));
// for avatar streams, modify by the set gain adjustment
if (stream.nodeStreamID.streamID.isNull()) {
gain *= stream.hrtf->getGainAdjustment();
}
return stream.positionalStream->getLastPopOutputTrailingLoudness() * gain;
};
bool AudioMixerSlave::prepareMix(const SharedNodePointer& listener) {
AvatarAudioStream* listenerAudioStream = static_cast<AudioMixerClientData*>(listener->getLinkedData())->getAvatarAudioStream();
AudioMixerClientData* listenerData = static_cast<AudioMixerClientData*>(listener->getLinkedData());
// if we received an invalid position from this listener, then refuse to make them a mix
// because we don't know how to do it properly
if (!listenerAudioStream->hasValidPosition()) {
return false;
}
// zero out the mix for this listener
memset(_mixSamples, 0, sizeof(_mixSamples));
bool isThrottling = _throttlingRatio > 0.0f;
std::vector<std::pair<float, SharedNodePointer>> throttledNodes;
bool isThrottling = _numToRetain != -1;
typedef void (AudioMixerSlave::*MixFunctor)(
AudioMixerClientData&, const QUuid&, const AvatarAudioStream&, const PositionalAudioStream&);
auto forAllStreams = [&](const SharedNodePointer& node, AudioMixerClientData* nodeData, MixFunctor mixFunctor) {
auto nodeID = node->getUUID();
for (auto& streamPair : nodeData->getAudioStreams()) {
auto nodeStream = streamPair.second;
(this->*mixFunctor)(*listenerData, nodeID, *listenerAudioStream, *nodeStream);
}
};
auto& streams = listenerData->getStreams();
#ifdef HIFI_AUDIO_MIXER_DEBUG
auto mixStart = p_high_resolution_clock::now();
#endif
addStreams(*listener, *listenerData);
std::for_each(_begin, _end, [&](const SharedNodePointer& node) {
AudioMixerClientData* nodeData = static_cast<AudioMixerClientData*>(node->getLinkedData());
if (!nodeData) {
return;
// Process skipped streams
erase_if(streams.skipped, [&](MixableStream& stream) {
if (shouldBeRemoved(stream, _sharedData)) {
return true;
}
if (*node == *listener) {
// only mix the echo, if requested
for (auto& streamPair : nodeData->getAudioStreams()) {
auto nodeStream = streamPair.second;
if (nodeStream->shouldLoopbackForNode()) {
mixStream(*listenerData, node->getUUID(), *listenerAudioStream, *nodeStream);
}
}
} else if (!listenerData->shouldIgnore(listener, node, _frame)) {
if (!isThrottling) {
forAllStreams(node, nodeData, &AudioMixerSlave::mixStream);
if (!shouldBeSkipped(stream, *listener, *listenerAudioStream, *listenerData)) {
if (shouldBeInactive(stream)) {
streams.inactive.push_back(move(stream));
++stats.skippedToInactive;
} else {
auto nodeID = node->getUUID();
streams.active.push_back(move(stream));
++stats.skippedToActive;
}
return true;
}
// compute the node's max relative volume
float nodeVolume = 0.0f;
for (auto& streamPair : nodeData->getAudioStreams()) {
auto nodeStream = streamPair.second;
if (!isThrottling) {
updateHRTFParameters(stream, *listenerAudioStream,
listenerData->getMasterAvatarGain());
}
return false;
});
// approximate the gain
glm::vec3 relativePosition = nodeStream->getPosition() - listenerAudioStream->getPosition();
float gain = approximateGain(*listenerAudioStream, *nodeStream, relativePosition);
// Process inactive streams
erase_if(streams.inactive, [&](MixableStream& stream) {
if (shouldBeRemoved(stream, _sharedData)) {
return true;
}
// modify by hrtf gain adjustment
auto& hrtf = listenerData->hrtfForStream(nodeID, nodeStream->getStreamIdentifier());
gain *= hrtf.getGainAdjustment();
if (shouldBeSkipped(stream, *listener, *listenerAudioStream, *listenerData)) {
streams.skipped.push_back(move(stream));
++stats.inactiveToSkipped;
return true;
}
auto streamVolume = nodeStream->getLastPopOutputTrailingLoudness() * gain;
nodeVolume = std::max(streamVolume, nodeVolume);
}
if (!shouldBeInactive(stream)) {
streams.active.push_back(move(stream));
++stats.inactiveToActive;
return true;
}
// max-heapify the nodes by relative volume
throttledNodes.push_back({ nodeVolume, node });
std::push_heap(throttledNodes.begin(), throttledNodes.end());
if (!isThrottling) {
updateHRTFParameters(stream, *listenerAudioStream,
listenerData->getMasterAvatarGain());
}
return false;
});
// Process active streams
erase_if(streams.active, [&](MixableStream& stream) {
if (shouldBeRemoved(stream, _sharedData)) {
return true;
}
if (isThrottling) {
// we're throttling, so we need to update the approximate volume for any un-skipped streams
// unless this is simply for an echo (in which case the approx volume is 1.0)
stream.approximateVolume = approximateVolume(stream, listenerAudioStream);
} else {
if (shouldBeSkipped(stream, *listener, *listenerAudioStream, *listenerData)) {
addStream(stream, *listenerAudioStream, 0.0f);
streams.skipped.push_back(move(stream));
++stats.activeToSkipped;
return true;
}
addStream(stream, *listenerAudioStream, listenerData->getMasterAvatarGain());
if (shouldBeInactive(stream)) {
// To reduce artifacts we still call render to flush the HRTF for every silent
// sources on the first frame where the source becomes silent
// this ensures the correct tail from last mixed block
streams.inactive.push_back(move(stream));
++stats.activeToInactive;
return true;
}
}
return false;
});
if (isThrottling) {
// pop the loudest nodes off the heap and mix their streams
int numToRetain = (int)(std::distance(_begin, _end) * (1 - _throttlingRatio));
for (int i = 0; i < numToRetain; i++) {
if (throttledNodes.empty()) {
break;
// since we're throttling, we need to partition the mixable into throttled and unthrottled streams
int numToRetain = min(_numToRetain, (int)streams.active.size()); // Make sure we don't overflow
auto throttlePoint = begin(streams.active) + numToRetain;
std::nth_element(streams.active.begin(), throttlePoint, streams.active.end(),
[](const auto& a, const auto& b)
{
return a.approximateVolume > b.approximateVolume;
});
SegmentedEraseIf<MixableStreamsVector> erase(streams.active);
erase.iterateTo(throttlePoint, [&](MixableStream& stream) {
if (shouldBeSkipped(stream, *listener, *listenerAudioStream, *listenerData)) {
resetHRTFState(stream);
streams.skipped.push_back(move(stream));
++stats.activeToSkipped;
return true;
}
std::pop_heap(throttledNodes.begin(), throttledNodes.end());
addStream(stream, *listenerAudioStream, listenerData->getMasterAvatarGain());
auto& node = throttledNodes.back().second;
AudioMixerClientData* nodeData = static_cast<AudioMixerClientData*>(node->getLinkedData());
forAllStreams(node, nodeData, &AudioMixerSlave::mixStream);
if (shouldBeInactive(stream)) {
// To reduce artifacts we still call render to flush the HRTF for every silent
// sources on the first frame where the source becomes silent
// this ensures the correct tail from last mixed block
streams.inactive.push_back(move(stream));
++stats.activeToInactive;
return true;
}
throttledNodes.pop_back();
}
return false;
});
erase.iterateTo(end(streams.active), [&](MixableStream& stream) {
// To reduce artifacts we reset the HRTF state for every throttled
// sources on the first frame where the source becomes throttled
// this ensures at least remove the tail from last mixed block
// preventing excessive artifacts on the next first block
resetHRTFState(stream);
// throttle the remaining nodes' streams
for (const std::pair<float, SharedNodePointer>& nodePair : throttledNodes) {
auto& node = nodePair.second;
AudioMixerClientData* nodeData = static_cast<AudioMixerClientData*>(node->getLinkedData());
forAllStreams(node, nodeData, &AudioMixerSlave::throttleStream);
}
if (shouldBeSkipped(stream, *listener, *listenerAudioStream, *listenerData)) {
streams.skipped.push_back(move(stream));
++stats.activeToSkipped;
return true;
}
if (shouldBeInactive(stream)) {
streams.inactive.push_back(move(stream));
++stats.activeToInactive;
return true;
}
return false;
});
}
stats.skipped += (int)streams.skipped.size();
stats.inactive += (int)streams.inactive.size();
stats.active += (int)streams.active.size();
// clear the newly ignored, un-ignored, ignoring, and un-ignoring streams now that we've processed them
listenerData->clearStagedIgnoreChanges();
#ifdef HIFI_AUDIO_MIXER_DEBUG
auto mixEnd = p_high_resolution_clock::now();
auto mixTime = std::chrono::duration_cast<std::chrono::nanoseconds>(mixEnd - mixStart);
@ -246,51 +482,35 @@ bool AudioMixerSlave::prepareMix(const SharedNodePointer& listener) {
return hasAudio;
}
void AudioMixerSlave::throttleStream(AudioMixerClientData& listenerNodeData, const QUuid& sourceNodeID,
const AvatarAudioStream& listeningNodeStream, const PositionalAudioStream& streamToAdd) {
// only throttle this stream to the mix if it has a valid position, we won't know how to mix it otherwise
if (streamToAdd.hasValidPosition()) {
addStream(listenerNodeData, sourceNodeID, listeningNodeStream, streamToAdd, true);
}
}
void AudioMixerSlave::mixStream(AudioMixerClientData& listenerNodeData, const QUuid& sourceNodeID,
const AvatarAudioStream& listeningNodeStream, const PositionalAudioStream& streamToAdd) {
// only add the stream to the mix if it has a valid position, we won't know how to mix it otherwise
if (streamToAdd.hasValidPosition()) {
addStream(listenerNodeData, sourceNodeID, listeningNodeStream, streamToAdd, false);
}
}
void AudioMixerSlave::addStream(AudioMixerClientData& listenerNodeData, const QUuid& sourceNodeID,
const AvatarAudioStream& listeningNodeStream, const PositionalAudioStream& streamToAdd,
bool throttle) {
void AudioMixerSlave::addStream(AudioMixerClientData::MixableStream& mixableStream,
AvatarAudioStream& listeningNodeStream,
float masterListenerGain) {
++stats.totalMixes;
// to reduce artifacts we call the HRTF functor for every source, even if throttled or silent
// this ensures the correct tail from last mixed block and the correct spatialization of next first block
auto streamToAdd = mixableStream.positionalStream;
// check if this is a server echo of a source back to itself
bool isEcho = (&streamToAdd == &listeningNodeStream);
bool isEcho = (streamToAdd == &listeningNodeStream);
glm::vec3 relativePosition = streamToAdd.getPosition() - listeningNodeStream.getPosition();
glm::vec3 relativePosition = streamToAdd->getPosition() - listeningNodeStream.getPosition();
float distance = glm::max(glm::length(relativePosition), EPSILON);
float gain = computeGain(listenerNodeData, listeningNodeStream, streamToAdd, relativePosition, distance, isEcho);
float gain = computeGain(masterListenerGain, listeningNodeStream, *streamToAdd, relativePosition, distance, isEcho);
float azimuth = isEcho ? 0.0f : computeAzimuth(listeningNodeStream, listeningNodeStream, relativePosition);
const int HRTF_DATASET_INDEX = 1;
if (!streamToAdd.lastPopSucceeded()) {
if (!streamToAdd->lastPopSucceeded()) {
bool forceSilentBlock = true;
if (!streamToAdd.getLastPopOutput().isNull()) {
bool isInjector = dynamic_cast<const InjectedAudioStream*>(&streamToAdd);
if (!streamToAdd->getLastPopOutput().isNull()) {
bool isInjector = dynamic_cast<const InjectedAudioStream*>(streamToAdd);
// in an injector, just go silent - the injector has likely ended
// in other inputs (microphone, &c.), repeat with fade to avoid the harsh jump to silence
if (!isInjector) {
// calculate its fade factor, which depends on how many times it's already been repeated.
float fadeFactor = calculateRepeatedFrameFadeFactor(streamToAdd.getConsecutiveNotMixedCount() - 1);
float fadeFactor = calculateRepeatedFrameFadeFactor(streamToAdd->getConsecutiveNotMixedCount() - 1);
if (fadeFactor > 0.0f) {
// apply the fadeFactor to the gain
gain *= fadeFactor;
@ -302,15 +522,12 @@ void AudioMixerSlave::addStream(AudioMixerClientData& listenerNodeData, const QU
if (forceSilentBlock) {
// call renderSilent with a forced silent block to reduce artifacts
// (this is not done for stereo streams since they do not go through the HRTF)
if (!streamToAdd.isStereo() && !isEcho) {
// get the existing listener-source HRTF object, or create a new one
auto& hrtf = listenerNodeData.hrtfForStream(sourceNodeID, streamToAdd.getStreamIdentifier());
if (!streamToAdd->isStereo() && !isEcho) {
static int16_t silentMonoBlock[AudioConstants::NETWORK_FRAME_SAMPLES_PER_CHANNEL] = {};
hrtf.renderSilent(silentMonoBlock, _mixSamples, HRTF_DATASET_INDEX, azimuth, distance, gain,
AudioConstants::NETWORK_FRAME_SAMPLES_PER_CHANNEL);
mixableStream.hrtf->render(silentMonoBlock, _mixSamples, HRTF_DATASET_INDEX, azimuth, distance, gain,
AudioConstants::NETWORK_FRAME_SAMPLES_PER_CHANNEL);
++stats.hrtfSilentRenders;
++stats.hrtfRenders;
}
return;
@ -318,16 +535,15 @@ void AudioMixerSlave::addStream(AudioMixerClientData& listenerNodeData, const QU
}
// grab the stream from the ring buffer
AudioRingBuffer::ConstIterator streamPopOutput = streamToAdd.getLastPopOutput();
AudioRingBuffer::ConstIterator streamPopOutput = streamToAdd->getLastPopOutput();
// stereo sources are not passed through HRTF
if (streamToAdd.isStereo()) {
if (streamToAdd->isStereo()) {
// apply the avatar gain adjustment
auto& hrtf = listenerNodeData.hrtfForStream(sourceNodeID, streamToAdd.getStreamIdentifier());
gain *= hrtf.getGainAdjustment();
gain *= mixableStream.hrtf->getGainAdjustment();
const float scale = 1/32768.0f; // int16_t to float
const float scale = 1 / 32768.0f; // int16_t to float
for (int i = 0; i < AudioConstants::NETWORK_FRAME_SAMPLES_PER_CHANNEL; i++) {
_mixSamples[2*i+0] += (float)streamPopOutput[2*i+0] * gain * scale;
@ -335,11 +551,8 @@ void AudioMixerSlave::addStream(AudioMixerClientData& listenerNodeData, const QU
}
++stats.manualStereoMixes;
return;
}
// echo sources are not passed through HRTF
if (isEcho) {
} else if (isEcho) {
// echo sources are not passed through HRTF
const float scale = 1/32768.0f; // int16_t to float
@ -350,41 +563,38 @@ void AudioMixerSlave::addStream(AudioMixerClientData& listenerNodeData, const QU
}
++stats.manualEchoMixes;
return;
} else {
streamPopOutput.readSamples(_bufferSamples, AudioConstants::NETWORK_FRAME_SAMPLES_PER_CHANNEL);
mixableStream.hrtf->render(_bufferSamples, _mixSamples, HRTF_DATASET_INDEX, azimuth, distance, gain,
AudioConstants::NETWORK_FRAME_SAMPLES_PER_CHANNEL);
++stats.hrtfRenders;
}
}
// get the existing listener-source HRTF object, or create a new one
auto& hrtf = listenerNodeData.hrtfForStream(sourceNodeID, streamToAdd.getStreamIdentifier());
void AudioMixerSlave::updateHRTFParameters(AudioMixerClientData::MixableStream& mixableStream,
AvatarAudioStream& listeningNodeStream,
float masterListenerGain) {
auto streamToAdd = mixableStream.positionalStream;
streamPopOutput.readSamples(_bufferSamples, AudioConstants::NETWORK_FRAME_SAMPLES_PER_CHANNEL);
// check if this is a server echo of a source back to itself
bool isEcho = (streamToAdd == &listeningNodeStream);
if (streamToAdd.getLastPopOutputLoudness() == 0.0f) {
// call renderSilent to reduce artifacts
hrtf.renderSilent(_bufferSamples, _mixSamples, HRTF_DATASET_INDEX, azimuth, distance, gain,
AudioConstants::NETWORK_FRAME_SAMPLES_PER_CHANNEL);
glm::vec3 relativePosition = streamToAdd->getPosition() - listeningNodeStream.getPosition();
++stats.hrtfSilentRenders;
return;
}
float distance = glm::max(glm::length(relativePosition), EPSILON);
float gain = computeGain(masterListenerGain, listeningNodeStream, *streamToAdd, relativePosition, distance, isEcho);
float azimuth = isEcho ? 0.0f : computeAzimuth(listeningNodeStream, listeningNodeStream, relativePosition);
if (throttle) {
// call renderSilent with actual frame data and a gain of 0.0f to reduce artifacts
hrtf.renderSilent(_bufferSamples, _mixSamples, HRTF_DATASET_INDEX, azimuth, distance, 0.0f,
AudioConstants::NETWORK_FRAME_SAMPLES_PER_CHANNEL);
mixableStream.hrtf->setParameterHistory(azimuth, distance, gain);
++stats.hrtfThrottleRenders;
return;
}
++stats.hrtfUpdates;
}
if (streamToAdd.getType() == PositionalAudioStream::Injector) {
// apply per-avatar gain to positional audio injectors, which wouldn't otherwise be affected by PAL sliders
hrtf.setGainAdjustment(listenerNodeData.hrtfForStream(sourceNodeID, QUuid()).getGainAdjustment());
}
hrtf.render(_bufferSamples, _mixSamples, HRTF_DATASET_INDEX, azimuth, distance, gain,
AudioConstants::NETWORK_FRAME_SAMPLES_PER_CHANNEL);
++stats.hrtfRenders;
void AudioMixerSlave::resetHRTFState(AudioMixerClientData::MixableStream& mixableStream) {
mixableStream.hrtf->reset();
++stats.hrtfResets;
}
std::unique_ptr<NLPacket> createAudioPacket(PacketType type, int size, quint16 sequence, QString codec) {
@ -443,12 +653,12 @@ void sendEnvironmentPacket(const SharedNodePointer& node, AudioMixerClientData&
glm::vec3 streamPosition = stream->getPosition();
// find reverb properties
for (int i = 0; i < reverbSettings.size(); ++i) {
AABox box = audioZones[reverbSettings[i].zone];
for (const auto& settings : reverbSettings) {
AABox box = audioZones[settings.zone].area;
if (box.contains(streamPosition)) {
hasReverb = true;
reverbTime = reverbSettings[i].reverbTime;
wetLevel = reverbSettings[i].wetLevel;
reverbTime = settings.reverbTime;
wetLevel = settings.wetLevel;
break;
}
}
@ -493,8 +703,7 @@ void sendEnvironmentPacket(const SharedNodePointer& node, AudioMixerClientData&
}
}
float approximateGain(const AvatarAudioStream& listeningNodeStream, const PositionalAudioStream& streamToAdd,
const glm::vec3& relativePosition) {
float approximateGain(const AvatarAudioStream& listeningNodeStream, const PositionalAudioStream& streamToAdd) {
float gain = 1.0f;
// injector: apply attenuation
@ -505,13 +714,14 @@ float approximateGain(const AvatarAudioStream& listeningNodeStream, const Positi
// avatar: skip attenuation - it is too costly to approximate
// distance attenuation: approximate, ignore zone-specific attenuations
glm::vec3 relativePosition = streamToAdd.getPosition() - listeningNodeStream.getPosition();
float distance = glm::length(relativePosition);
return gain / distance;
// avatar: skip master gain - it is constant for all streams
}
float computeGain(const AudioMixerClientData& listenerNodeData, const AvatarAudioStream& listeningNodeStream,
float computeGain(float masterListenerGain, const AvatarAudioStream& listeningNodeStream,
const PositionalAudioStream& streamToAdd, const glm::vec3& relativePosition, float distance, bool isEcho) {
float gain = 1.0f;
@ -534,7 +744,7 @@ float computeGain(const AudioMixerClientData& listenerNodeData, const AvatarAudi
gain *= offAxisCoefficient;
// apply master gain, only to avatars
gain *= listenerNodeData.getMasterAvatarGain();
gain *= masterListenerGain;
}
auto& audioZones = AudioMixer::getAudioZones();
@ -542,10 +752,10 @@ float computeGain(const AudioMixerClientData& listenerNodeData, const AvatarAudi
// find distance attenuation coefficient
float attenuationPerDoublingInDistance = AudioMixer::getAttenuationPerDoublingInDistance();
for (int i = 0; i < zoneSettings.length(); ++i) {
if (audioZones[zoneSettings[i].source].contains(streamToAdd.getPosition()) &&
audioZones[zoneSettings[i].listener].contains(listeningNodeStream.getPosition())) {
attenuationPerDoublingInDistance = zoneSettings[i].coefficient;
for (const auto& settings : zoneSettings) {
if (audioZones[settings.source].area.contains(streamToAdd.getPosition()) &&
audioZones[settings.listener].area.contains(listeningNodeStream.getPosition())) {
attenuationPerDoublingInDistance = settings.coefficient;
break;
}
}

36
assignment-client/src/audio/AudioMixerSlave.h
View file

@ -12,29 +12,39 @@
#ifndef hifi_AudioMixerSlave_h
#define hifi_AudioMixerSlave_h
#include <tbb/concurrent_vector.h>
#include <AABox.h>
#include <AudioHRTF.h>
#include <AudioRingBuffer.h>
#include <ThreadedAssignment.h>
#include <UUIDHasher.h>
#include <NodeList.h>
#include <PositionalAudioStream.h>
#include "AudioMixerClientData.h"
#include "AudioMixerStats.h"
class PositionalAudioStream;
class AvatarAudioStream;
class AudioHRTF;
class AudioMixerClientData;
class AudioMixerSlave {
public:
using ConstIter = NodeList::const_iterator;
struct SharedData {
AudioMixerClientData::ConcurrentAddedStreams addedStreams;
std::vector<Node::LocalID> removedNodes;
std::vector<NodeIDStreamID> removedStreams;
};
AudioMixerSlave(SharedData& sharedData) : _sharedData(sharedData) {};
// process packets for a given node (requires no configuration)
void processPackets(const SharedNodePointer& node);
// configure a round of mixing
void configureMix(ConstIter begin, ConstIter end, unsigned int frame, float throttlingRatio);
void configureMix(ConstIter begin, ConstIter end, unsigned int frame, int numToRetain);
// mix and broadcast non-ignored streams to the node (requires configuration using configureMix, above)
// returns true if a mixed packet was sent to the node
@ -45,13 +55,15 @@ public:
private:
// create mix, returns true if mix has audio
bool prepareMix(const SharedNodePointer& listener);
void throttleStream(AudioMixerClientData& listenerData, const QUuid& streamerID,
const AvatarAudioStream& listenerStream, const PositionalAudioStream& streamer);
void mixStream(AudioMixerClientData& listenerData, const QUuid& streamerID,
const AvatarAudioStream& listenerStream, const PositionalAudioStream& streamer);
void addStream(AudioMixerClientData& listenerData, const QUuid& streamerID,
const AvatarAudioStream& listenerStream, const PositionalAudioStream& streamer,
bool throttle);
void addStream(AudioMixerClientData::MixableStream& mixableStream,
AvatarAudioStream& listeningNodeStream,
float masterListenerGain);
void updateHRTFParameters(AudioMixerClientData::MixableStream& mixableStream,
AvatarAudioStream& listeningNodeStream,
float masterListenerGain);
void resetHRTFState(AudioMixerClientData::MixableStream& mixableStream);
void addStreams(Node& listener, AudioMixerClientData& listenerData);
// mixing buffers
float _mixSamples[AudioConstants::NETWORK_FRAME_SAMPLES_STEREO];
@ -61,7 +73,9 @@ private:
ConstIter _begin;
ConstIter _end;
unsigned int _frame { 0 };
float _throttlingRatio { 0.0f };
int _numToRetain { -1 };
SharedData& _sharedData;
};
#endif // hifi_AudioMixerSlave_h

8
assignment-client/src/audio/AudioMixerSlavePool.cpp
View file

@ -74,13 +74,11 @@ void AudioMixerSlavePool::processPackets(ConstIter begin, ConstIter end) {
run(begin, end);
}
void AudioMixerSlavePool::mix(ConstIter begin, ConstIter end, unsigned int frame, float throttlingRatio) {
void AudioMixerSlavePool::mix(ConstIter begin, ConstIter end, unsigned int frame, int numToRetain) {
_function = &AudioMixerSlave::mix;
_configure = [=](AudioMixerSlave& slave) {
slave.configureMix(_begin, _end, _frame, _throttlingRatio);
slave.configureMix(_begin, _end, frame, numToRetain);
};
_frame = frame;
_throttlingRatio = throttlingRatio;
run(begin, end);
}
@ -167,7 +165,7 @@ void AudioMixerSlavePool::resize(int numThreads) {
if (numThreads > _numThreads) {
// start new slaves
for (int i = 0; i < numThreads - _numThreads; ++i) {
auto slave = new AudioMixerSlaveThread(*this);
auto slave = new AudioMixerSlaveThread(*this, _workerSharedData);
slave->start();
_slaves.emplace_back(slave);
}

12
assignment-client/src/audio/AudioMixerSlavePool.h
View file

@ -31,7 +31,8 @@ class AudioMixerSlaveThread : public QThread, public AudioMixerSlave {
using Lock = std::unique_lock<Mutex>;
public:
AudioMixerSlaveThread(AudioMixerSlavePool& pool) : _pool(pool) {}
AudioMixerSlaveThread(AudioMixerSlavePool& pool, AudioMixerSlave::SharedData& sharedData)
: AudioMixerSlave(sharedData), _pool(pool) {}
void run() override final;
@ -58,14 +59,15 @@ class AudioMixerSlavePool {
public:
using ConstIter = NodeList::const_iterator;
AudioMixerSlavePool(int numThreads = QThread::idealThreadCount()) { setNumThreads(numThreads); }
AudioMixerSlavePool(AudioMixerSlave::SharedData& sharedData, int numThreads = QThread::idealThreadCount())
: _workerSharedData(sharedData) { setNumThreads(numThreads); }
~AudioMixerSlavePool() { resize(0); }
// process packets on slave threads
void processPackets(ConstIter begin, ConstIter end);
// mix on slave threads
void mix(ConstIter begin, ConstIter end, unsigned int frame, float throttlingRatio);
void mix(ConstIter begin, ConstIter end, unsigned int frame, int numToRetain);
// iterate over all slaves
void each(std::function<void(AudioMixerSlave& slave)> functor);
@ -96,10 +98,10 @@ private:
// frame state
Queue _queue;
unsigned int _frame { 0 };
float _throttlingRatio { 0.0f };
ConstIter _begin;
ConstIter _end;
AudioMixerSlave::SharedData& _workerSharedData;
};
#endif // hifi_AudioMixerSlavePool_h

38
assignment-client/src/audio/AudioMixerStats.cpp
View file

@ -15,12 +15,27 @@ void AudioMixerStats::reset() {
sumStreams = 0;
sumListeners = 0;
sumListenersSilent = 0;
totalMixes = 0;
hrtfRenders = 0;
hrtfSilentRenders = 0;
hrtfThrottleRenders = 0;
hrtfResets = 0;
hrtfUpdates = 0;
manualStereoMixes = 0;
manualEchoMixes = 0;
skippedToActive = 0;
skippedToInactive = 0;
inactiveToSkipped = 0;
inactiveToActive = 0;
activeToSkipped = 0;
activeToInactive = 0;
skipped = 0;
inactive = 0;
active = 0;
#ifdef HIFI_AUDIO_MIXER_DEBUG
mixTime = 0;
#endif
@ -30,12 +45,27 @@ void AudioMixerStats::accumulate(const AudioMixerStats& otherStats) {
sumStreams += otherStats.sumStreams;
sumListeners += otherStats.sumListeners;
sumListenersSilent += otherStats.sumListenersSilent;
totalMixes += otherStats.totalMixes;
hrtfRenders += otherStats.hrtfRenders;
hrtfSilentRenders += otherStats.hrtfSilentRenders;
hrtfThrottleRenders += otherStats.hrtfThrottleRenders;
hrtfResets += otherStats.hrtfResets;
hrtfUpdates += otherStats.hrtfUpdates;
manualStereoMixes += otherStats.manualStereoMixes;
manualEchoMixes += otherStats.manualEchoMixes;
skippedToActive += otherStats.skippedToActive;
skippedToInactive += otherStats.skippedToInactive;
inactiveToSkipped += otherStats.inactiveToSkipped;
inactiveToActive += otherStats.inactiveToActive;
activeToSkipped += otherStats.activeToSkipped;
activeToInactive += otherStats.activeToInactive;
skipped += otherStats.skipped;
inactive += otherStats.inactive;
active += otherStats.active;
#ifdef HIFI_AUDIO_MIXER_DEBUG
mixTime += otherStats.mixTime;
#endif

15
assignment-client/src/audio/AudioMixerStats.h
View file

@ -24,12 +24,23 @@ struct AudioMixerStats {
int totalMixes { 0 };
int hrtfRenders { 0 };
int hrtfSilentRenders { 0 };
int hrtfThrottleRenders { 0 };
int hrtfResets { 0 };
int hrtfUpdates { 0 };
int manualStereoMixes { 0 };
int manualEchoMixes { 0 };
int skippedToActive { 0 };
int skippedToInactive { 0 };
int inactiveToSkipped { 0 };
int inactiveToActive { 0 };
int activeToSkipped { 0 };
int activeToInactive { 0 };
int skipped { 0 };
int inactive { 0 };
int active { 0 };
#ifdef HIFI_AUDIO_MIXER_DEBUG
uint64_t mixTime { 0 };
#endif

10
assignment-client/src/audio/AvatarAudioStream.cpp
View file

@ -23,9 +23,9 @@ int AvatarAudioStream::parseStreamProperties(PacketType type, const QByteArray&
if (type == PacketType::SilentAudioFrame) {
const char* dataAt = packetAfterSeqNum.constData();
quint16 numSilentSamples = *(reinterpret_cast<const quint16*>(dataAt));
readBytes += sizeof(quint16);
numAudioSamples = (int)numSilentSamples;
SilentSamplesBytes numSilentSamples = *(reinterpret_cast<const quint16*>(dataAt));
readBytes += sizeof(SilentSamplesBytes);
numAudioSamples = (int) numSilentSamples;
// read the positional data
readBytes += parsePositionalData(packetAfterSeqNum.mid(readBytes));
@ -34,9 +34,9 @@ int AvatarAudioStream::parseStreamProperties(PacketType type, const QByteArray&
_shouldLoopbackForNode = (type == PacketType::MicrophoneAudioWithEcho);
// read the channel flag
quint8 channelFlag = packetAfterSeqNum.at(readBytes);
ChannelFlag channelFlag = packetAfterSeqNum.at(readBytes);
bool isStereo = channelFlag == 1;
readBytes += sizeof(quint8);
readBytes += sizeof(ChannelFlag);
// if isStereo value has changed, restart the ring buffer with new frame size
if (isStereo != _isStereo) {

2
assignment-client/src/audio/AvatarAudioStream.h
View file

@ -16,6 +16,8 @@
#include "PositionalAudioStream.h"
using SilentSamplesBytes = quint16;
class AvatarAudioStream : public PositionalAudioStream {
public:
AvatarAudioStream(bool isStereo, int numStaticJitterFrames = -1);

8
assignment-client/src/avatars/AvatarMixer.cpp
View file

@ -673,7 +673,13 @@ void AvatarMixer::handleNodeIgnoreRequestPacket(QSharedPointer<ReceivedMessage>
void AvatarMixer::handleRadiusIgnoreRequestPacket(QSharedPointer<ReceivedMessage> packet, SharedNodePointer sendingNode) {
auto start = usecTimestampNow();
sendingNode->parseIgnoreRadiusRequestMessage(packet);
bool enabled;
packet->readPrimitive(&enabled);
auto avatarData = getOrCreateClientData(sendingNode);
avatarData->setIsIgnoreRadiusEnabled(enabled);
auto end = usecTimestampNow();
_handleRadiusIgnoreRequestPacketElapsedTime += (end - start);
}

2
assignment-client/src/avatars/AvatarMixerClientData.cpp
View file

@ -227,7 +227,7 @@ void AvatarMixerClientData::ignoreOther(const Node* self, const Node* other) {
addToRadiusIgnoringSet(other->getUUID());
auto killPacket = NLPacket::create(PacketType::KillAvatar, NUM_BYTES_RFC4122_UUID + sizeof(KillAvatarReason), true);
killPacket->write(other->getUUID().toRfc4122());
if (self->isIgnoreRadiusEnabled()) {
if (_isIgnoreRadiusEnabled) {
killPacket->writePrimitive(KillAvatarReason::TheirAvatarEnteredYourBubble);
} else {
killPacket->writePrimitive(KillAvatarReason::YourAvatarEnteredTheirBubble);

5
assignment-client/src/avatars/AvatarMixerClientData.h
View file

@ -49,6 +49,9 @@ public:
const AvatarData* getConstAvatarData() const { return _avatar.get(); }
AvatarSharedPointer getAvatarSharedPointer() const { return _avatar; }
bool isIgnoreRadiusEnabled() const { return _isIgnoreRadiusEnabled; }
void setIsIgnoreRadiusEnabled(bool enabled) { _isIgnoreRadiusEnabled = enabled; }
uint16_t getLastBroadcastSequenceNumber(const QUuid& nodeUUID) const;
void setLastBroadcastSequenceNumber(const QUuid& nodeUUID, uint16_t sequenceNumber)
{ _lastBroadcastSequenceNumbers[nodeUUID] = sequenceNumber; }
@ -180,6 +183,8 @@ private:
std::unordered_map<Node::LocalID, TraitsCheckTimestamp> _lastSentTraitsTimestamps;
std::unordered_map<Node::LocalID, AvatarTraits::TraitVersions> _sentTraitVersions;
std::atomic_bool _isIgnoreRadiusEnabled { false };
};
#endif // hifi_AvatarMixerClientData_h

2
assignment-client/src/avatars/AvatarMixerSlave.cpp
View file

@ -345,7 +345,7 @@ void AvatarMixerSlave::broadcastAvatarDataToAgent(const SharedNodePointer& node)
} else {
// Check to see if the space bubble is enabled
// Don't bother with these checks if the other avatar has their bubble enabled and we're gettingAnyIgnored
if (destinationNode->isIgnoreRadiusEnabled() || (avatarNode->isIgnoreRadiusEnabled() && !getsAnyIgnored)) {
if (nodeData->isIgnoreRadiusEnabled() || (avatarClientNodeData->isIgnoreRadiusEnabled() && !getsAnyIgnored)) {
// Perform the collision check between the two bounding boxes
const float OTHER_AVATAR_BUBBLE_EXPANSION_FACTOR = 2.4f; // magic number determined empirically
AABox otherNodeBox = computeBubbleBox(avatarClientNodeData->getAvatar(), OTHER_AVATAR_BUBBLE_EXPANSION_FACTOR);

17
libraries/audio/src/AudioHRTF.cpp
View file

@ -1173,20 +1173,5 @@ void AudioHRTF::render(int16_t* input, float* output, int index, float azimuth,
// crossfade old/new output and accumulate
crossfade_4x2(bqBuffer, output, crossfadeTable, HRTF_BLOCK);
_silentState = false;
}
void AudioHRTF::renderSilent(int16_t* input, float* output, int index, float azimuth, float distance, float gain, int numFrames) {
// process the first silent block, to flush internal state
if (!_silentState) {
render(input, output, index, azimuth, distance, gain, numFrames);
}
// new parameters become old
_azimuthState = azimuth;
_distanceState = distance;
_gainState = gain;
_silentState = true;
_resetState = false;
}

37
libraries/audio/src/AudioHRTF.h
View file

@ -47,9 +47,14 @@ public:
void render(int16_t* input, float* output, int index, float azimuth, float distance, float gain, int numFrames);
//
// Fast path when input is known to be silent
// Fast path when input is known to be silent and state as been flushed
//
void renderSilent(int16_t* input, float* output, int index, float azimuth, float distance, float gain, int numFrames);
void setParameterHistory(float azimuth, float distance, float gain) {
// new parameters become old
_azimuthState = azimuth;
_distanceState = distance;
_gainState = gain;
}
//
// HRTF local gain adjustment in amplitude (1.0 == unity)
@ -59,23 +64,25 @@ public:
// clear internal state, but retain settings
void reset() {
// FIR history
memset(_firState, 0, sizeof(_firState));
if (!_resetState) {
// FIR history
memset(_firState, 0, sizeof(_firState));
// integer delay history
memset(_delayState, 0, sizeof(_delayState));
// integer delay history
memset(_delayState, 0, sizeof(_delayState));
// biquad history
memset(_bqState, 0, sizeof(_bqState));
// biquad history
memset(_bqState, 0, sizeof(_bqState));
// parameter history
_azimuthState = 0.0f;
_distanceState = 0.0f;
_gainState = 0.0f;
// parameter history
_azimuthState = 0.0f;
_distanceState = 0.0f;
_gainState = 0.0f;
// _gainAdjust is retained
// _gainAdjust is retained
_silentState = true;
_resetState = true;
}
}
private:
@ -110,7 +117,7 @@ private:
// global and local gain adjustment
float _gainAdjust = HRTF_GAIN;
bool _silentState = true;
bool _resetState = true;
};
#endif // AudioHRTF_h

2
libraries/audio/src/AudioStreamStats.h
View file

@ -66,4 +66,6 @@ public:
PacketStreamStats _packetStreamWindowStats;
};
static_assert(sizeof(AudioStreamStats) == 152, "AudioStreamStats size isn't right");
#endif // hifi_AudioStreamStats_h

9
libraries/audio/src/InboundAudioStream.cpp
View file

@ -171,7 +171,6 @@ int InboundAudioStream::parseData(ReceivedMessage& message) {
} else {
_mismatchedAudioCodecCount++;
qDebug(audio) << "Codec mismatch: expected" << _selectedCodecName << "got" << codecInPacket;
if (packetPCM) {
// If there are PCM packets in-flight after the codec is changed, use them.
@ -191,7 +190,8 @@ int InboundAudioStream::parseData(ReceivedMessage& message) {
auto sendingNode = DependencyManager::get<NodeList>()->nodeWithLocalID(message.getSourceID());
if (sendingNode) {
emit mismatchedAudioCodec(sendingNode, _selectedCodecName, codecInPacket);
qDebug(audio) << "Codec mismatch threshold exceeded, SelectedAudioFormat(" << _selectedCodecName << " ) sent";
qDebug(audio) << "Codec mismatch threshold exceeded, sent selected codec"
<< _selectedCodecName << "to" << message.getSenderSockAddr();
}
}
}
@ -208,7 +208,6 @@ int InboundAudioStream::parseData(ReceivedMessage& message) {
int framesAvailable = _ringBuffer.framesAvailable();
// if this stream was starved, check if we're still starved.
if (_isStarved && framesAvailable >= _desiredJitterBufferFrames) {
qCInfo(audiostream, "Starve ended");
_isStarved = false;
}
// if the ringbuffer exceeds the desired size by more than the threshold specified,
@ -378,10 +377,6 @@ void InboundAudioStream::framesAvailableChanged() {
}
void InboundAudioStream::setToStarved() {
if (!_isStarved) {
qCInfo(audiostream, "Starved");
}
_consecutiveNotMixedCount = 0;
_starveCount++;
// if we have more than the desired frames when setToStarved() is called, then we'll immediately

2
libraries/audio/src/InboundAudioStream.h
View file

@ -30,6 +30,8 @@
// Audio Env bitset
const int HAS_REVERB_BIT = 0; // 1st bit
using StreamSequenceNumber = quint16;
class InboundAudioStream : public NodeData {
Q_OBJECT

2
libraries/audio/src/InjectedAudioStream.cpp
View file

@ -50,7 +50,7 @@ int InjectedAudioStream::parseStreamProperties(PacketType type,
}
// pull the loopback flag and set our boolean
uchar shouldLoopback;
LoopbackFlag shouldLoopback;
packetStream >> shouldLoopback;
_shouldLoopbackForNode = (shouldLoopback == 1);

2
libraries/audio/src/InjectedAudioStream.h
View file

@ -16,6 +16,8 @@
#include "PositionalAudioStream.h"
using LoopbackFlag = uchar;
class InjectedAudioStream : public PositionalAudioStream {
public:
InjectedAudioStream(const QUuid& streamIdentifier, bool isStereo, int numStaticJitterFrames = -1);

42
libraries/audio/src/PositionalAudioStream.cpp
View file

@ -14,7 +14,6 @@
#include <cstring>
#include <glm/detail/func_common.hpp>
#include <QtCore/QDataStream>
#include <QtCore/QLoggingCategory>
@ -78,20 +77,15 @@ int PositionalAudioStream::parsePositionalData(const QByteArray& positionalByteA
QDataStream packetStream(positionalByteArray);
packetStream.readRawData(reinterpret_cast<char*>(&_position), sizeof(_position));
// if the client sends us a bad position, flag it so that we don't consider this stream for mixing
if (glm::isnan(_position.x) || glm::isnan(_position.y) || glm::isnan(_position.z)) {
HIFI_FDEBUG("PositionalAudioStream unpacked invalid position for node" << uuidStringWithoutCurlyBraces(getNodeID()) );
_hasValidPosition = false;
} else {
_hasValidPosition = true;
}
packetStream.readRawData(reinterpret_cast<char*>(&_orientation), sizeof(_orientation));
packetStream.readRawData(reinterpret_cast<char*>(&_avatarBoundingBoxCorner), sizeof(_avatarBoundingBoxCorner));
packetStream.readRawData(reinterpret_cast<char*>(&_avatarBoundingBoxScale), sizeof(_avatarBoundingBoxScale));
if (_avatarBoundingBoxCorner != _ignoreBox.getCorner()) {
// if the ignore box corner changes, we need to re-calculate the ignore box
calculateIgnoreBox();
}
// if this node sent us a NaN for first float in orientation then don't consider this good audio and bail
if (glm::isnan(_orientation.x)) {
// NOTE: why would we reset the ring buffer here?
@ -107,3 +101,29 @@ AudioStreamStats PositionalAudioStream::getAudioStreamStats() const {
streamStats._streamType = _type;
return streamStats;
}
void PositionalAudioStream::calculateIgnoreBox() {
if (_avatarBoundingBoxScale != glm::vec3(0)) {
auto scale = _avatarBoundingBoxScale;
// enforce a minimum scale
static const glm::vec3 MIN_IGNORE_BOX_SCALE = glm::vec3(0.3f, 1.3f, 0.3f);
if (glm::any(glm::lessThan(scale, MIN_IGNORE_BOX_SCALE))) {
scale = MIN_IGNORE_BOX_SCALE;
}
// (this is arbitrary number determined empirically for comfort)
const float IGNORE_BOX_SCALE_FACTOR = 2.4f;
scale *= IGNORE_BOX_SCALE_FACTOR;
// create the box (we use a box for the zone for convenience)
_ignoreBox.setBox(_avatarBoundingBoxCorner, scale);
}
}
void PositionalAudioStream::enableIgnoreBox() {
// re-calculate the ignore box using the latest values
calculateIgnoreBox();
_isIgnoreBoxEnabled = true;
}

35
libraries/audio/src/PositionalAudioStream.h
View file

@ -19,6 +19,24 @@
const int AUDIOMIXER_INBOUND_RING_BUFFER_FRAME_CAPACITY = 100;
using StreamID = QUuid;
const int NUM_STREAM_ID_BYTES = NUM_BYTES_RFC4122_UUID;
struct NodeIDStreamID {
QUuid nodeID;
Node::LocalID nodeLocalID;
StreamID streamID;
NodeIDStreamID(QUuid nodeID, Node::LocalID nodeLocalID, StreamID streamID)
: nodeID(nodeID), nodeLocalID(nodeLocalID), streamID(streamID) {};
bool operator==(const NodeIDStreamID& other) const {
return (nodeLocalID == other.nodeLocalID || nodeID == other.nodeID) && streamID == other.streamID;
}
};
using ChannelFlag = quint8;
class PositionalAudioStream : public InboundAudioStream {
Q_OBJECT
public:
@ -30,7 +48,7 @@ public:
PositionalAudioStream(PositionalAudioStream::Type type, bool isStereo, int numStaticJitterFrames = -1);
const QUuid DEFAULT_STREAM_IDENTIFIER = QUuid();
virtual const QUuid& getStreamIdentifier() const { return DEFAULT_STREAM_IDENTIFIER; }
virtual const StreamID& getStreamIdentifier() const { return DEFAULT_STREAM_IDENTIFIER; }
virtual void resetStats() override;
@ -51,7 +69,15 @@ public:
const glm::vec3& getAvatarBoundingBoxCorner() const { return _avatarBoundingBoxCorner; }
const glm::vec3& getAvatarBoundingBoxScale() const { return _avatarBoundingBoxScale; }
bool hasValidPosition() const { return _hasValidPosition; }
using IgnoreBox = AABox;
// called from single AudioMixerSlave while processing packets for node
void enableIgnoreBox();
void disableIgnoreBox() { _isIgnoreBoxEnabled = false; }
// thread-safe, called from AudioMixerSlave(s) while preparing mixes
bool isIgnoreBoxEnabled() const { return _isIgnoreBoxEnabled; }
const IgnoreBox& getIgnoreBox() const { return _ignoreBox; }
protected:
// disallow copying of PositionalAudioStream objects
@ -61,6 +87,8 @@ protected:
int parsePositionalData(const QByteArray& positionalByteArray);
protected:
void calculateIgnoreBox();
Type _type;
glm::vec3 _position;
glm::quat _orientation;
@ -79,7 +107,8 @@ protected:
float _quietestFrameLoudness;
int _frameCounter;
bool _hasValidPosition { false };
bool _isIgnoreBoxEnabled { false };
IgnoreBox _ignoreBox;
};
#endif // hifi_PositionalAudioStream_h

37
libraries/networking/src/Node.cpp
View file

@ -96,7 +96,6 @@ Node::Node(const QUuid& uuid, NodeType_t type, const HifiSockAddr& publicSocket,
{
// Update socket's object name
setType(_type);
_ignoreRadiusEnabled = false;
}
void Node::setType(char type) {
@ -114,9 +113,12 @@ void Node::updateClockSkewUsec(qint64 clockSkewSample) {
_clockSkewUsec = (quint64)_clockSkewMovingPercentile.getValueAtPercentile();
}
void Node::parseIgnoreRequestMessage(QSharedPointer<ReceivedMessage> message) {
Node::NodesIgnoredPair Node::parseIgnoreRequestMessage(QSharedPointer<ReceivedMessage> message) {
bool addToIgnore;
message->readPrimitive(&addToIgnore);
std::vector<QUuid> nodesIgnored;
while (message->getBytesLeftToRead()) {
// parse out the UUID being ignored from the packet
QUuid ignoredUUID = QUuid::fromRfc4122(message->readWithoutCopy(NUM_BYTES_RFC4122_UUID));
@ -126,17 +128,23 @@ void Node::parseIgnoreRequestMessage(QSharedPointer<ReceivedMessage> message) {
} else {
removeIgnoredNode(ignoredUUID);
}
nodesIgnored.push_back(ignoredUUID);
}
return { nodesIgnored, addToIgnore };
}
void Node::addIgnoredNode(const QUuid& otherNodeID) {
if (!otherNodeID.isNull() && otherNodeID != _uuid) {
QReadLocker lock { &_ignoredNodeIDSetLock };
QWriteLocker lock { &_ignoredNodeIDSetLock };
qCDebug(networking) << "Adding" << uuidStringWithoutCurlyBraces(otherNodeID) << "to ignore set for"
<< uuidStringWithoutCurlyBraces(_uuid);
<< uuidStringWithoutCurlyBraces(_uuid);
// add the session UUID to the set of ignored ones for this listening node
_ignoredNodeIDSet.insert(otherNodeID);
if (std::find(_ignoredNodeIDs.begin(), _ignoredNodeIDs.end(), otherNodeID) == _ignoredNodeIDs.end()) {
_ignoredNodeIDs.push_back(otherNodeID);
}
} else {
qCWarning(networking) << "Node::addIgnoredNode called with null ID or ID of ignoring node.";
}
@ -144,22 +152,25 @@ void Node::addIgnoredNode(const QUuid& otherNodeID) {
void Node::removeIgnoredNode(const QUuid& otherNodeID) {
if (!otherNodeID.isNull() && otherNodeID != _uuid) {
// insert/find are read locked concurrently. unsafe_erase is not concurrent, and needs a write lock.
QWriteLocker lock { &_ignoredNodeIDSetLock };
qCDebug(networking) << "Removing" << uuidStringWithoutCurlyBraces(otherNodeID) << "from ignore set for"
<< uuidStringWithoutCurlyBraces(_uuid);
<< uuidStringWithoutCurlyBraces(_uuid);
// remove the session UUID from the set of ignored ones for this listening node
_ignoredNodeIDSet.unsafe_erase(otherNodeID);
// remove the session UUID from the set of ignored ones for this listening node, if it exists
auto it = std::remove(_ignoredNodeIDs.begin(), _ignoredNodeIDs.end(), otherNodeID);
if (it != _ignoredNodeIDs.end()) {
_ignoredNodeIDs.erase(it);
}
} else {
qCWarning(networking) << "Node::removeIgnoredNode called with null ID or ID of ignoring node.";
}
}
void Node::parseIgnoreRadiusRequestMessage(QSharedPointer<ReceivedMessage> message) {
bool enabled;
message->readPrimitive(&enabled);
_ignoreRadiusEnabled = enabled;
bool Node::isIgnoringNodeWithID(const QUuid& nodeID) const {
QReadLocker lock { &_ignoredNodeIDSetLock };
// check if this node ID is present in the ignore node ID set
return std::find(_ignoredNodeIDs.begin(), _ignoredNodeIDs.end(), nodeID) != _ignoredNodeIDs.end();
}
QDataStream& operator<<(QDataStream& out, const Node& node) {

18
libraries/networking/src/Node.h
View file

@ -15,6 +15,7 @@
#include <memory>
#include <ostream>
#include <stdint.h>
#include <vector>
#include <QtCore/QDebug>
#include <QtCore/QMutex>
@ -80,17 +81,19 @@ public:
bool getCanKick() const { return _permissions.can(NodePermissions::Permission::canKick); }
bool getCanReplaceContent() const { return _permissions.can(NodePermissions::Permission::canReplaceDomainContent); }
void parseIgnoreRequestMessage(QSharedPointer<ReceivedMessage> message);
using NodesIgnoredPair = std::pair<std::vector<QUuid>, bool>;
NodesIgnoredPair parseIgnoreRequestMessage(QSharedPointer<ReceivedMessage> message);
void addIgnoredNode(const QUuid& otherNodeID);
void removeIgnoredNode(const QUuid& otherNodeID);
bool isIgnoringNodeWithID(const QUuid& nodeID) const { QReadLocker lock { &_ignoredNodeIDSetLock }; return _ignoredNodeIDSet.find(nodeID) != _ignoredNodeIDSet.cend(); }
void parseIgnoreRadiusRequestMessage(QSharedPointer<ReceivedMessage> message);
bool isIgnoringNodeWithID(const QUuid& nodeID) const;
using IgnoredNodeIDs = std::vector<QUuid>;
const IgnoredNodeIDs& getIgnoredNodeIDs() const { return _ignoredNodeIDs; }
friend QDataStream& operator<<(QDataStream& out, const Node& node);
friend QDataStream& operator>>(QDataStream& in, Node& node);
bool isIgnoreRadiusEnabled() const { return _ignoreRadiusEnabled; }
private:
// privatize copy and assignment operator to disallow Node copying
Node(const Node &otherNode);
@ -108,11 +111,10 @@ private:
MovingPercentile _clockSkewMovingPercentile;
NodePermissions _permissions;
bool _isUpstream { false };
tbb::concurrent_unordered_set<QUuid, UUIDHasher> _ignoredNodeIDSet;
IgnoredNodeIDs _ignoredNodeIDs;
mutable QReadWriteLock _ignoredNodeIDSetLock;
std::vector<QString> _replicatedUsernames { };
std::atomic_bool _ignoreRadiusEnabled;
};
Q_DECLARE_METATYPE(Node*)