From 6f7162132edea0908c2a892ee9aa5f5dc373566b Mon Sep 17 00:00:00 2001 From: Zach Pomerantz Date: Tue, 22 Nov 2016 11:37:39 -0500 Subject: [PATCH 01/48] add release/debug/gprof/valgrind to gitignore --- .gitignore | 4 ++++ 1 file changed, 4 insertions(+) diff --git a/.gitignore b/.gitignore index cd7fb34eaf..d6227f1f30 100644 --- a/.gitignore +++ b/.gitignore @@ -4,6 +4,10 @@ CMakeFiles/ CMakeScripts/ cmake_install.cmake build*/ +release*/ +debug*/ +gprof*/ +valgrind*/ ext/ Makefile *.user From 0170379d3ce0afe2dfe15c0941939932753a5a17 Mon Sep 17 00:00:00 2001 From: Zach Pomerantz Date: Mon, 21 Nov 2016 16:22:34 -0600 Subject: [PATCH 02/48] rm unused vars --- assignment-client/src/audio/AudioMixer.cpp | 18 +++++------------- assignment-client/src/audio/AudioMixer.h | 8 +------- 2 files changed, 6 insertions(+), 20 deletions(-) diff --git a/assignment-client/src/audio/AudioMixer.cpp b/assignment-client/src/audio/AudioMixer.cpp index 3dba1ce1c2..6c2cd25ba3 100644 --- a/assignment-client/src/audio/AudioMixer.cpp +++ b/assignment-client/src/audio/AudioMixer.cpp @@ -69,7 +69,6 @@ static const QString AUDIO_ENV_GROUP_KEY = "audio_env"; static const QString AUDIO_BUFFER_GROUP_KEY = "audio_buffer"; int AudioMixer::_numStaticJitterFrames{ -1 }; -bool AudioMixer::_enableFilter = true; bool AudioMixer::shouldMute(float quietestFrame) { return (quietestFrame > _noiseMutingThreshold); @@ -96,6 +95,7 @@ AudioMixer::AudioMixer(ReceivedMessage& message) : packetReceiver.registerListener(PacketType::KillAvatar, this, "handleKillAvatarPacket"); packetReceiver.registerListener(PacketType::NodeMuteRequest, this, "handleNodeMuteRequestPacket"); packetReceiver.registerListener(PacketType::RadiusIgnoreRequest, this, "handleRadiusIgnoreRequestPacket"); + connect(nodeList.data(), &NodeList::nodeKilled, this, &AudioMixer::handleNodeKilled); } @@ -848,7 +848,7 @@ void AudioMixer::broadcastMixes() { // if the stream should be muted, send mute packet if (nodeData->getAvatarAudioStream() - && (shouldMute(nodeData->getAvatarAudioStream()->getQuietestFrameLoudness()) + && (shouldMute(nodeData->getAvatarAudioStream()->getQuietestFrameLoudness()) || nodeData->shouldMuteClient())) { auto mutePacket = NLPacket::create(PacketType::NoisyMute, 0); nodeList->sendPacket(std::move(mutePacket), *node); @@ -865,8 +865,8 @@ void AudioMixer::broadcastMixes() { std::unique_ptr mixPacket; if (mixHasAudio || nodeData->shouldFlushEncoder()) { - - int mixPacketBytes = sizeof(quint16) + AudioConstants::MAX_CODEC_NAME_LENGTH_ON_WIRE + + int mixPacketBytes = sizeof(quint16) + AudioConstants::MAX_CODEC_NAME_LENGTH_ON_WIRE + AudioConstants::NETWORK_FRAME_BYTES_STEREO; mixPacket = NLPacket::create(PacketType::MixedAudio, mixPacketBytes); @@ -888,7 +888,7 @@ void AudioMixer::broadcastMixes() { } // pack mixed audio samples mixPacket->write(encodedBuffer.constData(), encodedBuffer.size()); - + } else { int silentPacketBytes = sizeof(quint16) + sizeof(quint16) + AudioConstants::MAX_CODEC_NAME_LENGTH_ON_WIRE; mixPacket = NLPacket::create(PacketType::SilentAudioFrame, silentPacketBytes); @@ -1051,14 +1051,6 @@ void AudioMixer::parseSettingsObject(const QJsonObject &settingsObject) { } } - const QString FILTER_KEY = "enable_filter"; - if (audioEnvGroupObject[FILTER_KEY].isBool()) { - _enableFilter = audioEnvGroupObject[FILTER_KEY].toBool(); - } - if (_enableFilter) { - qDebug() << "Filter enabled"; - } - const QString AUDIO_ZONES = "zones"; if (audioEnvGroupObject[AUDIO_ZONES].isObject()) { const QJsonObject& zones = audioEnvGroupObject[AUDIO_ZONES].toObject(); diff --git a/assignment-client/src/audio/AudioMixer.h b/assignment-client/src/audio/AudioMixer.h index 9bf337fe60..319ef47950 100644 --- a/assignment-client/src/audio/AudioMixer.h +++ b/assignment-client/src/audio/AudioMixer.h @@ -23,10 +23,6 @@ class AvatarAudioStream; class AudioHRTF; class AudioMixerClientData; -const int SAMPLE_PHASE_DELAY_AT_90 = 20; - -const int READ_DATAGRAMS_STATS_WINDOW_SECONDS = 30; - /// Handles assignments of type AudioMixer - mixing streams of audio and re-distributing to various clients. class AudioMixer : public ThreadedAssignment { Q_OBJECT @@ -57,7 +53,7 @@ private slots: private: AudioMixerClientData* getOrCreateClientData(Node* node); void domainSettingsRequestComplete(); - + /// adds one stream to the mix for a listening node void addStreamToMixForListeningNodeWithStream(AudioMixerClientData& listenerNodeData, const PositionalAudioStream& streamToAdd, @@ -118,8 +114,6 @@ private: QVector _zoneReverbSettings; static int _numStaticJitterFrames; // -1 denotes dynamic jitter buffering - - static bool _enableFilter; }; #endif // hifi_AudioMixer_h From 028ac5264f86c97df5410c88068e0e0dbef36d4a Mon Sep 17 00:00:00 2001 From: Zach Pomerantz Date: Mon, 28 Nov 2016 16:26:53 -0500 Subject: [PATCH 03/48] consolidate AudioMixer slots --- assignment-client/src/audio/AudioMixer.cpp | 11 ++--------- assignment-client/src/audio/AudioMixer.h | 3 +-- 2 files changed, 3 insertions(+), 11 deletions(-) diff --git a/assignment-client/src/audio/AudioMixer.cpp b/assignment-client/src/audio/AudioMixer.cpp index 6c2cd25ba3..fff5d1d698 100644 --- a/assignment-client/src/audio/AudioMixer.cpp +++ b/assignment-client/src/audio/AudioMixer.cpp @@ -744,7 +744,7 @@ void AudioMixer::run() { // wait until we have the domain-server settings, otherwise we bail DomainHandler& domainHandler = DependencyManager::get()->getDomainHandler(); - connect(&domainHandler, &DomainHandler::settingsReceived, this, &AudioMixer::domainSettingsRequestComplete); + connect(&domainHandler, &DomainHandler::settingsReceived, this, &AudioMixer::start); connect(&domainHandler, &DomainHandler::settingsReceiveFail, this, &AudioMixer::domainSettingsRequestFailed); ThreadedAssignment::commonInit(AUDIO_MIXER_LOGGING_TARGET_NAME, NodeType::AudioMixer); @@ -762,7 +762,7 @@ AudioMixerClientData* AudioMixer::getOrCreateClientData(Node* node) { return clientData; } -void AudioMixer::domainSettingsRequestComplete() { +void AudioMixer::start() { auto nodeList = DependencyManager::get(); nodeList->addNodeTypeToInterestSet(NodeType::Agent); @@ -775,11 +775,6 @@ void AudioMixer::domainSettingsRequestComplete() { // check the settings object to see if we have anything we can parse out parseSettingsObject(settingsObject); - // queue up a connection to start broadcasting mixes now that we're ready to go - QMetaObject::invokeMethod(this, "broadcastMixes", Qt::QueuedConnection); -} - -void AudioMixer::broadcastMixes() { const int TRAILING_AVERAGE_FRAMES = 100; const float CURRENT_FRAME_RATIO = 1.0f / TRAILING_AVERAGE_FRAMES; const float PREVIOUS_FRAMES_RATIO = 1.0f - CURRENT_FRAME_RATIO; @@ -789,8 +784,6 @@ void AudioMixer::broadcastMixes() { const float RATIO_BACK_OFF = 0.02f; - auto nodeList = DependencyManager::get(); - auto nextFrameTimestamp = p_high_resolution_clock::now(); auto timeToSleep = std::chrono::microseconds(0); diff --git a/assignment-client/src/audio/AudioMixer.h b/assignment-client/src/audio/AudioMixer.h index 319ef47950..6bc035e9b4 100644 --- a/assignment-client/src/audio/AudioMixer.h +++ b/assignment-client/src/audio/AudioMixer.h @@ -38,7 +38,6 @@ public slots: static int getStaticJitterFrames() { return _numStaticJitterFrames; } private slots: - void broadcastMixes(); void handleNodeAudioPacket(QSharedPointer packet, SharedNodePointer sendingNode); void handleMuteEnvironmentPacket(QSharedPointer packet, SharedNodePointer sendingNode); void handleNegotiateAudioFormat(QSharedPointer message, SharedNodePointer sendingNode); @@ -48,11 +47,11 @@ private slots: void handleKillAvatarPacket(QSharedPointer packet, SharedNodePointer sendingNode); void handleNodeMuteRequestPacket(QSharedPointer packet, SharedNodePointer sendingNode); + void start(); void removeHRTFsForFinishedInjector(const QUuid& streamID); private: AudioMixerClientData* getOrCreateClientData(Node* node); - void domainSettingsRequestComplete(); /// adds one stream to the mix for a listening node void addStreamToMixForListeningNodeWithStream(AudioMixerClientData& listenerNodeData, From 891084e9db05216c48841f3c27592dbb97b6a0e6 Mon Sep 17 00:00:00 2001 From: Zach Pomerantz Date: Tue, 22 Nov 2016 12:51:34 -0500 Subject: [PATCH 04/48] modularize audio mixing code --- assignment-client/src/audio/AudioMixer.cpp | 335 +++++++++++---------- assignment-client/src/audio/AudioMixer.h | 13 +- 2 files changed, 191 insertions(+), 157 deletions(-) diff --git a/assignment-client/src/audio/AudioMixer.cpp b/assignment-client/src/audio/AudioMixer.cpp index fff5d1d698..7d1f4aba7c 100644 --- a/assignment-client/src/audio/AudioMixer.cpp +++ b/assignment-client/src/audio/AudioMixer.cpp @@ -765,189 +765,214 @@ AudioMixerClientData* AudioMixer::getOrCreateClientData(Node* node) { void AudioMixer::start() { auto nodeList = DependencyManager::get(); + // prepare the NodeList nodeList->addNodeTypeToInterestSet(NodeType::Agent); - nodeList->linkedDataCreateCallback = [&](Node* node) { getOrCreateClientData(node); }; - DomainHandler& domainHandler = nodeList->getDomainHandler(); - const QJsonObject& settingsObject = domainHandler.getSettingsObject(); + // parse out any AudioMixer settings + { + DomainHandler& domainHandler = nodeList->getDomainHandler(); + const QJsonObject& settingsObject = domainHandler.getSettingsObject(); + parseSettingsObject(settingsObject); + } - // check the settings object to see if we have anything we can parse out - parseSettingsObject(settingsObject); + // manageLoad state + auto frameTimestamp = p_high_resolution_clock::time_point::min(); + unsigned int framesSinceManagement = std::numeric_limits::max(); - const int TRAILING_AVERAGE_FRAMES = 100; - const float CURRENT_FRAME_RATIO = 1.0f / TRAILING_AVERAGE_FRAMES; - const float PREVIOUS_FRAMES_RATIO = 1.0f - CURRENT_FRAME_RATIO; - - const float STRUGGLE_TRIGGER_SLEEP_PERCENTAGE_THRESHOLD = 0.10f; - const float BACK_OFF_TRIGGER_SLEEP_PERCENTAGE_THRESHOLD = 0.20f; - - const float RATIO_BACK_OFF = 0.02f; - - auto nextFrameTimestamp = p_high_resolution_clock::now(); - auto timeToSleep = std::chrono::microseconds(0); - - int currentFrame = 1; - int numFramesPerSecond = (int) ceil(AudioConstants::NETWORK_FRAMES_PER_SEC); - int framesSinceCutoffEvent = TRAILING_AVERAGE_FRAMES; + // mixFrame state + unsigned int frame = 1; while (!_isFinished) { - // manage mixer load - { - _trailingSleepRatio = (PREVIOUS_FRAMES_RATIO * _trailingSleepRatio) + - // ratio of frame spent sleeping / total frame time - ((CURRENT_FRAME_RATIO * timeToSleep.count()) / (float) AudioConstants::NETWORK_FRAME_USECS); - - bool hasRatioChanged = false; - - if (framesSinceCutoffEvent >= TRAILING_AVERAGE_FRAMES) { - if (_trailingSleepRatio <= STRUGGLE_TRIGGER_SLEEP_PERCENTAGE_THRESHOLD) { - qDebug() << "Mixer is struggling"; - // change our min required loudness to reduce some load - _performanceThrottlingRatio = _performanceThrottlingRatio + (0.5f * (1.0f - _performanceThrottlingRatio)); - hasRatioChanged = true; - } else if (_trailingSleepRatio >= BACK_OFF_TRIGGER_SLEEP_PERCENTAGE_THRESHOLD && _performanceThrottlingRatio != 0) { - qDebug() << "Mixer is recovering"; - // back off the required loudness - _performanceThrottlingRatio = std::max(0.0f, _performanceThrottlingRatio - RATIO_BACK_OFF); - hasRatioChanged = true; - } - - if (hasRatioChanged) { - // set out min audability threshold from the new ratio - _minAudibilityThreshold = LOUDNESS_TO_DISTANCE_RATIO / (2.0f * (1.0f - _performanceThrottlingRatio)); - framesSinceCutoffEvent = 0; - - qDebug() << "Sleeping" << _trailingSleepRatio << "of frame"; - qDebug() << "Cutoff is" << _performanceThrottlingRatio; - qDebug() << "Minimum audibility to be mixed is" << _minAudibilityThreshold; - } - } - - if (!hasRatioChanged) { - ++framesSinceCutoffEvent; - } - } - - // mix + manageLoad(frameTimestamp, framesSinceManagement); nodeList->eachNode([&](const SharedNodePointer& node) { - if (node->getLinkedData()) { - AudioMixerClientData* nodeData = (AudioMixerClientData*)node->getLinkedData(); - - // this function will attempt to pop a frame from each audio stream. - // a pointer to the popped data is stored as a member in InboundAudioStream. - // That's how the popped audio data will be read for mixing (but only if the pop was successful) - _sumStreams += nodeData->checkBuffersBeforeFrameSend(); - - // if the stream should be muted, send mute packet - if (nodeData->getAvatarAudioStream() - && (shouldMute(nodeData->getAvatarAudioStream()->getQuietestFrameLoudness()) - || nodeData->shouldMuteClient())) { - auto mutePacket = NLPacket::create(PacketType::NoisyMute, 0); - nodeList->sendPacket(std::move(mutePacket), *node); - - // probably now we just reset the flag, once should do it (?) - nodeData->setShouldMuteClient(false); - } - - if (node->getType() == NodeType::Agent && node->getActiveSocket() - && nodeData->getAvatarAudioStream()) { - - bool mixHasAudio = prepareMixForListeningNode(node.data()); - - std::unique_ptr mixPacket; - - if (mixHasAudio || nodeData->shouldFlushEncoder()) { - - int mixPacketBytes = sizeof(quint16) + AudioConstants::MAX_CODEC_NAME_LENGTH_ON_WIRE - + AudioConstants::NETWORK_FRAME_BYTES_STEREO; - mixPacket = NLPacket::create(PacketType::MixedAudio, mixPacketBytes); - - // pack sequence number - quint16 sequence = nodeData->getOutgoingSequenceNumber(); - mixPacket->writePrimitive(sequence); - - // write the codec - QString codecInPacket = nodeData->getCodecName(); - mixPacket->writeString(codecInPacket); - - QByteArray encodedBuffer; - if (mixHasAudio) { - QByteArray decodedBuffer(reinterpret_cast(_clampedSamples), AudioConstants::NETWORK_FRAME_BYTES_STEREO); - nodeData->encode(decodedBuffer, encodedBuffer); - } else { - // time to flush, which resets the shouldFlush until next time we encode something - nodeData->encodeFrameOfZeros(encodedBuffer); - } - // pack mixed audio samples - mixPacket->write(encodedBuffer.constData(), encodedBuffer.size()); - - } else { - int silentPacketBytes = sizeof(quint16) + sizeof(quint16) + AudioConstants::MAX_CODEC_NAME_LENGTH_ON_WIRE; - mixPacket = NLPacket::create(PacketType::SilentAudioFrame, silentPacketBytes); - - // pack sequence number - quint16 sequence = nodeData->getOutgoingSequenceNumber(); - mixPacket->writePrimitive(sequence); - - // write the codec - QString codecInPacket = nodeData->getCodecName(); - mixPacket->writeString(codecInPacket); - - // pack number of silent audio samples - quint16 numSilentSamples = AudioConstants::NETWORK_FRAME_SAMPLES_STEREO; - mixPacket->writePrimitive(numSilentSamples); - } - - // Send audio environment - sendAudioEnvironmentPacket(node); - - // send mixed audio packet - nodeList->sendPacket(std::move(mixPacket), *node); - nodeData->incrementOutgoingMixedAudioSequenceNumber(); - - // send an audio stream stats packet to the client approximately every second - ++currentFrame; - currentFrame %= numFramesPerSecond; - - if (nodeData->shouldSendStats(currentFrame)) { - nodeData->sendAudioStreamStatsPackets(node); - } - - ++_sumListeners; - } + _sumStreams += prepareFrame(node, frame); + }); + nodeList->eachNode([&](const SharedNodePointer& node) { + if(mixFrame(node, frame)) { + ++_sumListeners; } }); + ++frame; ++_numStatFrames; // play nice with qt event-looping { - // since we're a while loop we need to help qt's event processing + // since we're a while loop we need to yield to qt's event processing QCoreApplication::processEvents(); if (_isFinished) { - // alert qt that this is finished + // alert qt eventing that this is finished QCoreApplication::sendPostedEvents(this, QEvent::DeferredDelete); break; } } + } +} - // sleep until the next frame, if necessary - { - nextFrameTimestamp += std::chrono::microseconds(AudioConstants::NETWORK_FRAME_USECS); +void AudioMixer::manageLoad(p_high_resolution_clock::time_point& frameTimestamp, unsigned int& framesSinceCutoffEvent) { + auto timeToSleep = std::chrono::microseconds(0); - auto now = p_high_resolution_clock::now(); - timeToSleep = std::chrono::duration_cast(nextFrameTimestamp - now); - - if (timeToSleep.count() < 0) { - nextFrameTimestamp = now; - timeToSleep = std::chrono::microseconds(0); - } + // sleep until the next frame, if necessary + { + // advance the next frame + frameTimestamp += std::chrono::microseconds(AudioConstants::NETWORK_FRAME_USECS); + auto now = p_high_resolution_clock::now(); + // calculate sleep + if (frameTimestamp < now) { + frameTimestamp = now; + } else { + timeToSleep = std::chrono::duration_cast(frameTimestamp - now); std::this_thread::sleep_for(timeToSleep); } } + + // manage mixer load + { + const int TRAILING_AVERAGE_FRAMES = 100; + const float CURRENT_FRAME_RATIO = 1.0f / TRAILING_AVERAGE_FRAMES; + const float PREVIOUS_FRAMES_RATIO = 1.0f - CURRENT_FRAME_RATIO; + + const float STRUGGLE_TRIGGER_SLEEP_PERCENTAGE_THRESHOLD = 0.10f; + const float BACK_OFF_TRIGGER_SLEEP_PERCENTAGE_THRESHOLD = 0.20f; + + const float RATIO_BACK_OFF = 0.02f; + + _trailingSleepRatio = (PREVIOUS_FRAMES_RATIO * _trailingSleepRatio) + + // ratio of frame spent sleeping / total frame time + ((CURRENT_FRAME_RATIO * timeToSleep.count()) / (float) AudioConstants::NETWORK_FRAME_USECS); + + bool hasRatioChanged = false; + + if (framesSinceCutoffEvent >= TRAILING_AVERAGE_FRAMES) { + if (_trailingSleepRatio <= STRUGGLE_TRIGGER_SLEEP_PERCENTAGE_THRESHOLD) { + qDebug() << "Mixer is struggling"; + // change our min required loudness to reduce some load + _performanceThrottlingRatio = _performanceThrottlingRatio + (0.5f * (1.0f - _performanceThrottlingRatio)); + hasRatioChanged = true; + } else if (_trailingSleepRatio >= BACK_OFF_TRIGGER_SLEEP_PERCENTAGE_THRESHOLD && _performanceThrottlingRatio != 0) { + qDebug() << "Mixer is recovering"; + // back off the required loudness + _performanceThrottlingRatio = std::max(0.0f, _performanceThrottlingRatio - RATIO_BACK_OFF); + hasRatioChanged = true; + } + + if (hasRatioChanged) { + // set out min audability threshold from the new ratio + _minAudibilityThreshold = LOUDNESS_TO_DISTANCE_RATIO / (2.0f * (1.0f - _performanceThrottlingRatio)); + framesSinceCutoffEvent = 0; + + qDebug() << "Sleeping" << _trailingSleepRatio << "of frame"; + qDebug() << "Cutoff is" << _performanceThrottlingRatio; + qDebug() << "Minimum audibility to be mixed is" << _minAudibilityThreshold; + } + } + + if (!hasRatioChanged) { + ++framesSinceCutoffEvent; + } + } +} + +int AudioMixer::prepareFrame(const SharedNodePointer& node, unsigned int frame) { + AudioMixerClientData* data = (AudioMixerClientData*)node->getLinkedData(); + if (data == nullptr) { + return 0; + } + + return data->checkBuffersBeforeFrameSend(); +} + +bool AudioMixer::mixFrame(const SharedNodePointer& node, unsigned int frame) { + AudioMixerClientData* data = (AudioMixerClientData*)node->getLinkedData(); + if (data == nullptr) { + return false; + } + + auto avatarStream = data->getAvatarAudioStream(); + if (avatarStream == nullptr) { + return false; + } + + auto nodeList = DependencyManager::get(); + + // mute the avatar, if necessary + if (shouldMute(avatarStream->getQuietestFrameLoudness()) || data->shouldMuteClient()) { + auto mutePacket = NLPacket::create(PacketType::NoisyMute, 0); + nodeList->sendPacket(std::move(mutePacket), *node); + + // probably now we just reset the flag, once should do it (?) + data->setShouldMuteClient(false); + } + + // mix streams + if (node->getType() == NodeType::Agent && node->getActiveSocket()) { + + bool mixHasAudio = prepareMixForListeningNode(node.data()); + + std::unique_ptr mixPacket; + + if (mixHasAudio || data->shouldFlushEncoder()) { + + int mixPacketBytes = sizeof(quint16) + AudioConstants::MAX_CODEC_NAME_LENGTH_ON_WIRE + + AudioConstants::NETWORK_FRAME_BYTES_STEREO; + mixPacket = NLPacket::create(PacketType::MixedAudio, mixPacketBytes); + + // pack sequence number + quint16 sequence = data->getOutgoingSequenceNumber(); + mixPacket->writePrimitive(sequence); + + // write the codec + QString codecInPacket = data->getCodecName(); + mixPacket->writeString(codecInPacket); + + QByteArray encodedBuffer; + if (mixHasAudio) { + QByteArray decodedBuffer(reinterpret_cast(_clampedSamples), AudioConstants::NETWORK_FRAME_BYTES_STEREO); + data->encode(decodedBuffer, encodedBuffer); + } else { + // time to flush, which resets the shouldFlush until next time we encode something + data->encodeFrameOfZeros(encodedBuffer); + } + // pack mixed audio samples + mixPacket->write(encodedBuffer.constData(), encodedBuffer.size()); + + } else { + int silentPacketBytes = sizeof(quint16) + sizeof(quint16) + AudioConstants::MAX_CODEC_NAME_LENGTH_ON_WIRE; + mixPacket = NLPacket::create(PacketType::SilentAudioFrame, silentPacketBytes); + + // pack sequence number + quint16 sequence = data->getOutgoingSequenceNumber(); + mixPacket->writePrimitive(sequence); + + // write the codec + QString codecInPacket = data->getCodecName(); + mixPacket->writeString(codecInPacket); + + // pack number of silent audio samples + quint16 numSilentSamples = AudioConstants::NETWORK_FRAME_SAMPLES_STEREO; + mixPacket->writePrimitive(numSilentSamples); + } + + // Send audio environment + sendAudioEnvironmentPacket(node); + + // send mixed audio packet + nodeList->sendPacket(std::move(mixPacket), *node); + data->incrementOutgoingMixedAudioSequenceNumber(); + + // send an audio stream stats packet to the client approximately every second + static const unsigned int NUM_FRAMES_PER_SEC = (int) ceil(AudioConstants::NETWORK_FRAMES_PER_SEC); + if (data->shouldSendStats(frame % NUM_FRAMES_PER_SEC)) { + data->sendAudioStreamStatsPackets(node); + } + + return true; + } + + return false; } void AudioMixer::parseSettingsObject(const QJsonObject &settingsObject) { diff --git a/assignment-client/src/audio/AudioMixer.h b/assignment-client/src/audio/AudioMixer.h index 6bc035e9b4..78e9e85e13 100644 --- a/assignment-client/src/audio/AudioMixer.h +++ b/assignment-client/src/audio/AudioMixer.h @@ -30,14 +30,13 @@ public: AudioMixer(ReceivedMessage& message); public slots: - /// threaded run of assignment void run() override; - void sendStatsPacket() override; static int getStaticJitterFrames() { return _numStaticJitterFrames; } private slots: + // packet handlers void handleNodeAudioPacket(QSharedPointer packet, SharedNodePointer sendingNode); void handleMuteEnvironmentPacket(QSharedPointer packet, SharedNodePointer sendingNode); void handleNegotiateAudioFormat(QSharedPointer message, SharedNodePointer sendingNode); @@ -51,6 +50,16 @@ private slots: void removeHRTFsForFinishedInjector(const QUuid& streamID); private: + // mixing helpers + // check and maybe throttle mixer load by changing audibility threshold + void manageLoad(p_high_resolution_clock::time_point& frameTimestamp, unsigned int& framesSinceManagement); + // pop a frame from any streams on the node + // returns the number of available streams + int prepareFrame(const SharedNodePointer& node, unsigned int frame); + // mix and broadcast non-ignored streams to the node + // returns true if a listener mix was broadcast for the node + bool mixFrame(const SharedNodePointer& node, unsigned int frame); + AudioMixerClientData* getOrCreateClientData(Node* node); /// adds one stream to the mix for a listening node From 192f4791d5e17f122be8fb06d1b8692ce306a2b7 Mon Sep 17 00:00:00 2001 From: Zach Pomerantz Date: Tue, 22 Nov 2016 16:19:18 -0500 Subject: [PATCH 05/48] move mixing into AudioMixerSlave --- assignment-client/src/audio/AudioMixer.cpp | 900 +++++++++++---------- assignment-client/src/audio/AudioMixer.h | 117 +-- 2 files changed, 528 insertions(+), 489 deletions(-) diff --git a/assignment-client/src/audio/AudioMixer.cpp b/assignment-client/src/audio/AudioMixer.cpp index 7d1f4aba7c..c1402529fa 100644 --- a/assignment-client/src/audio/AudioMixer.cpp +++ b/assignment-client/src/audio/AudioMixer.cpp @@ -69,19 +69,17 @@ static const QString AUDIO_ENV_GROUP_KEY = "audio_env"; static const QString AUDIO_BUFFER_GROUP_KEY = "audio_buffer"; int AudioMixer::_numStaticJitterFrames{ -1 }; - -bool AudioMixer::shouldMute(float quietestFrame) { - return (quietestFrame > _noiseMutingThreshold); -} +float AudioMixer::_noiseMutingThreshold{ DEFAULT_NOISE_MUTING_THRESHOLD }; +float AudioMixer::_attenuationPerDoublingInDistance{ DEFAULT_ATTENUATION_PER_DOUBLING_IN_DISTANCE }; +float AudioMixer::_trailingSleepRatio{ 1.0f }; +float AudioMixer::_performanceThrottlingRatio{ 0.0f }; +float AudioMixer::_minAudibilityThreshold{ LOUDNESS_TO_DISTANCE_RATIO / 2.0f }; +QHash AudioMixer::_audioZones; +QVector AudioMixer::_zoneSettings; +QVector AudioMixer::_zoneReverbSettings; AudioMixer::AudioMixer(ReceivedMessage& message) : - ThreadedAssignment(message), - _trailingSleepRatio(1.0f), - _minAudibilityThreshold(LOUDNESS_TO_DISTANCE_RATIO / 2.0f), - _performanceThrottlingRatio(0.0f), - _attenuationPerDoublingInDistance(DEFAULT_ATTENUATION_PER_DOUBLING_IN_DISTANCE), - _noiseMutingThreshold(DEFAULT_NOISE_MUTING_THRESHOLD) -{ + ThreadedAssignment(message) { auto nodeList = DependencyManager::get(); auto& packetReceiver = nodeList->getPacketReceiver(); @@ -158,343 +156,6 @@ static inline float fastexp2(float x) { return x * xi.f; } -float AudioMixer::gainForSource(const PositionalAudioStream& streamToAdd, - const AvatarAudioStream& listeningNodeStream, const glm::vec3& relativePosition, bool isEcho) { - float gain = 1.0f; - - float distanceBetween = glm::length(relativePosition); - - if (distanceBetween < EPSILON) { - distanceBetween = EPSILON; - } - - if (streamToAdd.getType() == PositionalAudioStream::Injector) { - gain *= reinterpret_cast(&streamToAdd)->getAttenuationRatio(); - } - - if (!isEcho && (streamToAdd.getType() == PositionalAudioStream::Microphone)) { - // source is another avatar, apply fixed off-axis attenuation to make them quieter as they turn away from listener - glm::vec3 rotatedListenerPosition = glm::inverse(streamToAdd.getOrientation()) * relativePosition; - - float angleOfDelivery = glm::angle(glm::vec3(0.0f, 0.0f, -1.0f), - glm::normalize(rotatedListenerPosition)); - - const float MAX_OFF_AXIS_ATTENUATION = 0.2f; - const float OFF_AXIS_ATTENUATION_FORMULA_STEP = (1 - MAX_OFF_AXIS_ATTENUATION) / 2.0f; - - float offAxisCoefficient = MAX_OFF_AXIS_ATTENUATION + - (OFF_AXIS_ATTENUATION_FORMULA_STEP * (angleOfDelivery / PI_OVER_TWO)); - - // multiply the current attenuation coefficient by the calculated off axis coefficient - gain *= offAxisCoefficient; - } - - float attenuationPerDoublingInDistance = _attenuationPerDoublingInDistance; - for (int i = 0; i < _zonesSettings.length(); ++i) { - if (_audioZones[_zonesSettings[i].source].contains(streamToAdd.getPosition()) && - _audioZones[_zonesSettings[i].listener].contains(listeningNodeStream.getPosition())) { - attenuationPerDoublingInDistance = _zonesSettings[i].coefficient; - break; - } - } - - if (distanceBetween >= ATTENUATION_BEGINS_AT_DISTANCE) { - - // translate the zone setting to gain per log2(distance) - float g = 1.0f - attenuationPerDoublingInDistance; - g = (g < EPSILON) ? EPSILON : g; - g = (g > 1.0f) ? 1.0f : g; - - // calculate the distance coefficient using the distance to this node - float distanceCoefficient = fastexp2(fastlog2(g) * fastlog2(distanceBetween/ATTENUATION_BEGINS_AT_DISTANCE)); - - // multiply the current attenuation coefficient by the distance coefficient - gain *= distanceCoefficient; - } - - return gain; -} - -float AudioMixer::azimuthForSource(const PositionalAudioStream& streamToAdd, const AvatarAudioStream& listeningNodeStream, - const glm::vec3& relativePosition) { - glm::quat inverseOrientation = glm::inverse(listeningNodeStream.getOrientation()); - - // Compute sample delay for the two ears to create phase panning - glm::vec3 rotatedSourcePosition = inverseOrientation * relativePosition; - - // project the rotated source position vector onto the XZ plane - rotatedSourcePosition.y = 0.0f; - - static const float SOURCE_DISTANCE_THRESHOLD = 1e-30f; - - if (glm::length2(rotatedSourcePosition) > SOURCE_DISTANCE_THRESHOLD) { - // produce an oriented angle about the y-axis - return glm::orientedAngle(glm::vec3(0.0f, 0.0f, -1.0f), glm::normalize(rotatedSourcePosition), glm::vec3(0.0f, -1.0f, 0.0f)); - } else { - // there is no distance between listener and source - return no azimuth - return 0; - } -} - -void AudioMixer::addStreamToMixForListeningNodeWithStream(AudioMixerClientData& listenerNodeData, - const PositionalAudioStream& streamToAdd, - const QUuid& sourceNodeID, - const AvatarAudioStream& listeningNodeStream) { - - - // to reduce artifacts we calculate the gain and azimuth for every source for this listener - // even if we are not going to end up mixing in this source - - ++_totalMixes; - - // this ensures that the tail of any previously mixed audio or the first block of new audio sounds correct - - // check if this is a server echo of a source back to itself - bool isEcho = (&streamToAdd == &listeningNodeStream); - - glm::vec3 relativePosition = streamToAdd.getPosition() - listeningNodeStream.getPosition(); - - // figure out the distance between source and listener - float distance = glm::max(glm::length(relativePosition), EPSILON); - - // figure out the gain for this source at the listener - float gain = gainForSource(streamToAdd, listeningNodeStream, relativePosition, isEcho); - - // figure out the azimuth to this source at the listener - float azimuth = isEcho ? 0.0f : azimuthForSource(streamToAdd, listeningNodeStream, relativePosition); - - float repeatedFrameFadeFactor = 1.0f; - - static const int HRTF_DATASET_INDEX = 1; - - if (!streamToAdd.lastPopSucceeded()) { - bool forceSilentBlock = true; - - if (!streamToAdd.getLastPopOutput().isNull()) { - bool isInjector = dynamic_cast(&streamToAdd); - - // in an injector, just go silent - the injector has likely ended - // in other inputs (microphone, &c.), repeat with fade to avoid the harsh jump to silence - - // we'll repeat the last block until it has a block to mix - // and we'll gradually fade that repeated block into silence. - - // calculate its fade factor, which depends on how many times it's already been repeated. - repeatedFrameFadeFactor = calculateRepeatedFrameFadeFactor(streamToAdd.getConsecutiveNotMixedCount() - 1); - if (!isInjector && repeatedFrameFadeFactor > 0.0f) { - // apply the repeatedFrameFadeFactor to the gain - gain *= repeatedFrameFadeFactor; - - forceSilentBlock = false; - } - } - - if (forceSilentBlock) { - // we're deciding not to repeat either since we've already done it enough times or repetition with fade is disabled - // in this case we will call renderSilent with a forced silent block - // this ensures the correct tail from the previously mixed block and the correct spatialization of first block - // of any upcoming audio - - if (!streamToAdd.isStereo() && !isEcho) { - // get the existing listener-source HRTF object, or create a new one - auto& hrtf = listenerNodeData.hrtfForStream(sourceNodeID, streamToAdd.getStreamIdentifier()); - - // this is not done for stereo streams since they do not go through the HRTF - static int16_t silentMonoBlock[AudioConstants::NETWORK_FRAME_SAMPLES_PER_CHANNEL] = {}; - hrtf.renderSilent(silentMonoBlock, _mixedSamples, HRTF_DATASET_INDEX, azimuth, distance, gain, - AudioConstants::NETWORK_FRAME_SAMPLES_PER_CHANNEL); - - ++_hrtfSilentRenders;; - } - - return; - } - } - - // grab the stream from the ring buffer - AudioRingBuffer::ConstIterator streamPopOutput = streamToAdd.getLastPopOutput(); - - if (streamToAdd.isStereo() || isEcho) { - // this is a stereo source or server echo so we do not pass it through the HRTF - // simply apply our calculated gain to each sample - if (streamToAdd.isStereo()) { - for (int i = 0; i < AudioConstants::NETWORK_FRAME_SAMPLES_STEREO; ++i) { - _mixedSamples[i] += float(streamPopOutput[i] * gain / AudioConstants::MAX_SAMPLE_VALUE); - } - - ++_manualStereoMixes; - } else { - for (int i = 0; i < AudioConstants::NETWORK_FRAME_SAMPLES_STEREO; i += 2) { - auto monoSample = float(streamPopOutput[i / 2] * gain / AudioConstants::MAX_SAMPLE_VALUE); - _mixedSamples[i] += monoSample; - _mixedSamples[i + 1] += monoSample; - } - - ++_manualEchoMixes; - } - - return; - } - - // get the existing listener-source HRTF object, or create a new one - auto& hrtf = listenerNodeData.hrtfForStream(sourceNodeID, streamToAdd.getStreamIdentifier()); - - static int16_t streamBlock[AudioConstants::NETWORK_FRAME_SAMPLES_PER_CHANNEL]; - - streamPopOutput.readSamples(streamBlock, AudioConstants::NETWORK_FRAME_SAMPLES_PER_CHANNEL); - - // if the frame we're about to mix is silent, simply call render silent and move on - if (streamToAdd.getLastPopOutputLoudness() == 0.0f) { - // silent frame from source - - // we still need to call renderSilent via the HRTF for mono source - hrtf.renderSilent(streamBlock, _mixedSamples, HRTF_DATASET_INDEX, azimuth, distance, gain, - AudioConstants::NETWORK_FRAME_SAMPLES_PER_CHANNEL); - - ++_hrtfSilentRenders; - - return; - } - - if (_performanceThrottlingRatio > 0.0f - && streamToAdd.getLastPopOutputTrailingLoudness() / glm::length(relativePosition) <= _minAudibilityThreshold) { - // the mixer is struggling so we're going to drop off some streams - - // we call renderSilent via the HRTF with the actual frame data and a gain of 0.0 - hrtf.renderSilent(streamBlock, _mixedSamples, HRTF_DATASET_INDEX, azimuth, distance, 0.0f, - AudioConstants::NETWORK_FRAME_SAMPLES_PER_CHANNEL); - - ++_hrtfStruggleRenders; - - return; - } - - ++_hrtfRenders; - - // mono stream, call the HRTF with our block and calculated azimuth and gain - hrtf.render(streamBlock, _mixedSamples, HRTF_DATASET_INDEX, azimuth, distance, gain, - AudioConstants::NETWORK_FRAME_SAMPLES_PER_CHANNEL); -} - -bool AudioMixer::prepareMixForListeningNode(Node* node) { - AvatarAudioStream* nodeAudioStream = static_cast(node->getLinkedData())->getAvatarAudioStream(); - AudioMixerClientData* listenerNodeData = static_cast(node->getLinkedData()); - - // zero out the client mix for this node - memset(_mixedSamples, 0, sizeof(_mixedSamples)); - - // loop through all other nodes that have sufficient audio to mix - - DependencyManager::get()->eachNode([&](const SharedNodePointer& otherNode){ - // make sure that we have audio data for this other node - // and that it isn't being ignored by our listening node - // and that it isn't ignoring our listening node - if (otherNode->getLinkedData() - && !node->isIgnoringNodeWithID(otherNode->getUUID()) && !otherNode->isIgnoringNodeWithID(node->getUUID())) { - AudioMixerClientData* otherNodeClientData = (AudioMixerClientData*) otherNode->getLinkedData(); - - // check to see if we're ignoring in radius - bool insideIgnoreRadius = false; - if (node->isIgnoreRadiusEnabled() || otherNode->isIgnoreRadiusEnabled()) { - AudioMixerClientData* otherData = reinterpret_cast(otherNode->getLinkedData()); - AudioMixerClientData* nodeData = reinterpret_cast(node->getLinkedData()); - float ignoreRadius = glm::min(node->getIgnoreRadius(), otherNode->getIgnoreRadius()); - if (glm::distance(nodeData->getPosition(), otherData->getPosition()) < ignoreRadius) { - insideIgnoreRadius = true; - } - } - - if (!insideIgnoreRadius) { - // enumerate the ARBs attached to the otherNode and add all that should be added to mix - auto streamsCopy = otherNodeClientData->getAudioStreams(); - for (auto& streamPair : streamsCopy) { - auto otherNodeStream = streamPair.second; - if (*otherNode != *node || otherNodeStream->shouldLoopbackForNode()) { - addStreamToMixForListeningNodeWithStream(*listenerNodeData, *otherNodeStream, otherNode->getUUID(), - *nodeAudioStream); - } - } - } - } - }); - - // use the per listner AudioLimiter to render the mixed data... - listenerNodeData->audioLimiter.render(_mixedSamples, _clampedSamples, AudioConstants::NETWORK_FRAME_SAMPLES_PER_CHANNEL); - - // check for silent audio after the peak limitor has converted the samples - bool hasAudio = false; - for (int i = 0; i < AudioConstants::NETWORK_FRAME_SAMPLES_STEREO; ++i) { - if (_clampedSamples[i] != 0) { - hasAudio = true; - break; - } - } - return hasAudio; -} - -void AudioMixer::sendAudioEnvironmentPacket(SharedNodePointer node) { - // Send stream properties - bool hasReverb = false; - float reverbTime, wetLevel; - // find reverb properties - for (int i = 0; i < _zoneReverbSettings.size(); ++i) { - AudioMixerClientData* data = static_cast(node->getLinkedData()); - glm::vec3 streamPosition = data->getAvatarAudioStream()->getPosition(); - AABox box = _audioZones[_zoneReverbSettings[i].zone]; - if (box.contains(streamPosition)) { - hasReverb = true; - reverbTime = _zoneReverbSettings[i].reverbTime; - wetLevel = _zoneReverbSettings[i].wetLevel; - - break; - } - } - - AudioMixerClientData* nodeData = static_cast(node->getLinkedData()); - AvatarAudioStream* stream = nodeData->getAvatarAudioStream(); - bool dataChanged = (stream->hasReverb() != hasReverb) || - (stream->hasReverb() && (stream->getRevebTime() != reverbTime || - stream->getWetLevel() != wetLevel)); - if (dataChanged) { - // Update stream - if (hasReverb) { - stream->setReverb(reverbTime, wetLevel); - } else { - stream->clearReverb(); - } - } - - // Send at change or every so often - float CHANCE_OF_SEND = 0.01f; - bool sendData = dataChanged || (randFloat() < CHANCE_OF_SEND); - - if (sendData) { - auto nodeList = DependencyManager::get(); - - unsigned char bitset = 0; - - int packetSize = sizeof(bitset); - - if (hasReverb) { - packetSize += sizeof(reverbTime) + sizeof(wetLevel); - } - - auto envPacket = NLPacket::create(PacketType::AudioEnvironment, packetSize); - - if (hasReverb) { - setAtBit(bitset, HAS_REVERB_BIT); - } - - envPacket->writePrimitive(bitset); - - if (hasReverb) { - envPacket->writePrimitive(reverbTime); - envPacket->writePrimitive(wetLevel); - } - nodeList->sendPacket(std::move(envPacket), *node); - } -} - void AudioMixer::handleNodeAudioPacket(QSharedPointer message, SharedNodePointer sendingNode) { getOrCreateClientData(sendingNode.data()); DependencyManager::get()->updateNodeWithDataFromPacket(message, sendingNode); @@ -780,20 +441,25 @@ void AudioMixer::start() { auto frameTimestamp = p_high_resolution_clock::time_point::min(); unsigned int framesSinceManagement = std::numeric_limits::max(); - // mixFrame state + // mix state unsigned int frame = 1; while (!_isFinished) { manageLoad(frameTimestamp, framesSinceManagement); + + slave.resetStats(); + nodeList->eachNode([&](const SharedNodePointer& node) { _sumStreams += prepareFrame(node, frame); }); nodeList->eachNode([&](const SharedNodePointer& node) { - if(mixFrame(node, frame)) { + if (slave.mix(node, frame)) { ++_sumListeners; } }); + slave.getStats(); + ++frame; ++_numStatFrames; @@ -885,96 +551,6 @@ int AudioMixer::prepareFrame(const SharedNodePointer& node, unsigned int frame) return data->checkBuffersBeforeFrameSend(); } -bool AudioMixer::mixFrame(const SharedNodePointer& node, unsigned int frame) { - AudioMixerClientData* data = (AudioMixerClientData*)node->getLinkedData(); - if (data == nullptr) { - return false; - } - - auto avatarStream = data->getAvatarAudioStream(); - if (avatarStream == nullptr) { - return false; - } - - auto nodeList = DependencyManager::get(); - - // mute the avatar, if necessary - if (shouldMute(avatarStream->getQuietestFrameLoudness()) || data->shouldMuteClient()) { - auto mutePacket = NLPacket::create(PacketType::NoisyMute, 0); - nodeList->sendPacket(std::move(mutePacket), *node); - - // probably now we just reset the flag, once should do it (?) - data->setShouldMuteClient(false); - } - - // mix streams - if (node->getType() == NodeType::Agent && node->getActiveSocket()) { - - bool mixHasAudio = prepareMixForListeningNode(node.data()); - - std::unique_ptr mixPacket; - - if (mixHasAudio || data->shouldFlushEncoder()) { - - int mixPacketBytes = sizeof(quint16) + AudioConstants::MAX_CODEC_NAME_LENGTH_ON_WIRE - + AudioConstants::NETWORK_FRAME_BYTES_STEREO; - mixPacket = NLPacket::create(PacketType::MixedAudio, mixPacketBytes); - - // pack sequence number - quint16 sequence = data->getOutgoingSequenceNumber(); - mixPacket->writePrimitive(sequence); - - // write the codec - QString codecInPacket = data->getCodecName(); - mixPacket->writeString(codecInPacket); - - QByteArray encodedBuffer; - if (mixHasAudio) { - QByteArray decodedBuffer(reinterpret_cast(_clampedSamples), AudioConstants::NETWORK_FRAME_BYTES_STEREO); - data->encode(decodedBuffer, encodedBuffer); - } else { - // time to flush, which resets the shouldFlush until next time we encode something - data->encodeFrameOfZeros(encodedBuffer); - } - // pack mixed audio samples - mixPacket->write(encodedBuffer.constData(), encodedBuffer.size()); - - } else { - int silentPacketBytes = sizeof(quint16) + sizeof(quint16) + AudioConstants::MAX_CODEC_NAME_LENGTH_ON_WIRE; - mixPacket = NLPacket::create(PacketType::SilentAudioFrame, silentPacketBytes); - - // pack sequence number - quint16 sequence = data->getOutgoingSequenceNumber(); - mixPacket->writePrimitive(sequence); - - // write the codec - QString codecInPacket = data->getCodecName(); - mixPacket->writeString(codecInPacket); - - // pack number of silent audio samples - quint16 numSilentSamples = AudioConstants::NETWORK_FRAME_SAMPLES_STEREO; - mixPacket->writePrimitive(numSilentSamples); - } - - // Send audio environment - sendAudioEnvironmentPacket(node); - - // send mixed audio packet - nodeList->sendPacket(std::move(mixPacket), *node); - data->incrementOutgoingMixedAudioSequenceNumber(); - - // send an audio stream stats packet to the client approximately every second - static const unsigned int NUM_FRAMES_PER_SEC = (int) ceil(AudioConstants::NETWORK_FRAMES_PER_SEC); - if (data->shouldSendStats(frame % NUM_FRAMES_PER_SEC)) { - data->sendAudioStreamStatsPackets(node); - } - - return true; - } - - return false; -} - void AudioMixer::parseSettingsObject(const QJsonObject &settingsObject) { if (settingsObject.contains(AUDIO_BUFFER_GROUP_KEY)) { QJsonObject audioBufferGroupObject = settingsObject[AUDIO_BUFFER_GROUP_KEY].toObject(); @@ -1126,7 +702,7 @@ void AudioMixer::parseSettingsObject(const QJsonObject &settingsObject) { coefficientObject.contains(LISTENER) && coefficientObject.contains(COEFFICIENT)) { - ZonesSettings settings; + ZoneSettings settings; bool ok; settings.source = coefficientObject.value(SOURCE).toString(); @@ -1136,7 +712,7 @@ void AudioMixer::parseSettingsObject(const QJsonObject &settingsObject) { if (ok && settings.coefficient >= 0.0f && settings.coefficient <= 1.0f && _audioZones.contains(settings.source) && _audioZones.contains(settings.listener)) { - _zonesSettings.push_back(settings); + _zoneSettings.push_back(settings); qDebug() << "Added Coefficient:" << settings.source << settings.listener << settings.coefficient; } } @@ -1169,6 +745,7 @@ void AudioMixer::parseSettingsObject(const QJsonObject &settingsObject) { settings.wetLevel = wetLevel; _zoneReverbSettings.push_back(settings); + qDebug() << "Added Reverb:" << zone << reverbTime << wetLevel; } } @@ -1176,3 +753,442 @@ void AudioMixer::parseSettingsObject(const QJsonObject &settingsObject) { } } } + +bool AudioMixerSlave::mix(const SharedNodePointer& node, unsigned int frame) { + AudioMixerClientData* data = (AudioMixerClientData*)node->getLinkedData(); + if (data == nullptr) { + return false; + } + + auto avatarStream = data->getAvatarAudioStream(); + if (avatarStream == nullptr) { + return false; + } + + auto nodeList = DependencyManager::get(); + + // mute the avatar, if necessary + if (AudioMixer::shouldMute(avatarStream->getQuietestFrameLoudness()) || data->shouldMuteClient()) { + auto mutePacket = NLPacket::create(PacketType::NoisyMute, 0); + nodeList->sendPacket(std::move(mutePacket), *node); + + // probably now we just reset the flag, once should do it (?) + data->setShouldMuteClient(false); + } + + // generate and send audio packets + if (node->getType() == NodeType::Agent && node->getActiveSocket()) { + + // mix streams + bool mixHasAudio = prepareMix(node); + + // write the packet + std::unique_ptr mixPacket; + if (mixHasAudio || data->shouldFlushEncoder()) { + // encode the audio + QByteArray encodedBuffer; + if (mixHasAudio) { + QByteArray decodedBuffer(reinterpret_cast(_clampedSamples), AudioConstants::NETWORK_FRAME_BYTES_STEREO); + data->encode(decodedBuffer, encodedBuffer); + } else { + // time to flush, which resets the shouldFlush until next time we encode something + data->encodeFrameOfZeros(encodedBuffer); + } + + // write it to a packet + writeMixPacket(mixPacket, data, encodedBuffer); + } else { + writeSilentPacket(mixPacket, data); + } + + // send audio environment packet + sendEnvironmentPacket(node); + + // send mixed audio packet + nodeList->sendPacket(std::move(mixPacket), *node); + data->incrementOutgoingMixedAudioSequenceNumber(); + + // send an audio stream stats packet to the client approximately every second + static const unsigned int NUM_FRAMES_PER_SEC = (int) ceil(AudioConstants::NETWORK_FRAMES_PER_SEC); + if (data->shouldSendStats(frame % NUM_FRAMES_PER_SEC)) { + data->sendAudioStreamStatsPackets(node); + } + + return true; + } + + return false; +} + +void AudioMixerSlave::writeMixPacket(std::unique_ptr& mixPacket, AudioMixerClientData* data, QByteArray& buffer) { + int mixPacketBytes = sizeof(quint16) + AudioConstants::MAX_CODEC_NAME_LENGTH_ON_WIRE + + AudioConstants::NETWORK_FRAME_BYTES_STEREO; + mixPacket = NLPacket::create(PacketType::MixedAudio, mixPacketBytes); + + // pack sequence number + quint16 sequence = data->getOutgoingSequenceNumber(); + mixPacket->writePrimitive(sequence); + + // write the codec + QString codecInPacket = data->getCodecName(); + mixPacket->writeString(codecInPacket); + + // pack mixed audio samples + mixPacket->write(buffer.constData(), buffer.size()); +} + +void AudioMixerSlave::writeSilentPacket(std::unique_ptr& mixPacket, AudioMixerClientData* data) { + int silentPacketBytes = sizeof(quint16) + sizeof(quint16) + AudioConstants::MAX_CODEC_NAME_LENGTH_ON_WIRE; + mixPacket = NLPacket::create(PacketType::SilentAudioFrame, silentPacketBytes); + + // pack sequence number + quint16 sequence = data->getOutgoingSequenceNumber(); + mixPacket->writePrimitive(sequence); + + // write the codec + QString codecInPacket = data->getCodecName(); + mixPacket->writeString(codecInPacket); + + // pack number of silent audio samples + quint16 numSilentSamples = AudioConstants::NETWORK_FRAME_SAMPLES_STEREO; + mixPacket->writePrimitive(numSilentSamples); +} + +void AudioMixerSlave::sendEnvironmentPacket(const SharedNodePointer& node) { + // Send stream properties + bool hasReverb = false; + float reverbTime, wetLevel; + + auto& reverbSettings = AudioMixer::getReverbSettings(); + auto& audioZones = AudioMixer::getAudioZones(); + + // find reverb properties + for (int i = 0; i < reverbSettings.size(); ++i) { + AudioMixerClientData* data = static_cast(node->getLinkedData()); + glm::vec3 streamPosition = data->getAvatarAudioStream()->getPosition(); + AABox box = audioZones[reverbSettings[i].zone]; + if (box.contains(streamPosition)) { + hasReverb = true; + reverbTime = reverbSettings[i].reverbTime; + wetLevel = reverbSettings[i].wetLevel; + + break; + } + } + + AudioMixerClientData* nodeData = static_cast(node->getLinkedData()); + AvatarAudioStream* stream = nodeData->getAvatarAudioStream(); + bool dataChanged = (stream->hasReverb() != hasReverb) || + (stream->hasReverb() && (stream->getRevebTime() != reverbTime || + stream->getWetLevel() != wetLevel)); + if (dataChanged) { + // Update stream + if (hasReverb) { + stream->setReverb(reverbTime, wetLevel); + } else { + stream->clearReverb(); + } + } + + // Send at change or every so often + float CHANCE_OF_SEND = 0.01f; + bool sendData = dataChanged || (randFloat() < CHANCE_OF_SEND); + + if (sendData) { + auto nodeList = DependencyManager::get(); + + unsigned char bitset = 0; + + int packetSize = sizeof(bitset); + + if (hasReverb) { + packetSize += sizeof(reverbTime) + sizeof(wetLevel); + } + + auto envPacket = NLPacket::create(PacketType::AudioEnvironment, packetSize); + + if (hasReverb) { + setAtBit(bitset, HAS_REVERB_BIT); + } + + envPacket->writePrimitive(bitset); + + if (hasReverb) { + envPacket->writePrimitive(reverbTime); + envPacket->writePrimitive(wetLevel); + } + nodeList->sendPacket(std::move(envPacket), *node); + } +} + +bool AudioMixerSlave::prepareMix(const SharedNodePointer& node) { + AvatarAudioStream* nodeAudioStream = static_cast(node->getLinkedData())->getAvatarAudioStream(); + AudioMixerClientData* listenerNodeData = static_cast(node->getLinkedData()); + + // zero out the client mix for this node + memset(_mixedSamples, 0, sizeof(_mixedSamples)); + + // loop through all other nodes that have sufficient audio to mix + + DependencyManager::get()->eachNode([&](const SharedNodePointer& otherNode){ + // make sure that we have audio data for this other node + // and that it isn't being ignored by our listening node + // and that it isn't ignoring our listening node + if (otherNode->getLinkedData() + && !node->isIgnoringNodeWithID(otherNode->getUUID()) && !otherNode->isIgnoringNodeWithID(node->getUUID())) { + AudioMixerClientData* otherNodeClientData = (AudioMixerClientData*) otherNode->getLinkedData(); + + // check to see if we're ignoring in radius + bool insideIgnoreRadius = false; + if (node->isIgnoreRadiusEnabled() || otherNode->isIgnoreRadiusEnabled()) { + AudioMixerClientData* otherData = reinterpret_cast(otherNode->getLinkedData()); + AudioMixerClientData* nodeData = reinterpret_cast(node->getLinkedData()); + float ignoreRadius = glm::min(node->getIgnoreRadius(), otherNode->getIgnoreRadius()); + if (glm::distance(nodeData->getPosition(), otherData->getPosition()) < ignoreRadius) { + insideIgnoreRadius = true; + } + } + + if (!insideIgnoreRadius) { + // enumerate the ARBs attached to the otherNode and add all that should be added to mix + auto streamsCopy = otherNodeClientData->getAudioStreams(); + for (auto& streamPair : streamsCopy) { + auto otherNodeStream = streamPair.second; + if (*otherNode != *node || otherNodeStream->shouldLoopbackForNode()) { + addStreamToMix(*listenerNodeData, otherNode->getUUID(), *nodeAudioStream, *otherNodeStream); + } + } + } + } + }); + + // use the per listner AudioLimiter to render the mixed data... + listenerNodeData->audioLimiter.render(_mixedSamples, _clampedSamples, AudioConstants::NETWORK_FRAME_SAMPLES_PER_CHANNEL); + + // check for silent audio after the peak limitor has converted the samples + bool hasAudio = false; + for (int i = 0; i < AudioConstants::NETWORK_FRAME_SAMPLES_STEREO; ++i) { + if (_clampedSamples[i] != 0) { + hasAudio = true; + break; + } + } + return hasAudio; +} + +void AudioMixerSlave::addStreamToMix(AudioMixerClientData& listenerNodeData, const QUuid& sourceNodeID, + const AvatarAudioStream& listeningNodeStream, const PositionalAudioStream& streamToAdd) { + // to reduce artifacts we calculate the gain and azimuth for every source for this listener + // even if we are not going to end up mixing in this source + + ++_totalMixes; + + // this ensures that the tail of any previously mixed audio or the first block of new audio sounds correct + + // check if this is a server echo of a source back to itself + bool isEcho = (&streamToAdd == &listeningNodeStream); + + glm::vec3 relativePosition = streamToAdd.getPosition() - listeningNodeStream.getPosition(); + + // figure out the distance between source and listener + float distance = glm::max(glm::length(relativePosition), EPSILON); + + // figure out the gain for this source at the listener + float gain = gainForSource(listeningNodeStream, streamToAdd, relativePosition, isEcho); + + // figure out the azimuth to this source at the listener + float azimuth = isEcho ? 0.0f : azimuthForSource(listeningNodeStream, listeningNodeStream, relativePosition); + + float repeatedFrameFadeFactor = 1.0f; + + static const int HRTF_DATASET_INDEX = 1; + + if (!streamToAdd.lastPopSucceeded()) { + bool forceSilentBlock = true; + + if (!streamToAdd.getLastPopOutput().isNull()) { + bool isInjector = dynamic_cast(&streamToAdd); + + // in an injector, just go silent - the injector has likely ended + // in other inputs (microphone, &c.), repeat with fade to avoid the harsh jump to silence + + // we'll repeat the last block until it has a block to mix + // and we'll gradually fade that repeated block into silence. + + // calculate its fade factor, which depends on how many times it's already been repeated. + repeatedFrameFadeFactor = calculateRepeatedFrameFadeFactor(streamToAdd.getConsecutiveNotMixedCount() - 1); + if (!isInjector && repeatedFrameFadeFactor > 0.0f) { + // apply the repeatedFrameFadeFactor to the gain + gain *= repeatedFrameFadeFactor; + + forceSilentBlock = false; + } + } + + if (forceSilentBlock) { + // we're deciding not to repeat either since we've already done it enough times or repetition with fade is disabled + // in this case we will call renderSilent with a forced silent block + // this ensures the correct tail from the previously mixed block and the correct spatialization of first block + // of any upcoming audio + + if (!streamToAdd.isStereo() && !isEcho) { + // get the existing listener-source HRTF object, or create a new one + auto& hrtf = listenerNodeData.hrtfForStream(sourceNodeID, streamToAdd.getStreamIdentifier()); + + // this is not done for stereo streams since they do not go through the HRTF + static int16_t silentMonoBlock[AudioConstants::NETWORK_FRAME_SAMPLES_PER_CHANNEL] = {}; + hrtf.renderSilent(silentMonoBlock, _mixedSamples, HRTF_DATASET_INDEX, azimuth, distance, gain, + AudioConstants::NETWORK_FRAME_SAMPLES_PER_CHANNEL); + + ++_hrtfSilentRenders;; + } + + return; + } + } + + // grab the stream from the ring buffer + AudioRingBuffer::ConstIterator streamPopOutput = streamToAdd.getLastPopOutput(); + + if (streamToAdd.isStereo() || isEcho) { + // this is a stereo source or server echo so we do not pass it through the HRTF + // simply apply our calculated gain to each sample + if (streamToAdd.isStereo()) { + for (int i = 0; i < AudioConstants::NETWORK_FRAME_SAMPLES_STEREO; ++i) { + _mixedSamples[i] += float(streamPopOutput[i] * gain / AudioConstants::MAX_SAMPLE_VALUE); + } + + ++_manualStereoMixes; + } else { + for (int i = 0; i < AudioConstants::NETWORK_FRAME_SAMPLES_STEREO; i += 2) { + auto monoSample = float(streamPopOutput[i / 2] * gain / AudioConstants::MAX_SAMPLE_VALUE); + _mixedSamples[i] += monoSample; + _mixedSamples[i + 1] += monoSample; + } + + ++_manualEchoMixes; + } + + return; + } + + // get the existing listener-source HRTF object, or create a new one + auto& hrtf = listenerNodeData.hrtfForStream(sourceNodeID, streamToAdd.getStreamIdentifier()); + + static int16_t streamBlock[AudioConstants::NETWORK_FRAME_SAMPLES_PER_CHANNEL]; + + streamPopOutput.readSamples(streamBlock, AudioConstants::NETWORK_FRAME_SAMPLES_PER_CHANNEL); + + // if the frame we're about to mix is silent, simply call render silent and move on + if (streamToAdd.getLastPopOutputLoudness() == 0.0f) { + // silent frame from source + + // we still need to call renderSilent via the HRTF for mono source + hrtf.renderSilent(streamBlock, _mixedSamples, HRTF_DATASET_INDEX, azimuth, distance, gain, + AudioConstants::NETWORK_FRAME_SAMPLES_PER_CHANNEL); + + ++_hrtfSilentRenders; + + return; + } + + float audibilityThreshold = AudioMixer::getMinimumAudibilityThreshold(); + if (audibilityThreshold > 0.0f && + streamToAdd.getLastPopOutputTrailingLoudness() / glm::length(relativePosition) <= audibilityThreshold) { + // the mixer is struggling so we're going to drop off some streams + + // we call renderSilent via the HRTF with the actual frame data and a gain of 0.0 + hrtf.renderSilent(streamBlock, _mixedSamples, HRTF_DATASET_INDEX, azimuth, distance, 0.0f, + AudioConstants::NETWORK_FRAME_SAMPLES_PER_CHANNEL); + + ++_hrtfStruggleRenders; + + return; + } + + ++_hrtfRenders; + + // mono stream, call the HRTF with our block and calculated azimuth and gain + hrtf.render(streamBlock, _mixedSamples, HRTF_DATASET_INDEX, azimuth, distance, gain, + AudioConstants::NETWORK_FRAME_SAMPLES_PER_CHANNEL); +} + +float AudioMixerSlave::gainForSource(const AvatarAudioStream& listeningNodeStream, const PositionalAudioStream& streamToAdd, + const glm::vec3& relativePosition, bool isEcho) { + float gain = 1.0f; + + float distanceBetween = glm::length(relativePosition); + + if (distanceBetween < EPSILON) { + distanceBetween = EPSILON; + } + + if (streamToAdd.getType() == PositionalAudioStream::Injector) { + gain *= reinterpret_cast(&streamToAdd)->getAttenuationRatio(); + } + + if (!isEcho && (streamToAdd.getType() == PositionalAudioStream::Microphone)) { + // source is another avatar, apply fixed off-axis attenuation to make them quieter as they turn away from listener + glm::vec3 rotatedListenerPosition = glm::inverse(streamToAdd.getOrientation()) * relativePosition; + + float angleOfDelivery = glm::angle(glm::vec3(0.0f, 0.0f, -1.0f), + glm::normalize(rotatedListenerPosition)); + + const float MAX_OFF_AXIS_ATTENUATION = 0.2f; + const float OFF_AXIS_ATTENUATION_FORMULA_STEP = (1 - MAX_OFF_AXIS_ATTENUATION) / 2.0f; + + float offAxisCoefficient = MAX_OFF_AXIS_ATTENUATION + + (OFF_AXIS_ATTENUATION_FORMULA_STEP * (angleOfDelivery / PI_OVER_TWO)); + + // multiply the current attenuation coefficient by the calculated off axis coefficient + gain *= offAxisCoefficient; + } + + float attenuationPerDoublingInDistance = AudioMixer::getAttenuationPerDoublingInDistance(); + auto& zoneSettings = AudioMixer::getZoneSettings(); + auto& audioZones = AudioMixer::getAudioZones(); + for (int i = 0; i < zoneSettings.length(); ++i) { + if (audioZones[zoneSettings[i].source].contains(streamToAdd.getPosition()) && + audioZones[zoneSettings[i].listener].contains(listeningNodeStream.getPosition())) { + attenuationPerDoublingInDistance = zoneSettings[i].coefficient; + break; + } + } + + if (distanceBetween >= ATTENUATION_BEGINS_AT_DISTANCE) { + + // translate the zone setting to gain per log2(distance) + float g = 1.0f - attenuationPerDoublingInDistance; + g = (g < EPSILON) ? EPSILON : g; + g = (g > 1.0f) ? 1.0f : g; + + // calculate the distance coefficient using the distance to this node + float distanceCoefficient = fastexp2(fastlog2(g) * fastlog2(distanceBetween/ATTENUATION_BEGINS_AT_DISTANCE)); + + // multiply the current attenuation coefficient by the distance coefficient + gain *= distanceCoefficient; + } + + return gain; +} + +float AudioMixerSlave::azimuthForSource(const AvatarAudioStream& listeningNodeStream, const PositionalAudioStream& streamToAdd, + const glm::vec3& relativePosition) { + glm::quat inverseOrientation = glm::inverse(listeningNodeStream.getOrientation()); + + // Compute sample delay for the two ears to create phase panning + glm::vec3 rotatedSourcePosition = inverseOrientation * relativePosition; + + // project the rotated source position vector onto the XZ plane + rotatedSourcePosition.y = 0.0f; + + static const float SOURCE_DISTANCE_THRESHOLD = 1e-30f; + + if (glm::length2(rotatedSourcePosition) > SOURCE_DISTANCE_THRESHOLD) { + // produce an oriented angle about the y-axis + return glm::orientedAngle(glm::vec3(0.0f, 0.0f, -1.0f), glm::normalize(rotatedSourcePosition), glm::vec3(0.0f, -1.0f, 0.0f)); + } else { + // there is no distance between listener and source - return no azimuth + return 0; + } +} diff --git a/assignment-client/src/audio/AudioMixer.h b/assignment-client/src/audio/AudioMixer.h index 78e9e85e13..3011dda62d 100644 --- a/assignment-client/src/audio/AudioMixer.h +++ b/assignment-client/src/audio/AudioMixer.h @@ -23,18 +23,77 @@ class AvatarAudioStream; class AudioHRTF; class AudioMixerClientData; +class AudioMixerSlave { +public: + // mix and broadcast non-ignored streams to the node + // returns true if a listener mix was broadcast for the node + bool mix(const SharedNodePointer& node, unsigned int frame); + + // reset statistics accumulated over mixes + void resetStats() { /* TODO */ }; + // get statistics accumulated over mixes + void getStats() { /* TODO */ }; + +private: + void writeMixPacket(std::unique_ptr& mixPacket, AudioMixerClientData* data, QByteArray& buffer); + void writeSilentPacket(std::unique_ptr& mixPacket, AudioMixerClientData* data); + + void sendEnvironmentPacket(const SharedNodePointer& node); + + // create mix, returns true if mix has audio + bool prepareMix(const SharedNodePointer& node); + // add a stream to the mix + void addStreamToMix(AudioMixerClientData& listenerData, const QUuid& streamerID, + const AvatarAudioStream& listenerStream, const PositionalAudioStream& streamer); + + float gainForSource(const AvatarAudioStream& listener, const PositionalAudioStream& streamer, + const glm::vec3& relativePosition, bool isEcho); + float azimuthForSource(const AvatarAudioStream& listener, const PositionalAudioStream& streamer, + const glm::vec3& relativePosition); + + // mixing buffers + float _mixedSamples[AudioConstants::NETWORK_FRAME_SAMPLES_STEREO]; + int16_t _clampedSamples[AudioConstants::NETWORK_FRAME_SAMPLES_STEREO]; + + // mixing statistics + unsigned int _sumListeners{ 0 }; + unsigned int _totalMixes{ 0 }; + unsigned int _hrtfRenders{ 0 }; + unsigned int _hrtfStruggleRenders{ 0 }; + unsigned int _hrtfSilentRenders{ 0 }; + unsigned int _manualStereoMixes{ 0 }; + unsigned int _manualEchoMixes{ 0 }; +}; + /// Handles assignments of type AudioMixer - mixing streams of audio and re-distributing to various clients. class AudioMixer : public ThreadedAssignment { Q_OBJECT public: AudioMixer(ReceivedMessage& message); + struct ZoneSettings { + QString source; + QString listener; + float coefficient; + }; + struct ReverbSettings { + QString zone; + float reverbTime; + float wetLevel; + }; + + static int getStaticJitterFrames() { return _numStaticJitterFrames; } + static bool shouldMute(float quietestFrame) { return quietestFrame > _noiseMutingThreshold; } + static float getAttenuationPerDoublingInDistance() { return _attenuationPerDoublingInDistance; } + static float getMinimumAudibilityThreshold() { return _performanceThrottlingRatio > 0.0f ? _minAudibilityThreshold : 0.0f; } + static const QHash& getAudioZones() { return _audioZones; } + static const QVector& getZoneSettings() { return _zoneSettings; } + static const QVector& getReverbSettings() { return _zoneReverbSettings; } + public slots: void run() override; void sendStatsPacket() override; - static int getStaticJitterFrames() { return _numStaticJitterFrames; } - private slots: // packet handlers void handleNodeAudioPacket(QSharedPointer packet, SharedNodePointer sendingNode); @@ -56,42 +115,13 @@ private: // pop a frame from any streams on the node // returns the number of available streams int prepareFrame(const SharedNodePointer& node, unsigned int frame); - // mix and broadcast non-ignored streams to the node - // returns true if a listener mix was broadcast for the node - bool mixFrame(const SharedNodePointer& node, unsigned int frame); AudioMixerClientData* getOrCreateClientData(Node* node); - /// adds one stream to the mix for a listening node - void addStreamToMixForListeningNodeWithStream(AudioMixerClientData& listenerNodeData, - const PositionalAudioStream& streamToAdd, - const QUuid& sourceNodeID, - const AvatarAudioStream& listeningNodeStream); - - float gainForSource(const PositionalAudioStream& streamToAdd, const AvatarAudioStream& listeningNodeStream, - const glm::vec3& relativePosition, bool isEcho); - float azimuthForSource(const PositionalAudioStream& streamToAdd, const AvatarAudioStream& listeningNodeStream, - const glm::vec3& relativePosition); - - /// prepares and sends a mix to one Node - bool prepareMixForListeningNode(Node* node); - - /// Send Audio Environment packet for a single node - void sendAudioEnvironmentPacket(SharedNodePointer node); - - void perSecondActions(); - QString percentageForMixStats(int counter); - bool shouldMute(float quietestFrame); - void parseSettingsObject(const QJsonObject& settingsObject); - float _trailingSleepRatio; - float _minAudibilityThreshold; - float _performanceThrottlingRatio; - float _attenuationPerDoublingInDistance; - float _noiseMutingThreshold; int _numStatFrames { 0 }; int _sumStreams { 0 }; int _sumListeners { 0 }; @@ -104,24 +134,17 @@ private: QString _codecPreferenceOrder; - float _mixedSamples[AudioConstants::NETWORK_FRAME_SAMPLES_STEREO]; - int16_t _clampedSamples[AudioConstants::NETWORK_FRAME_SAMPLES_STEREO]; - - QHash _audioZones; - struct ZonesSettings { - QString source; - QString listener; - float coefficient; - }; - QVector _zonesSettings; - struct ReverbSettings { - QString zone; - float reverbTime; - float wetLevel; - }; - QVector _zoneReverbSettings; + AudioMixerSlave slave; static int _numStaticJitterFrames; // -1 denotes dynamic jitter buffering + static float _noiseMutingThreshold; + static float _attenuationPerDoublingInDistance; + static float _trailingSleepRatio; + static float _performanceThrottlingRatio; + static float _minAudibilityThreshold; + static QHash _audioZones; + static QVector _zoneSettings; + static QVector _zoneReverbSettings; }; #endif // hifi_AudioMixer_h From 2e619b230d15290bc154b144bc92ad0b1164ab2c Mon Sep 17 00:00:00 2001 From: Zach Pomerantz Date: Tue, 22 Nov 2016 17:25:00 -0500 Subject: [PATCH 06/48] add AudioMixerStats --- assignment-client/src/audio/AudioMixer.cpp | 91 ++++++++++++---------- assignment-client/src/audio/AudioMixer.h | 43 +++++----- 2 files changed, 70 insertions(+), 64 deletions(-) diff --git a/assignment-client/src/audio/AudioMixer.cpp b/assignment-client/src/audio/AudioMixer.cpp index c1402529fa..d3dcacfd4f 100644 --- a/assignment-client/src/audio/AudioMixer.cpp +++ b/assignment-client/src/audio/AudioMixer.cpp @@ -329,8 +329,8 @@ void AudioMixer::removeHRTFsForFinishedInjector(const QUuid& streamID) { } QString AudioMixer::percentageForMixStats(int counter) { - if (_totalMixes > 0) { - float mixPercentage = (float(counter) / _totalMixes) * 100.0f; + if (_stats.totalMixes > 0) { + float mixPercentage = (float(counter) / _stats.totalMixes) * 100.0f; return QString::number(mixPercentage, 'f', 2); } else { return QString("0.0"); @@ -348,30 +348,23 @@ void AudioMixer::sendStatsPacket() { statsObject["trailing_sleep_percentage"] = _trailingSleepRatio * 100.0f; statsObject["performance_throttling_ratio"] = _performanceThrottlingRatio; - statsObject["avg_streams_per_frame"] = (float)_sumStreams / (float)_numStatFrames; - statsObject["avg_listeners_per_frame"] = (float)_sumListeners / (float)_numStatFrames; + statsObject["avg_streams_per_frame"] = (float)_stats.sumStreams / (float)_numStatFrames; + statsObject["avg_listeners_per_frame"] = (float)_stats.sumListeners / (float)_numStatFrames; QJsonObject mixStats; - mixStats["%_hrtf_mixes"] = percentageForMixStats(_hrtfRenders); - mixStats["%_hrtf_silent_mixes"] = percentageForMixStats(_hrtfSilentRenders); - mixStats["%_hrtf_struggle_mixes"] = percentageForMixStats(_hrtfStruggleRenders); - mixStats["%_manual_stereo_mixes"] = percentageForMixStats(_manualStereoMixes); - mixStats["%_manual_echo_mixes"] = percentageForMixStats(_manualEchoMixes); + mixStats["%_hrtf_mixes"] = percentageForMixStats(_stats.hrtfRenders); + mixStats["%_hrtf_silent_mixes"] = percentageForMixStats(_stats.hrtfSilentRenders); + mixStats["%_hrtf_struggle_mixes"] = percentageForMixStats(_stats.hrtfStruggleRenders); + mixStats["%_manual_stereo_mixes"] = percentageForMixStats(_stats.manualStereoMixes); + mixStats["%_manual_echo_mixes"] = percentageForMixStats(_stats.manualEchoMixes); - mixStats["total_mixes"] = _totalMixes; - mixStats["avg_mixes_per_block"] = _totalMixes / _numStatFrames; + mixStats["total_mixes"] = _stats.totalMixes; + mixStats["avg_mixes_per_block"] = _stats.totalMixes / _numStatFrames; statsObject["mix_stats"] = mixStats; - _sumStreams = 0; - _sumListeners = 0; - _hrtfRenders = 0; - _hrtfSilentRenders = 0; - _hrtfStruggleRenders = 0; - _manualStereoMixes = 0; - _manualEchoMixes = 0; - _totalMixes = 0; _numStatFrames = 0; + _stats.reset(); // add stats for each listerner auto nodeList = DependencyManager::get(); @@ -447,18 +440,12 @@ void AudioMixer::start() { while (!_isFinished) { manageLoad(frameTimestamp, framesSinceManagement); - slave.resetStats(); + slave.stats.reset(); - nodeList->eachNode([&](const SharedNodePointer& node) { - _sumStreams += prepareFrame(node, frame); - }); - nodeList->eachNode([&](const SharedNodePointer& node) { - if (slave.mix(node, frame)) { - ++_sumListeners; - } - }); + nodeList->eachNode([&](const SharedNodePointer& node) { _stats.sumStreams += prepareFrame(node, frame); }); + nodeList->eachNode([&](const SharedNodePointer& node) { slave.mix(node, frame); }); - slave.getStats(); + _stats.accumulate(slave.stats); ++frame; ++_numStatFrames; @@ -754,15 +741,15 @@ void AudioMixer::parseSettingsObject(const QJsonObject &settingsObject) { } } -bool AudioMixerSlave::mix(const SharedNodePointer& node, unsigned int frame) { +void AudioMixerSlave::mix(const SharedNodePointer& node, unsigned int frame) { AudioMixerClientData* data = (AudioMixerClientData*)node->getLinkedData(); if (data == nullptr) { - return false; + return; } auto avatarStream = data->getAvatarAudioStream(); if (avatarStream == nullptr) { - return false; + return; } auto nodeList = DependencyManager::get(); @@ -814,10 +801,8 @@ bool AudioMixerSlave::mix(const SharedNodePointer& node, unsigned int frame) { data->sendAudioStreamStatsPackets(node); } - return true; + ++stats.sumListeners; } - - return false; } void AudioMixerSlave::writeMixPacket(std::unique_ptr& mixPacket, AudioMixerClientData* data, QByteArray& buffer) { @@ -981,7 +966,7 @@ void AudioMixerSlave::addStreamToMix(AudioMixerClientData& listenerNodeData, con // to reduce artifacts we calculate the gain and azimuth for every source for this listener // even if we are not going to end up mixing in this source - ++_totalMixes; + ++stats.totalMixes; // this ensures that the tail of any previously mixed audio or the first block of new audio sounds correct @@ -1040,7 +1025,7 @@ void AudioMixerSlave::addStreamToMix(AudioMixerClientData& listenerNodeData, con hrtf.renderSilent(silentMonoBlock, _mixedSamples, HRTF_DATASET_INDEX, azimuth, distance, gain, AudioConstants::NETWORK_FRAME_SAMPLES_PER_CHANNEL); - ++_hrtfSilentRenders;; + ++stats.hrtfSilentRenders; } return; @@ -1058,7 +1043,7 @@ void AudioMixerSlave::addStreamToMix(AudioMixerClientData& listenerNodeData, con _mixedSamples[i] += float(streamPopOutput[i] * gain / AudioConstants::MAX_SAMPLE_VALUE); } - ++_manualStereoMixes; + ++stats.manualStereoMixes; } else { for (int i = 0; i < AudioConstants::NETWORK_FRAME_SAMPLES_STEREO; i += 2) { auto monoSample = float(streamPopOutput[i / 2] * gain / AudioConstants::MAX_SAMPLE_VALUE); @@ -1066,7 +1051,7 @@ void AudioMixerSlave::addStreamToMix(AudioMixerClientData& listenerNodeData, con _mixedSamples[i + 1] += monoSample; } - ++_manualEchoMixes; + ++stats.manualEchoMixes; } return; @@ -1087,7 +1072,7 @@ void AudioMixerSlave::addStreamToMix(AudioMixerClientData& listenerNodeData, con hrtf.renderSilent(streamBlock, _mixedSamples, HRTF_DATASET_INDEX, azimuth, distance, gain, AudioConstants::NETWORK_FRAME_SAMPLES_PER_CHANNEL); - ++_hrtfSilentRenders; + ++stats.hrtfSilentRenders; return; } @@ -1101,12 +1086,12 @@ void AudioMixerSlave::addStreamToMix(AudioMixerClientData& listenerNodeData, con hrtf.renderSilent(streamBlock, _mixedSamples, HRTF_DATASET_INDEX, azimuth, distance, 0.0f, AudioConstants::NETWORK_FRAME_SAMPLES_PER_CHANNEL); - ++_hrtfStruggleRenders; + ++stats.hrtfStruggleRenders; return; } - ++_hrtfRenders; + ++stats.hrtfRenders; // mono stream, call the HRTF with our block and calculated azimuth and gain hrtf.render(streamBlock, _mixedSamples, HRTF_DATASET_INDEX, azimuth, distance, gain, @@ -1192,3 +1177,25 @@ float AudioMixerSlave::azimuthForSource(const AvatarAudioStream& listeningNodeSt return 0; } } + +void AudioMixerStats::reset() { + sumStreams = 0; + sumListeners = 0; + totalMixes = 0; + hrtfRenders = 0; + hrtfSilentRenders = 0; + hrtfStruggleRenders = 0; + manualStereoMixes = 0; + manualEchoMixes = 0; +} + +void AudioMixerStats::accumulate(const AudioMixerStats& otherStats) { + sumStreams += otherStats.sumStreams; + sumListeners += otherStats.sumListeners; + totalMixes += otherStats.totalMixes; + hrtfRenders += otherStats.hrtfRenders; + hrtfSilentRenders += otherStats.hrtfSilentRenders; + hrtfStruggleRenders += otherStats.hrtfStruggleRenders; + manualStereoMixes += otherStats.manualStereoMixes; + manualEchoMixes += otherStats.manualEchoMixes; +} diff --git a/assignment-client/src/audio/AudioMixer.h b/assignment-client/src/audio/AudioMixer.h index 3011dda62d..e4fda2fddb 100644 --- a/assignment-client/src/audio/AudioMixer.h +++ b/assignment-client/src/audio/AudioMixer.h @@ -23,16 +23,31 @@ class AvatarAudioStream; class AudioHRTF; class AudioMixerClientData; + +struct AudioMixerStats { + int sumStreams { 0 }; + int sumListeners { 0 }; + + int totalMixes { 0 }; + + int hrtfRenders { 0 }; + int hrtfSilentRenders { 0 }; + int hrtfStruggleRenders { 0 }; + + int manualStereoMixes { 0 }; + int manualEchoMixes { 0 }; + + void reset(); + void accumulate(const AudioMixerStats& otherStats); +}; + class AudioMixerSlave { public: // mix and broadcast non-ignored streams to the node // returns true if a listener mix was broadcast for the node - bool mix(const SharedNodePointer& node, unsigned int frame); + void mix(const SharedNodePointer& node, unsigned int frame); - // reset statistics accumulated over mixes - void resetStats() { /* TODO */ }; - // get statistics accumulated over mixes - void getStats() { /* TODO */ }; + AudioMixerStats stats; private: void writeMixPacket(std::unique_ptr& mixPacket, AudioMixerClientData* data, QByteArray& buffer); @@ -54,15 +69,6 @@ private: // mixing buffers float _mixedSamples[AudioConstants::NETWORK_FRAME_SAMPLES_STEREO]; int16_t _clampedSamples[AudioConstants::NETWORK_FRAME_SAMPLES_STEREO]; - - // mixing statistics - unsigned int _sumListeners{ 0 }; - unsigned int _totalMixes{ 0 }; - unsigned int _hrtfRenders{ 0 }; - unsigned int _hrtfStruggleRenders{ 0 }; - unsigned int _hrtfSilentRenders{ 0 }; - unsigned int _manualStereoMixes{ 0 }; - unsigned int _manualEchoMixes{ 0 }; }; /// Handles assignments of type AudioMixer - mixing streams of audio and re-distributing to various clients. @@ -123,14 +129,7 @@ private: void parseSettingsObject(const QJsonObject& settingsObject); int _numStatFrames { 0 }; - int _sumStreams { 0 }; - int _sumListeners { 0 }; - int _hrtfRenders { 0 }; - int _hrtfSilentRenders { 0 }; - int _hrtfStruggleRenders { 0 }; - int _manualStereoMixes { 0 }; - int _manualEchoMixes { 0 }; - int _totalMixes { 0 }; + AudioMixerStats _stats; QString _codecPreferenceOrder; From bf137cd589497316fb2e7287a9613af623e652f4 Mon Sep 17 00:00:00 2001 From: Zach Pomerantz Date: Tue, 22 Nov 2016 17:39:35 -0500 Subject: [PATCH 07/48] break AudioMixer slave/stats into own files --- assignment-client/src/audio/AudioMixer.cpp | 544 ------------------ assignment-client/src/audio/AudioMixer.h | 51 +- .../src/audio/AudioMixerSlave.cpp | 524 +++++++++++++++++ assignment-client/src/audio/AudioMixerSlave.h | 58 ++ .../src/audio/AudioMixerStats.cpp | 34 ++ assignment-client/src/audio/AudioMixerStats.h | 32 ++ 6 files changed, 651 insertions(+), 592 deletions(-) create mode 100644 assignment-client/src/audio/AudioMixerSlave.cpp create mode 100644 assignment-client/src/audio/AudioMixerSlave.h create mode 100644 assignment-client/src/audio/AudioMixerStats.cpp create mode 100644 assignment-client/src/audio/AudioMixerStats.h diff --git a/assignment-client/src/audio/AudioMixer.cpp b/assignment-client/src/audio/AudioMixer.cpp index d3dcacfd4f..5a6014aee9 100644 --- a/assignment-client/src/audio/AudioMixer.cpp +++ b/assignment-client/src/audio/AudioMixer.cpp @@ -9,38 +9,12 @@ // See the accompanying file LICENSE or http://www.apache.org/licenses/LICENSE-2.0.html // -#include -#include -#include -#include -#include -#include -#include -#include -#include -#include #include -#ifdef _WIN32 -#include -#else -#include -#include -#include -#endif //_WIN32 - -#include -#include -#include - -#include #include #include #include #include -#include -#include -#include #include #include @@ -97,65 +71,6 @@ AudioMixer::AudioMixer(ReceivedMessage& message) : connect(nodeList.data(), &NodeList::nodeKilled, this, &AudioMixer::handleNodeKilled); } -const float ATTENUATION_BEGINS_AT_DISTANCE = 1.0f; - -const int IEEE754_MANT_BITS = 23; -const int IEEE754_EXPN_BIAS = 127; - -// -// for x > 0.0f, returns log2(x) -// for x <= 0.0f, returns large negative value -// -// abs |error| < 8e-3, smooth (exact for x=2^N) for NPOLY=3 -// abs |error| < 2e-4, smooth (exact for x=2^N) for NPOLY=5 -// rel |error| < 0.4 from precision loss very close to 1.0f -// -static inline float fastlog2(float x) { - - union { float f; int32_t i; } mant, bits = { x }; - - // split into mantissa and exponent - mant.i = (bits.i & ((1 << IEEE754_MANT_BITS) - 1)) | (IEEE754_EXPN_BIAS << IEEE754_MANT_BITS); - int32_t expn = (bits.i >> IEEE754_MANT_BITS) - IEEE754_EXPN_BIAS; - - mant.f -= 1.0f; - - // polynomial for log2(1+x) over x=[0,1] - //x = (-0.346555386f * mant.f + 1.346555386f) * mant.f; - x = (((-0.0821307180f * mant.f + 0.321188984f) * mant.f - 0.677784014f) * mant.f + 1.43872575f) * mant.f; - - return x + expn; -} - -// -// for -126 <= x < 128, returns exp2(x) -// -// rel |error| < 3e-3, smooth (exact for x=N) for NPOLY=3 -// rel |error| < 9e-6, smooth (exact for x=N) for NPOLY=5 -// -static inline float fastexp2(float x) { - - union { float f; int32_t i; } xi; - - // bias such that x > 0 - x += IEEE754_EXPN_BIAS; - //x = MAX(x, 1.0f); - //x = MIN(x, 254.9999f); - - // split into integer and fraction - xi.i = (int32_t)x; - x -= xi.i; - - // construct exp2(xi) as a float - xi.i <<= IEEE754_MANT_BITS; - - // polynomial for exp2(x) over x=[0,1] - //x = (0.339766028f * x + 0.660233972f) * x + 1.0f; - x = (((0.0135557472f * x + 0.0520323690f) * x + 0.241379763f) * x + 0.693032121f) * x + 1.0f; - - return x * xi.f; -} - void AudioMixer::handleNodeAudioPacket(QSharedPointer message, SharedNodePointer sendingNode) { getOrCreateClientData(sendingNode.data()); DependencyManager::get()->updateNodeWithDataFromPacket(message, sendingNode); @@ -740,462 +655,3 @@ void AudioMixer::parseSettingsObject(const QJsonObject &settingsObject) { } } } - -void AudioMixerSlave::mix(const SharedNodePointer& node, unsigned int frame) { - AudioMixerClientData* data = (AudioMixerClientData*)node->getLinkedData(); - if (data == nullptr) { - return; - } - - auto avatarStream = data->getAvatarAudioStream(); - if (avatarStream == nullptr) { - return; - } - - auto nodeList = DependencyManager::get(); - - // mute the avatar, if necessary - if (AudioMixer::shouldMute(avatarStream->getQuietestFrameLoudness()) || data->shouldMuteClient()) { - auto mutePacket = NLPacket::create(PacketType::NoisyMute, 0); - nodeList->sendPacket(std::move(mutePacket), *node); - - // probably now we just reset the flag, once should do it (?) - data->setShouldMuteClient(false); - } - - // generate and send audio packets - if (node->getType() == NodeType::Agent && node->getActiveSocket()) { - - // mix streams - bool mixHasAudio = prepareMix(node); - - // write the packet - std::unique_ptr mixPacket; - if (mixHasAudio || data->shouldFlushEncoder()) { - // encode the audio - QByteArray encodedBuffer; - if (mixHasAudio) { - QByteArray decodedBuffer(reinterpret_cast(_clampedSamples), AudioConstants::NETWORK_FRAME_BYTES_STEREO); - data->encode(decodedBuffer, encodedBuffer); - } else { - // time to flush, which resets the shouldFlush until next time we encode something - data->encodeFrameOfZeros(encodedBuffer); - } - - // write it to a packet - writeMixPacket(mixPacket, data, encodedBuffer); - } else { - writeSilentPacket(mixPacket, data); - } - - // send audio environment packet - sendEnvironmentPacket(node); - - // send mixed audio packet - nodeList->sendPacket(std::move(mixPacket), *node); - data->incrementOutgoingMixedAudioSequenceNumber(); - - // send an audio stream stats packet to the client approximately every second - static const unsigned int NUM_FRAMES_PER_SEC = (int) ceil(AudioConstants::NETWORK_FRAMES_PER_SEC); - if (data->shouldSendStats(frame % NUM_FRAMES_PER_SEC)) { - data->sendAudioStreamStatsPackets(node); - } - - ++stats.sumListeners; - } -} - -void AudioMixerSlave::writeMixPacket(std::unique_ptr& mixPacket, AudioMixerClientData* data, QByteArray& buffer) { - int mixPacketBytes = sizeof(quint16) + AudioConstants::MAX_CODEC_NAME_LENGTH_ON_WIRE - + AudioConstants::NETWORK_FRAME_BYTES_STEREO; - mixPacket = NLPacket::create(PacketType::MixedAudio, mixPacketBytes); - - // pack sequence number - quint16 sequence = data->getOutgoingSequenceNumber(); - mixPacket->writePrimitive(sequence); - - // write the codec - QString codecInPacket = data->getCodecName(); - mixPacket->writeString(codecInPacket); - - // pack mixed audio samples - mixPacket->write(buffer.constData(), buffer.size()); -} - -void AudioMixerSlave::writeSilentPacket(std::unique_ptr& mixPacket, AudioMixerClientData* data) { - int silentPacketBytes = sizeof(quint16) + sizeof(quint16) + AudioConstants::MAX_CODEC_NAME_LENGTH_ON_WIRE; - mixPacket = NLPacket::create(PacketType::SilentAudioFrame, silentPacketBytes); - - // pack sequence number - quint16 sequence = data->getOutgoingSequenceNumber(); - mixPacket->writePrimitive(sequence); - - // write the codec - QString codecInPacket = data->getCodecName(); - mixPacket->writeString(codecInPacket); - - // pack number of silent audio samples - quint16 numSilentSamples = AudioConstants::NETWORK_FRAME_SAMPLES_STEREO; - mixPacket->writePrimitive(numSilentSamples); -} - -void AudioMixerSlave::sendEnvironmentPacket(const SharedNodePointer& node) { - // Send stream properties - bool hasReverb = false; - float reverbTime, wetLevel; - - auto& reverbSettings = AudioMixer::getReverbSettings(); - auto& audioZones = AudioMixer::getAudioZones(); - - // find reverb properties - for (int i = 0; i < reverbSettings.size(); ++i) { - AudioMixerClientData* data = static_cast(node->getLinkedData()); - glm::vec3 streamPosition = data->getAvatarAudioStream()->getPosition(); - AABox box = audioZones[reverbSettings[i].zone]; - if (box.contains(streamPosition)) { - hasReverb = true; - reverbTime = reverbSettings[i].reverbTime; - wetLevel = reverbSettings[i].wetLevel; - - break; - } - } - - AudioMixerClientData* nodeData = static_cast(node->getLinkedData()); - AvatarAudioStream* stream = nodeData->getAvatarAudioStream(); - bool dataChanged = (stream->hasReverb() != hasReverb) || - (stream->hasReverb() && (stream->getRevebTime() != reverbTime || - stream->getWetLevel() != wetLevel)); - if (dataChanged) { - // Update stream - if (hasReverb) { - stream->setReverb(reverbTime, wetLevel); - } else { - stream->clearReverb(); - } - } - - // Send at change or every so often - float CHANCE_OF_SEND = 0.01f; - bool sendData = dataChanged || (randFloat() < CHANCE_OF_SEND); - - if (sendData) { - auto nodeList = DependencyManager::get(); - - unsigned char bitset = 0; - - int packetSize = sizeof(bitset); - - if (hasReverb) { - packetSize += sizeof(reverbTime) + sizeof(wetLevel); - } - - auto envPacket = NLPacket::create(PacketType::AudioEnvironment, packetSize); - - if (hasReverb) { - setAtBit(bitset, HAS_REVERB_BIT); - } - - envPacket->writePrimitive(bitset); - - if (hasReverb) { - envPacket->writePrimitive(reverbTime); - envPacket->writePrimitive(wetLevel); - } - nodeList->sendPacket(std::move(envPacket), *node); - } -} - -bool AudioMixerSlave::prepareMix(const SharedNodePointer& node) { - AvatarAudioStream* nodeAudioStream = static_cast(node->getLinkedData())->getAvatarAudioStream(); - AudioMixerClientData* listenerNodeData = static_cast(node->getLinkedData()); - - // zero out the client mix for this node - memset(_mixedSamples, 0, sizeof(_mixedSamples)); - - // loop through all other nodes that have sufficient audio to mix - - DependencyManager::get()->eachNode([&](const SharedNodePointer& otherNode){ - // make sure that we have audio data for this other node - // and that it isn't being ignored by our listening node - // and that it isn't ignoring our listening node - if (otherNode->getLinkedData() - && !node->isIgnoringNodeWithID(otherNode->getUUID()) && !otherNode->isIgnoringNodeWithID(node->getUUID())) { - AudioMixerClientData* otherNodeClientData = (AudioMixerClientData*) otherNode->getLinkedData(); - - // check to see if we're ignoring in radius - bool insideIgnoreRadius = false; - if (node->isIgnoreRadiusEnabled() || otherNode->isIgnoreRadiusEnabled()) { - AudioMixerClientData* otherData = reinterpret_cast(otherNode->getLinkedData()); - AudioMixerClientData* nodeData = reinterpret_cast(node->getLinkedData()); - float ignoreRadius = glm::min(node->getIgnoreRadius(), otherNode->getIgnoreRadius()); - if (glm::distance(nodeData->getPosition(), otherData->getPosition()) < ignoreRadius) { - insideIgnoreRadius = true; - } - } - - if (!insideIgnoreRadius) { - // enumerate the ARBs attached to the otherNode and add all that should be added to mix - auto streamsCopy = otherNodeClientData->getAudioStreams(); - for (auto& streamPair : streamsCopy) { - auto otherNodeStream = streamPair.second; - if (*otherNode != *node || otherNodeStream->shouldLoopbackForNode()) { - addStreamToMix(*listenerNodeData, otherNode->getUUID(), *nodeAudioStream, *otherNodeStream); - } - } - } - } - }); - - // use the per listner AudioLimiter to render the mixed data... - listenerNodeData->audioLimiter.render(_mixedSamples, _clampedSamples, AudioConstants::NETWORK_FRAME_SAMPLES_PER_CHANNEL); - - // check for silent audio after the peak limitor has converted the samples - bool hasAudio = false; - for (int i = 0; i < AudioConstants::NETWORK_FRAME_SAMPLES_STEREO; ++i) { - if (_clampedSamples[i] != 0) { - hasAudio = true; - break; - } - } - return hasAudio; -} - -void AudioMixerSlave::addStreamToMix(AudioMixerClientData& listenerNodeData, const QUuid& sourceNodeID, - const AvatarAudioStream& listeningNodeStream, const PositionalAudioStream& streamToAdd) { - // to reduce artifacts we calculate the gain and azimuth for every source for this listener - // even if we are not going to end up mixing in this source - - ++stats.totalMixes; - - // this ensures that the tail of any previously mixed audio or the first block of new audio sounds correct - - // check if this is a server echo of a source back to itself - bool isEcho = (&streamToAdd == &listeningNodeStream); - - glm::vec3 relativePosition = streamToAdd.getPosition() - listeningNodeStream.getPosition(); - - // figure out the distance between source and listener - float distance = glm::max(glm::length(relativePosition), EPSILON); - - // figure out the gain for this source at the listener - float gain = gainForSource(listeningNodeStream, streamToAdd, relativePosition, isEcho); - - // figure out the azimuth to this source at the listener - float azimuth = isEcho ? 0.0f : azimuthForSource(listeningNodeStream, listeningNodeStream, relativePosition); - - float repeatedFrameFadeFactor = 1.0f; - - static const int HRTF_DATASET_INDEX = 1; - - if (!streamToAdd.lastPopSucceeded()) { - bool forceSilentBlock = true; - - if (!streamToAdd.getLastPopOutput().isNull()) { - bool isInjector = dynamic_cast(&streamToAdd); - - // in an injector, just go silent - the injector has likely ended - // in other inputs (microphone, &c.), repeat with fade to avoid the harsh jump to silence - - // we'll repeat the last block until it has a block to mix - // and we'll gradually fade that repeated block into silence. - - // calculate its fade factor, which depends on how many times it's already been repeated. - repeatedFrameFadeFactor = calculateRepeatedFrameFadeFactor(streamToAdd.getConsecutiveNotMixedCount() - 1); - if (!isInjector && repeatedFrameFadeFactor > 0.0f) { - // apply the repeatedFrameFadeFactor to the gain - gain *= repeatedFrameFadeFactor; - - forceSilentBlock = false; - } - } - - if (forceSilentBlock) { - // we're deciding not to repeat either since we've already done it enough times or repetition with fade is disabled - // in this case we will call renderSilent with a forced silent block - // this ensures the correct tail from the previously mixed block and the correct spatialization of first block - // of any upcoming audio - - if (!streamToAdd.isStereo() && !isEcho) { - // get the existing listener-source HRTF object, or create a new one - auto& hrtf = listenerNodeData.hrtfForStream(sourceNodeID, streamToAdd.getStreamIdentifier()); - - // this is not done for stereo streams since they do not go through the HRTF - static int16_t silentMonoBlock[AudioConstants::NETWORK_FRAME_SAMPLES_PER_CHANNEL] = {}; - hrtf.renderSilent(silentMonoBlock, _mixedSamples, HRTF_DATASET_INDEX, azimuth, distance, gain, - AudioConstants::NETWORK_FRAME_SAMPLES_PER_CHANNEL); - - ++stats.hrtfSilentRenders; - } - - return; - } - } - - // grab the stream from the ring buffer - AudioRingBuffer::ConstIterator streamPopOutput = streamToAdd.getLastPopOutput(); - - if (streamToAdd.isStereo() || isEcho) { - // this is a stereo source or server echo so we do not pass it through the HRTF - // simply apply our calculated gain to each sample - if (streamToAdd.isStereo()) { - for (int i = 0; i < AudioConstants::NETWORK_FRAME_SAMPLES_STEREO; ++i) { - _mixedSamples[i] += float(streamPopOutput[i] * gain / AudioConstants::MAX_SAMPLE_VALUE); - } - - ++stats.manualStereoMixes; - } else { - for (int i = 0; i < AudioConstants::NETWORK_FRAME_SAMPLES_STEREO; i += 2) { - auto monoSample = float(streamPopOutput[i / 2] * gain / AudioConstants::MAX_SAMPLE_VALUE); - _mixedSamples[i] += monoSample; - _mixedSamples[i + 1] += monoSample; - } - - ++stats.manualEchoMixes; - } - - return; - } - - // get the existing listener-source HRTF object, or create a new one - auto& hrtf = listenerNodeData.hrtfForStream(sourceNodeID, streamToAdd.getStreamIdentifier()); - - static int16_t streamBlock[AudioConstants::NETWORK_FRAME_SAMPLES_PER_CHANNEL]; - - streamPopOutput.readSamples(streamBlock, AudioConstants::NETWORK_FRAME_SAMPLES_PER_CHANNEL); - - // if the frame we're about to mix is silent, simply call render silent and move on - if (streamToAdd.getLastPopOutputLoudness() == 0.0f) { - // silent frame from source - - // we still need to call renderSilent via the HRTF for mono source - hrtf.renderSilent(streamBlock, _mixedSamples, HRTF_DATASET_INDEX, azimuth, distance, gain, - AudioConstants::NETWORK_FRAME_SAMPLES_PER_CHANNEL); - - ++stats.hrtfSilentRenders; - - return; - } - - float audibilityThreshold = AudioMixer::getMinimumAudibilityThreshold(); - if (audibilityThreshold > 0.0f && - streamToAdd.getLastPopOutputTrailingLoudness() / glm::length(relativePosition) <= audibilityThreshold) { - // the mixer is struggling so we're going to drop off some streams - - // we call renderSilent via the HRTF with the actual frame data and a gain of 0.0 - hrtf.renderSilent(streamBlock, _mixedSamples, HRTF_DATASET_INDEX, azimuth, distance, 0.0f, - AudioConstants::NETWORK_FRAME_SAMPLES_PER_CHANNEL); - - ++stats.hrtfStruggleRenders; - - return; - } - - ++stats.hrtfRenders; - - // mono stream, call the HRTF with our block and calculated azimuth and gain - hrtf.render(streamBlock, _mixedSamples, HRTF_DATASET_INDEX, azimuth, distance, gain, - AudioConstants::NETWORK_FRAME_SAMPLES_PER_CHANNEL); -} - -float AudioMixerSlave::gainForSource(const AvatarAudioStream& listeningNodeStream, const PositionalAudioStream& streamToAdd, - const glm::vec3& relativePosition, bool isEcho) { - float gain = 1.0f; - - float distanceBetween = glm::length(relativePosition); - - if (distanceBetween < EPSILON) { - distanceBetween = EPSILON; - } - - if (streamToAdd.getType() == PositionalAudioStream::Injector) { - gain *= reinterpret_cast(&streamToAdd)->getAttenuationRatio(); - } - - if (!isEcho && (streamToAdd.getType() == PositionalAudioStream::Microphone)) { - // source is another avatar, apply fixed off-axis attenuation to make them quieter as they turn away from listener - glm::vec3 rotatedListenerPosition = glm::inverse(streamToAdd.getOrientation()) * relativePosition; - - float angleOfDelivery = glm::angle(glm::vec3(0.0f, 0.0f, -1.0f), - glm::normalize(rotatedListenerPosition)); - - const float MAX_OFF_AXIS_ATTENUATION = 0.2f; - const float OFF_AXIS_ATTENUATION_FORMULA_STEP = (1 - MAX_OFF_AXIS_ATTENUATION) / 2.0f; - - float offAxisCoefficient = MAX_OFF_AXIS_ATTENUATION + - (OFF_AXIS_ATTENUATION_FORMULA_STEP * (angleOfDelivery / PI_OVER_TWO)); - - // multiply the current attenuation coefficient by the calculated off axis coefficient - gain *= offAxisCoefficient; - } - - float attenuationPerDoublingInDistance = AudioMixer::getAttenuationPerDoublingInDistance(); - auto& zoneSettings = AudioMixer::getZoneSettings(); - auto& audioZones = AudioMixer::getAudioZones(); - for (int i = 0; i < zoneSettings.length(); ++i) { - if (audioZones[zoneSettings[i].source].contains(streamToAdd.getPosition()) && - audioZones[zoneSettings[i].listener].contains(listeningNodeStream.getPosition())) { - attenuationPerDoublingInDistance = zoneSettings[i].coefficient; - break; - } - } - - if (distanceBetween >= ATTENUATION_BEGINS_AT_DISTANCE) { - - // translate the zone setting to gain per log2(distance) - float g = 1.0f - attenuationPerDoublingInDistance; - g = (g < EPSILON) ? EPSILON : g; - g = (g > 1.0f) ? 1.0f : g; - - // calculate the distance coefficient using the distance to this node - float distanceCoefficient = fastexp2(fastlog2(g) * fastlog2(distanceBetween/ATTENUATION_BEGINS_AT_DISTANCE)); - - // multiply the current attenuation coefficient by the distance coefficient - gain *= distanceCoefficient; - } - - return gain; -} - -float AudioMixerSlave::azimuthForSource(const AvatarAudioStream& listeningNodeStream, const PositionalAudioStream& streamToAdd, - const glm::vec3& relativePosition) { - glm::quat inverseOrientation = glm::inverse(listeningNodeStream.getOrientation()); - - // Compute sample delay for the two ears to create phase panning - glm::vec3 rotatedSourcePosition = inverseOrientation * relativePosition; - - // project the rotated source position vector onto the XZ plane - rotatedSourcePosition.y = 0.0f; - - static const float SOURCE_DISTANCE_THRESHOLD = 1e-30f; - - if (glm::length2(rotatedSourcePosition) > SOURCE_DISTANCE_THRESHOLD) { - // produce an oriented angle about the y-axis - return glm::orientedAngle(glm::vec3(0.0f, 0.0f, -1.0f), glm::normalize(rotatedSourcePosition), glm::vec3(0.0f, -1.0f, 0.0f)); - } else { - // there is no distance between listener and source - return no azimuth - return 0; - } -} - -void AudioMixerStats::reset() { - sumStreams = 0; - sumListeners = 0; - totalMixes = 0; - hrtfRenders = 0; - hrtfSilentRenders = 0; - hrtfStruggleRenders = 0; - manualStereoMixes = 0; - manualEchoMixes = 0; -} - -void AudioMixerStats::accumulate(const AudioMixerStats& otherStats) { - sumStreams += otherStats.sumStreams; - sumListeners += otherStats.sumListeners; - totalMixes += otherStats.totalMixes; - hrtfRenders += otherStats.hrtfRenders; - hrtfSilentRenders += otherStats.hrtfSilentRenders; - hrtfStruggleRenders += otherStats.hrtfStruggleRenders; - manualStereoMixes += otherStats.manualStereoMixes; - manualEchoMixes += otherStats.manualEchoMixes; -} diff --git a/assignment-client/src/audio/AudioMixer.h b/assignment-client/src/audio/AudioMixer.h index e4fda2fddb..0bfcabd2ab 100644 --- a/assignment-client/src/audio/AudioMixer.h +++ b/assignment-client/src/audio/AudioMixer.h @@ -18,59 +18,14 @@ #include #include +#include "AudioMixerStats.h" +#include "AudioMixerSlave.h" + class PositionalAudioStream; class AvatarAudioStream; class AudioHRTF; class AudioMixerClientData; - -struct AudioMixerStats { - int sumStreams { 0 }; - int sumListeners { 0 }; - - int totalMixes { 0 }; - - int hrtfRenders { 0 }; - int hrtfSilentRenders { 0 }; - int hrtfStruggleRenders { 0 }; - - int manualStereoMixes { 0 }; - int manualEchoMixes { 0 }; - - void reset(); - void accumulate(const AudioMixerStats& otherStats); -}; - -class AudioMixerSlave { -public: - // mix and broadcast non-ignored streams to the node - // returns true if a listener mix was broadcast for the node - void mix(const SharedNodePointer& node, unsigned int frame); - - AudioMixerStats stats; - -private: - void writeMixPacket(std::unique_ptr& mixPacket, AudioMixerClientData* data, QByteArray& buffer); - void writeSilentPacket(std::unique_ptr& mixPacket, AudioMixerClientData* data); - - void sendEnvironmentPacket(const SharedNodePointer& node); - - // create mix, returns true if mix has audio - bool prepareMix(const SharedNodePointer& node); - // add a stream to the mix - void addStreamToMix(AudioMixerClientData& listenerData, const QUuid& streamerID, - const AvatarAudioStream& listenerStream, const PositionalAudioStream& streamer); - - float gainForSource(const AvatarAudioStream& listener, const PositionalAudioStream& streamer, - const glm::vec3& relativePosition, bool isEcho); - float azimuthForSource(const AvatarAudioStream& listener, const PositionalAudioStream& streamer, - const glm::vec3& relativePosition); - - // mixing buffers - float _mixedSamples[AudioConstants::NETWORK_FRAME_SAMPLES_STEREO]; - int16_t _clampedSamples[AudioConstants::NETWORK_FRAME_SAMPLES_STEREO]; -}; - /// Handles assignments of type AudioMixer - mixing streams of audio and re-distributing to various clients. class AudioMixer : public ThreadedAssignment { Q_OBJECT diff --git a/assignment-client/src/audio/AudioMixerSlave.cpp b/assignment-client/src/audio/AudioMixerSlave.cpp new file mode 100644 index 0000000000..ef4d7e0375 --- /dev/null +++ b/assignment-client/src/audio/AudioMixerSlave.cpp @@ -0,0 +1,524 @@ +// +// AudioMixerSlave.cpp +// assignment-client/src/audio +// +// Created by Zach Pomerantz on 11/22/16. +// Copyright 2016 High Fidelity, Inc. +// +// Distributed under the Apache License, Version 2.0. +// See the accompanying file LICENSE or http://www.apache.org/licenses/LICENSE-2.0.html +// + +#include +#include +#include + +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include + +#include "AudioRingBuffer.h" +#include "AudioMixer.h" +#include "AudioMixerClientData.h" +#include "AvatarAudioStream.h" +#include "InjectedAudioStream.h" + +#include "AudioMixerSlave.h" + +void AudioMixerSlave::mix(const SharedNodePointer& node, unsigned int frame) { + AudioMixerClientData* data = (AudioMixerClientData*)node->getLinkedData(); + if (data == nullptr) { + return; + } + + auto avatarStream = data->getAvatarAudioStream(); + if (avatarStream == nullptr) { + return; + } + + auto nodeList = DependencyManager::get(); + + // mute the avatar, if necessary + if (AudioMixer::shouldMute(avatarStream->getQuietestFrameLoudness()) || data->shouldMuteClient()) { + auto mutePacket = NLPacket::create(PacketType::NoisyMute, 0); + nodeList->sendPacket(std::move(mutePacket), *node); + + // probably now we just reset the flag, once should do it (?) + data->setShouldMuteClient(false); + } + + // generate and send audio packets + if (node->getType() == NodeType::Agent && node->getActiveSocket()) { + + // mix streams + bool mixHasAudio = prepareMix(node); + + // write the packet + std::unique_ptr mixPacket; + if (mixHasAudio || data->shouldFlushEncoder()) { + // encode the audio + QByteArray encodedBuffer; + if (mixHasAudio) { + QByteArray decodedBuffer(reinterpret_cast(_clampedSamples), AudioConstants::NETWORK_FRAME_BYTES_STEREO); + data->encode(decodedBuffer, encodedBuffer); + } else { + // time to flush, which resets the shouldFlush until next time we encode something + data->encodeFrameOfZeros(encodedBuffer); + } + + // write it to a packet + writeMixPacket(mixPacket, data, encodedBuffer); + } else { + writeSilentPacket(mixPacket, data); + } + + // send audio environment packet + sendEnvironmentPacket(node); + + // send mixed audio packet + nodeList->sendPacket(std::move(mixPacket), *node); + data->incrementOutgoingMixedAudioSequenceNumber(); + + // send an audio stream stats packet to the client approximately every second + static const unsigned int NUM_FRAMES_PER_SEC = (int) ceil(AudioConstants::NETWORK_FRAMES_PER_SEC); + if (data->shouldSendStats(frame % NUM_FRAMES_PER_SEC)) { + data->sendAudioStreamStatsPackets(node); + } + + ++stats.sumListeners; + } +} + +void AudioMixerSlave::writeMixPacket(std::unique_ptr& mixPacket, AudioMixerClientData* data, QByteArray& buffer) { + int mixPacketBytes = sizeof(quint16) + AudioConstants::MAX_CODEC_NAME_LENGTH_ON_WIRE + + AudioConstants::NETWORK_FRAME_BYTES_STEREO; + mixPacket = NLPacket::create(PacketType::MixedAudio, mixPacketBytes); + + // pack sequence number + quint16 sequence = data->getOutgoingSequenceNumber(); + mixPacket->writePrimitive(sequence); + + // write the codec + QString codecInPacket = data->getCodecName(); + mixPacket->writeString(codecInPacket); + + // pack mixed audio samples + mixPacket->write(buffer.constData(), buffer.size()); +} + +void AudioMixerSlave::writeSilentPacket(std::unique_ptr& mixPacket, AudioMixerClientData* data) { + int silentPacketBytes = sizeof(quint16) + sizeof(quint16) + AudioConstants::MAX_CODEC_NAME_LENGTH_ON_WIRE; + mixPacket = NLPacket::create(PacketType::SilentAudioFrame, silentPacketBytes); + + // pack sequence number + quint16 sequence = data->getOutgoingSequenceNumber(); + mixPacket->writePrimitive(sequence); + + // write the codec + QString codecInPacket = data->getCodecName(); + mixPacket->writeString(codecInPacket); + + // pack number of silent audio samples + quint16 numSilentSamples = AudioConstants::NETWORK_FRAME_SAMPLES_STEREO; + mixPacket->writePrimitive(numSilentSamples); +} + +void AudioMixerSlave::sendEnvironmentPacket(const SharedNodePointer& node) { + // Send stream properties + bool hasReverb = false; + float reverbTime, wetLevel; + + auto& reverbSettings = AudioMixer::getReverbSettings(); + auto& audioZones = AudioMixer::getAudioZones(); + + // find reverb properties + for (int i = 0; i < reverbSettings.size(); ++i) { + AudioMixerClientData* data = static_cast(node->getLinkedData()); + glm::vec3 streamPosition = data->getAvatarAudioStream()->getPosition(); + AABox box = audioZones[reverbSettings[i].zone]; + if (box.contains(streamPosition)) { + hasReverb = true; + reverbTime = reverbSettings[i].reverbTime; + wetLevel = reverbSettings[i].wetLevel; + + break; + } + } + + AudioMixerClientData* nodeData = static_cast(node->getLinkedData()); + AvatarAudioStream* stream = nodeData->getAvatarAudioStream(); + bool dataChanged = (stream->hasReverb() != hasReverb) || + (stream->hasReverb() && (stream->getRevebTime() != reverbTime || + stream->getWetLevel() != wetLevel)); + if (dataChanged) { + // Update stream + if (hasReverb) { + stream->setReverb(reverbTime, wetLevel); + } else { + stream->clearReverb(); + } + } + + // Send at change or every so often + float CHANCE_OF_SEND = 0.01f; + bool sendData = dataChanged || (randFloat() < CHANCE_OF_SEND); + + if (sendData) { + auto nodeList = DependencyManager::get(); + + unsigned char bitset = 0; + + int packetSize = sizeof(bitset); + + if (hasReverb) { + packetSize += sizeof(reverbTime) + sizeof(wetLevel); + } + + auto envPacket = NLPacket::create(PacketType::AudioEnvironment, packetSize); + + if (hasReverb) { + setAtBit(bitset, HAS_REVERB_BIT); + } + + envPacket->writePrimitive(bitset); + + if (hasReverb) { + envPacket->writePrimitive(reverbTime); + envPacket->writePrimitive(wetLevel); + } + nodeList->sendPacket(std::move(envPacket), *node); + } +} + +bool AudioMixerSlave::prepareMix(const SharedNodePointer& node) { + AvatarAudioStream* nodeAudioStream = static_cast(node->getLinkedData())->getAvatarAudioStream(); + AudioMixerClientData* nodeData = static_cast(node->getLinkedData()); + + // zero out the client mix for this node + memset(_mixedSamples, 0, sizeof(_mixedSamples)); + + // loop through all other nodes that have sufficient audio to mix + DependencyManager::get()->eachNode([&](const SharedNodePointer& otherNode){ + // make sure that we have audio data for this other node + // and that it isn't being ignored by our listening node + // and that it isn't ignoring our listening node + if (otherNode->getLinkedData() + && !node->isIgnoringNodeWithID(otherNode->getUUID()) && !otherNode->isIgnoringNodeWithID(node->getUUID())) { + AudioMixerClientData* otherData = static_cast(otherNode->getLinkedData()); + + // check if distance is inside ignore radius + if (node->isIgnoreRadiusEnabled() || otherNode->isIgnoreRadiusEnabled()) { + float ignoreRadius = glm::min(node->getIgnoreRadius(), otherNode->getIgnoreRadius()); + if (glm::distance(nodeData->getPosition(), otherData->getPosition()) < ignoreRadius) { + // skip, distance is inside ignore radius + return; + } + } + + // enumerate the ARBs attached to the otherNode and add all that should be added to mix + auto streamsCopy = otherData->getAudioStreams(); + for (auto& streamPair : streamsCopy) { + auto otherNodeStream = streamPair.second; + if (*otherNode != *node || otherNodeStream->shouldLoopbackForNode()) { + addStreamToMix(*nodeData, otherNode->getUUID(), *nodeAudioStream, *otherNodeStream); + } + } + } + }); + + // use the per listner AudioLimiter to render the mixed data... + nodeData->audioLimiter.render(_mixedSamples, _clampedSamples, AudioConstants::NETWORK_FRAME_SAMPLES_PER_CHANNEL); + + // check for silent audio after the peak limitor has converted the samples + bool hasAudio = false; + for (int i = 0; i < AudioConstants::NETWORK_FRAME_SAMPLES_STEREO; ++i) { + if (_clampedSamples[i] != 0) { + hasAudio = true; + break; + } + } + return hasAudio; +} + +void AudioMixerSlave::addStreamToMix(AudioMixerClientData& listenerNodeData, const QUuid& sourceNodeID, + const AvatarAudioStream& listeningNodeStream, const PositionalAudioStream& streamToAdd) { + // to reduce artifacts we calculate the gain and azimuth for every source for this listener + // even if we are not going to end up mixing in this source + + ++stats.totalMixes; + + // this ensures that the tail of any previously mixed audio or the first block of new audio sounds correct + + // check if this is a server echo of a source back to itself + bool isEcho = (&streamToAdd == &listeningNodeStream); + + glm::vec3 relativePosition = streamToAdd.getPosition() - listeningNodeStream.getPosition(); + + // figure out the distance between source and listener + float distance = glm::max(glm::length(relativePosition), EPSILON); + + // figure out the gain for this source at the listener + float gain = gainForSource(listeningNodeStream, streamToAdd, relativePosition, isEcho); + + // figure out the azimuth to this source at the listener + float azimuth = isEcho ? 0.0f : azimuthForSource(listeningNodeStream, listeningNodeStream, relativePosition); + + float repeatedFrameFadeFactor = 1.0f; + + static const int HRTF_DATASET_INDEX = 1; + + if (!streamToAdd.lastPopSucceeded()) { + bool forceSilentBlock = true; + + if (!streamToAdd.getLastPopOutput().isNull()) { + bool isInjector = dynamic_cast(&streamToAdd); + + // in an injector, just go silent - the injector has likely ended + // in other inputs (microphone, &c.), repeat with fade to avoid the harsh jump to silence + + // we'll repeat the last block until it has a block to mix + // and we'll gradually fade that repeated block into silence. + + // calculate its fade factor, which depends on how many times it's already been repeated. + repeatedFrameFadeFactor = calculateRepeatedFrameFadeFactor(streamToAdd.getConsecutiveNotMixedCount() - 1); + if (!isInjector && repeatedFrameFadeFactor > 0.0f) { + // apply the repeatedFrameFadeFactor to the gain + gain *= repeatedFrameFadeFactor; + + forceSilentBlock = false; + } + } + + if (forceSilentBlock) { + // we're deciding not to repeat either since we've already done it enough times or repetition with fade is disabled + // in this case we will call renderSilent with a forced silent block + // this ensures the correct tail from the previously mixed block and the correct spatialization of first block + // of any upcoming audio + + if (!streamToAdd.isStereo() && !isEcho) { + // get the existing listener-source HRTF object, or create a new one + auto& hrtf = listenerNodeData.hrtfForStream(sourceNodeID, streamToAdd.getStreamIdentifier()); + + // this is not done for stereo streams since they do not go through the HRTF + static int16_t silentMonoBlock[AudioConstants::NETWORK_FRAME_SAMPLES_PER_CHANNEL] = {}; + hrtf.renderSilent(silentMonoBlock, _mixedSamples, HRTF_DATASET_INDEX, azimuth, distance, gain, + AudioConstants::NETWORK_FRAME_SAMPLES_PER_CHANNEL); + + ++stats.hrtfSilentRenders; + } + + return; + } + } + + // grab the stream from the ring buffer + AudioRingBuffer::ConstIterator streamPopOutput = streamToAdd.getLastPopOutput(); + + if (streamToAdd.isStereo() || isEcho) { + // this is a stereo source or server echo so we do not pass it through the HRTF + // simply apply our calculated gain to each sample + if (streamToAdd.isStereo()) { + for (int i = 0; i < AudioConstants::NETWORK_FRAME_SAMPLES_STEREO; ++i) { + _mixedSamples[i] += float(streamPopOutput[i] * gain / AudioConstants::MAX_SAMPLE_VALUE); + } + + ++stats.manualStereoMixes; + } else { + for (int i = 0; i < AudioConstants::NETWORK_FRAME_SAMPLES_STEREO; i += 2) { + auto monoSample = float(streamPopOutput[i / 2] * gain / AudioConstants::MAX_SAMPLE_VALUE); + _mixedSamples[i] += monoSample; + _mixedSamples[i + 1] += monoSample; + } + + ++stats.manualEchoMixes; + } + + return; + } + + // get the existing listener-source HRTF object, or create a new one + auto& hrtf = listenerNodeData.hrtfForStream(sourceNodeID, streamToAdd.getStreamIdentifier()); + + static int16_t streamBlock[AudioConstants::NETWORK_FRAME_SAMPLES_PER_CHANNEL]; + + streamPopOutput.readSamples(streamBlock, AudioConstants::NETWORK_FRAME_SAMPLES_PER_CHANNEL); + + // if the frame we're about to mix is silent, simply call render silent and move on + if (streamToAdd.getLastPopOutputLoudness() == 0.0f) { + // silent frame from source + + // we still need to call renderSilent via the HRTF for mono source + hrtf.renderSilent(streamBlock, _mixedSamples, HRTF_DATASET_INDEX, azimuth, distance, gain, + AudioConstants::NETWORK_FRAME_SAMPLES_PER_CHANNEL); + + ++stats.hrtfSilentRenders; + + return; + } + + float audibilityThreshold = AudioMixer::getMinimumAudibilityThreshold(); + if (audibilityThreshold > 0.0f && + streamToAdd.getLastPopOutputTrailingLoudness() / glm::length(relativePosition) <= audibilityThreshold) { + // the mixer is struggling so we're going to drop off some streams + + // we call renderSilent via the HRTF with the actual frame data and a gain of 0.0 + hrtf.renderSilent(streamBlock, _mixedSamples, HRTF_DATASET_INDEX, azimuth, distance, 0.0f, + AudioConstants::NETWORK_FRAME_SAMPLES_PER_CHANNEL); + + ++stats.hrtfStruggleRenders; + + return; + } + + ++stats.hrtfRenders; + + // mono stream, call the HRTF with our block and calculated azimuth and gain + hrtf.render(streamBlock, _mixedSamples, HRTF_DATASET_INDEX, azimuth, distance, gain, + AudioConstants::NETWORK_FRAME_SAMPLES_PER_CHANNEL); +} + +const int IEEE754_MANT_BITS = 23; +const int IEEE754_EXPN_BIAS = 127; + +// +// for x > 0.0f, returns log2(x) +// for x <= 0.0f, returns large negative value +// +// abs |error| < 8e-3, smooth (exact for x=2^N) for NPOLY=3 +// abs |error| < 2e-4, smooth (exact for x=2^N) for NPOLY=5 +// rel |error| < 0.4 from precision loss very close to 1.0f +// +static inline float fastlog2(float x) { + + union { float f; int32_t i; } mant, bits = { x }; + + // split into mantissa and exponent + mant.i = (bits.i & ((1 << IEEE754_MANT_BITS) - 1)) | (IEEE754_EXPN_BIAS << IEEE754_MANT_BITS); + int32_t expn = (bits.i >> IEEE754_MANT_BITS) - IEEE754_EXPN_BIAS; + + mant.f -= 1.0f; + + // polynomial for log2(1+x) over x=[0,1] + //x = (-0.346555386f * mant.f + 1.346555386f) * mant.f; + x = (((-0.0821307180f * mant.f + 0.321188984f) * mant.f - 0.677784014f) * mant.f + 1.43872575f) * mant.f; + + return x + expn; +} + +// +// for -126 <= x < 128, returns exp2(x) +// +// rel |error| < 3e-3, smooth (exact for x=N) for NPOLY=3 +// rel |error| < 9e-6, smooth (exact for x=N) for NPOLY=5 +// +static inline float fastexp2(float x) { + + union { float f; int32_t i; } xi; + + // bias such that x > 0 + x += IEEE754_EXPN_BIAS; + //x = MAX(x, 1.0f); + //x = MIN(x, 254.9999f); + + // split into integer and fraction + xi.i = (int32_t)x; + x -= xi.i; + + // construct exp2(xi) as a float + xi.i <<= IEEE754_MANT_BITS; + + // polynomial for exp2(x) over x=[0,1] + //x = (0.339766028f * x + 0.660233972f) * x + 1.0f; + x = (((0.0135557472f * x + 0.0520323690f) * x + 0.241379763f) * x + 0.693032121f) * x + 1.0f; + + return x * xi.f; +} + +float AudioMixerSlave::gainForSource(const AvatarAudioStream& listeningNodeStream, const PositionalAudioStream& streamToAdd, + const glm::vec3& relativePosition, bool isEcho) { + float gain = 1.0f; + + float distanceBetween = glm::length(relativePosition); + + if (distanceBetween < EPSILON) { + distanceBetween = EPSILON; + } + + if (streamToAdd.getType() == PositionalAudioStream::Injector) { + gain *= reinterpret_cast(&streamToAdd)->getAttenuationRatio(); + } + + if (!isEcho && (streamToAdd.getType() == PositionalAudioStream::Microphone)) { + // source is another avatar, apply fixed off-axis attenuation to make them quieter as they turn away from listener + glm::vec3 rotatedListenerPosition = glm::inverse(streamToAdd.getOrientation()) * relativePosition; + + float angleOfDelivery = glm::angle(glm::vec3(0.0f, 0.0f, -1.0f), + glm::normalize(rotatedListenerPosition)); + + const float MAX_OFF_AXIS_ATTENUATION = 0.2f; + const float OFF_AXIS_ATTENUATION_FORMULA_STEP = (1 - MAX_OFF_AXIS_ATTENUATION) / 2.0f; + + float offAxisCoefficient = MAX_OFF_AXIS_ATTENUATION + + (OFF_AXIS_ATTENUATION_FORMULA_STEP * (angleOfDelivery / PI_OVER_TWO)); + + // multiply the current attenuation coefficient by the calculated off axis coefficient + gain *= offAxisCoefficient; + } + + float attenuationPerDoublingInDistance = AudioMixer::getAttenuationPerDoublingInDistance(); + auto& zoneSettings = AudioMixer::getZoneSettings(); + auto& audioZones = AudioMixer::getAudioZones(); + for (int i = 0; i < zoneSettings.length(); ++i) { + if (audioZones[zoneSettings[i].source].contains(streamToAdd.getPosition()) && + audioZones[zoneSettings[i].listener].contains(listeningNodeStream.getPosition())) { + attenuationPerDoublingInDistance = zoneSettings[i].coefficient; + break; + } + } + + const float ATTENUATION_BEGINS_AT_DISTANCE = 1.0f; + if (distanceBetween >= ATTENUATION_BEGINS_AT_DISTANCE) { + + // translate the zone setting to gain per log2(distance) + float g = 1.0f - attenuationPerDoublingInDistance; + g = (g < EPSILON) ? EPSILON : g; + g = (g > 1.0f) ? 1.0f : g; + + // calculate the distance coefficient using the distance to this node + float distanceCoefficient = fastexp2(fastlog2(g) * fastlog2(distanceBetween/ATTENUATION_BEGINS_AT_DISTANCE)); + + // multiply the current attenuation coefficient by the distance coefficient + gain *= distanceCoefficient; + } + + return gain; +} + +float AudioMixerSlave::azimuthForSource(const AvatarAudioStream& listeningNodeStream, const PositionalAudioStream& streamToAdd, + const glm::vec3& relativePosition) { + glm::quat inverseOrientation = glm::inverse(listeningNodeStream.getOrientation()); + + // Compute sample delay for the two ears to create phase panning + glm::vec3 rotatedSourcePosition = inverseOrientation * relativePosition; + + // project the rotated source position vector onto the XZ plane + rotatedSourcePosition.y = 0.0f; + + static const float SOURCE_DISTANCE_THRESHOLD = 1e-30f; + + if (glm::length2(rotatedSourcePosition) > SOURCE_DISTANCE_THRESHOLD) { + // produce an oriented angle about the y-axis + return glm::orientedAngle(glm::vec3(0.0f, 0.0f, -1.0f), glm::normalize(rotatedSourcePosition), glm::vec3(0.0f, -1.0f, 0.0f)); + } else { + // there is no distance between listener and source - return no azimuth + return 0; + } +} diff --git a/assignment-client/src/audio/AudioMixerSlave.h b/assignment-client/src/audio/AudioMixerSlave.h new file mode 100644 index 0000000000..ccfc3da2af --- /dev/null +++ b/assignment-client/src/audio/AudioMixerSlave.h @@ -0,0 +1,58 @@ +// +// AudioMixerSlave.h +// assignment-client/src/audio +// +// Created by Zach Pomerantz on 11/22/16. +// Copyright 2016 High Fidelity, Inc. +// +// Distributed under the Apache License, Version 2.0. +// See the accompanying file LICENSE or http://www.apache.org/licenses/LICENSE-2.0.html +// + +#ifndef hifi_AudioMixerSlave_h +#define hifi_AudioMixerSlave_h + +#include +#include +#include +#include +#include + +#include "AudioMixerStats.h" + +class PositionalAudioStream; +class AvatarAudioStream; +class AudioHRTF; +class AudioMixerClientData; + +class AudioMixerSlave { +public: + // mix and broadcast non-ignored streams to the node + // returns true if a listener mix was broadcast for the node + void mix(const SharedNodePointer& node, unsigned int frame); + + AudioMixerStats stats; + +private: + void writeMixPacket(std::unique_ptr& mixPacket, AudioMixerClientData* data, QByteArray& buffer); + void writeSilentPacket(std::unique_ptr& mixPacket, AudioMixerClientData* data); + + void sendEnvironmentPacket(const SharedNodePointer& node); + + // create mix, returns true if mix has audio + bool prepareMix(const SharedNodePointer& node); + // add a stream to the mix + void addStreamToMix(AudioMixerClientData& listenerData, const QUuid& streamerID, + const AvatarAudioStream& listenerStream, const PositionalAudioStream& streamer); + + float gainForSource(const AvatarAudioStream& listener, const PositionalAudioStream& streamer, + const glm::vec3& relativePosition, bool isEcho); + float azimuthForSource(const AvatarAudioStream& listener, const PositionalAudioStream& streamer, + const glm::vec3& relativePosition); + + // mixing buffers + float _mixedSamples[AudioConstants::NETWORK_FRAME_SAMPLES_STEREO]; + int16_t _clampedSamples[AudioConstants::NETWORK_FRAME_SAMPLES_STEREO]; +}; + +#endif // hifi_AudioMixerSlave_h diff --git a/assignment-client/src/audio/AudioMixerStats.cpp b/assignment-client/src/audio/AudioMixerStats.cpp new file mode 100644 index 0000000000..94115ad5ff --- /dev/null +++ b/assignment-client/src/audio/AudioMixerStats.cpp @@ -0,0 +1,34 @@ +// +// AudioMixerStats.cpp +// assignment-client/src/audio +// +// Created by Zach Pomerantz on 11/22/16. +// Copyright 2016 High Fidelity, Inc. +// +// Distributed under the Apache License, Version 2.0. +// See the accompanying file LICENSE or http://www.apache.org/licenses/LICENSE-2.0.html +// + +#include "AudioMixerStats.h" + +void AudioMixerStats::reset() { + sumStreams = 0; + sumListeners = 0; + totalMixes = 0; + hrtfRenders = 0; + hrtfSilentRenders = 0; + hrtfStruggleRenders = 0; + manualStereoMixes = 0; + manualEchoMixes = 0; +} + +void AudioMixerStats::accumulate(const AudioMixerStats& otherStats) { + sumStreams += otherStats.sumStreams; + sumListeners += otherStats.sumListeners; + totalMixes += otherStats.totalMixes; + hrtfRenders += otherStats.hrtfRenders; + hrtfSilentRenders += otherStats.hrtfSilentRenders; + hrtfStruggleRenders += otherStats.hrtfStruggleRenders; + manualStereoMixes += otherStats.manualStereoMixes; + manualEchoMixes += otherStats.manualEchoMixes; +} diff --git a/assignment-client/src/audio/AudioMixerStats.h b/assignment-client/src/audio/AudioMixerStats.h new file mode 100644 index 0000000000..5aefe611f0 --- /dev/null +++ b/assignment-client/src/audio/AudioMixerStats.h @@ -0,0 +1,32 @@ +// +// AudioMixerStats.h +// assignment-client/src/audio +// +// Created by Zach Pomerantz on 11/22/16. +// Copyright 2016 High Fidelity, Inc. +// +// Distributed under the Apache License, Version 2.0. +// See the accompanying file LICENSE or http://www.apache.org/licenses/LICENSE-2.0.html +// + +#ifndef hifi_AudioMixerStats_h +#define hifi_AudioMixerStats_h + +struct AudioMixerStats { + int sumStreams { 0 }; + int sumListeners { 0 }; + + int totalMixes { 0 }; + + int hrtfRenders { 0 }; + int hrtfSilentRenders { 0 }; + int hrtfStruggleRenders { 0 }; + + int manualStereoMixes { 0 }; + int manualEchoMixes { 0 }; + + void reset(); + void accumulate(const AudioMixerStats& otherStats); +}; + +#endif // hifi_AudioMixerStats_h From 41ef105456376559cd214e897b0a761a2deaaa7d Mon Sep 17 00:00:00 2001 From: Zach Pomerantz Date: Wed, 23 Nov 2016 15:08:11 -0500 Subject: [PATCH 08/48] clean up AudioMixerSlave --- .../src/audio/AudioMixerSlave.cpp | 231 +++++++++--------- assignment-client/src/audio/AudioMixerSlave.h | 7 +- 2 files changed, 111 insertions(+), 127 deletions(-) diff --git a/assignment-client/src/audio/AudioMixerSlave.cpp b/assignment-client/src/audio/AudioMixerSlave.cpp index ef4d7e0375..6c302f4cbc 100644 --- a/assignment-client/src/audio/AudioMixerSlave.cpp +++ b/assignment-client/src/audio/AudioMixerSlave.cpp @@ -33,7 +33,106 @@ #include "AudioMixerSlave.h" +std::unique_ptr createAudioPacket(PacketType type, int size, quint16 sequence, QString codec) { + auto audioPacket = NLPacket::create(type, size); + audioPacket->writePrimitive(sequence); + audioPacket->writeString(codec); + return audioPacket; +} + +void sendMixPacket(const SharedNodePointer& node, AudioMixerClientData& data, QByteArray& buffer) { + static const int MIX_PACKET_SIZE = + sizeof(quint16) + AudioConstants::MAX_CODEC_NAME_LENGTH_ON_WIRE + AudioConstants::NETWORK_FRAME_BYTES_STEREO; + quint16 sequence = data.getOutgoingSequenceNumber(); + QString codec = data.getCodecName(); + auto mixPacket = createAudioPacket(PacketType::MixedAudio, MIX_PACKET_SIZE, sequence, codec); + + // pack samples + mixPacket->write(buffer.constData(), buffer.size()); + + // send packet + DependencyManager::get()->sendPacket(std::move(mixPacket), *node); + data.incrementOutgoingMixedAudioSequenceNumber(); +} + +void sendSilentPacket(const SharedNodePointer& node, AudioMixerClientData& data) { + static const int SILENT_PACKET_SIZE = + sizeof(quint16) + AudioConstants::MAX_CODEC_NAME_LENGTH_ON_WIRE + sizeof(quint16); + quint16 sequence = data.getOutgoingSequenceNumber(); + QString codec = data.getCodecName(); + auto mixPacket = createAudioPacket(PacketType::SilentAudioFrame, SILENT_PACKET_SIZE, sequence, codec); + + // pack number of samples + mixPacket->writePrimitive(AudioConstants::NETWORK_FRAME_SAMPLES_STEREO); + + // send packet + DependencyManager::get()->sendPacket(std::move(mixPacket), *node); + data.incrementOutgoingMixedAudioSequenceNumber(); +} + +void sendEnvironmentPacket(const SharedNodePointer& node, AudioMixerClientData& data) { + bool hasReverb = false; + float reverbTime, wetLevel; + + auto& reverbSettings = AudioMixer::getReverbSettings(); + auto& audioZones = AudioMixer::getAudioZones(); + + AvatarAudioStream* stream = data.getAvatarAudioStream(); + glm::vec3 streamPosition = stream->getPosition(); + + // find reverb properties + for (int i = 0; i < reverbSettings.size(); ++i) { + AABox box = audioZones[reverbSettings[i].zone]; + if (box.contains(streamPosition)) { + hasReverb = true; + reverbTime = reverbSettings[i].reverbTime; + wetLevel = reverbSettings[i].wetLevel; + break; + } + } + + // check if data changed + bool dataChanged = (stream->hasReverb() != hasReverb) || + (stream->hasReverb() && (stream->getRevebTime() != reverbTime || stream->getWetLevel() != wetLevel)); + if (dataChanged) { + // update stream + if (hasReverb) { + stream->setReverb(reverbTime, wetLevel); + } else { + stream->clearReverb(); + } + } + + // send packet at change or every so often + float CHANCE_OF_SEND = 0.01f; + bool sendData = dataChanged || (randFloat() < CHANCE_OF_SEND); + + if (sendData) { + // size the packet + unsigned char bitset = 0; + int packetSize = sizeof(bitset); + if (hasReverb) { + packetSize += sizeof(reverbTime) + sizeof(wetLevel); + } + + // write the packet + auto envPacket = NLPacket::create(PacketType::AudioEnvironment, packetSize); + if (hasReverb) { + setAtBit(bitset, HAS_REVERB_BIT); + } + envPacket->writePrimitive(bitset); + if (hasReverb) { + envPacket->writePrimitive(reverbTime); + envPacket->writePrimitive(wetLevel); + } + + // send the packet + DependencyManager::get()->sendPacket(std::move(envPacket), *node); + } +} + void AudioMixerSlave::mix(const SharedNodePointer& node, unsigned int frame) { + // check that the node is valid AudioMixerClientData* data = (AudioMixerClientData*)node->getLinkedData(); if (data == nullptr) { return; @@ -44,25 +143,23 @@ void AudioMixerSlave::mix(const SharedNodePointer& node, unsigned int frame) { return; } - auto nodeList = DependencyManager::get(); - - // mute the avatar, if necessary + // send mute packet, if necessary if (AudioMixer::shouldMute(avatarStream->getQuietestFrameLoudness()) || data->shouldMuteClient()) { auto mutePacket = NLPacket::create(PacketType::NoisyMute, 0); - nodeList->sendPacket(std::move(mutePacket), *node); + DependencyManager::get()->sendPacket(std::move(mutePacket), *node); // probably now we just reset the flag, once should do it (?) data->setShouldMuteClient(false); } - // generate and send audio packets + // send audio packets, if necessary if (node->getType() == NodeType::Agent && node->getActiveSocket()) { + ++stats.sumListeners; - // mix streams + // mix the audio bool mixHasAudio = prepareMix(node); - // write the packet - std::unique_ptr mixPacket; + // send audio packet if (mixHasAudio || data->shouldFlushEncoder()) { // encode the audio QByteArray encodedBuffer; @@ -74,127 +171,19 @@ void AudioMixerSlave::mix(const SharedNodePointer& node, unsigned int frame) { data->encodeFrameOfZeros(encodedBuffer); } - // write it to a packet - writeMixPacket(mixPacket, data, encodedBuffer); + sendMixPacket(node, *data, encodedBuffer); } else { - writeSilentPacket(mixPacket, data); + sendSilentPacket(node, *data); } - // send audio environment packet - sendEnvironmentPacket(node); + // send environment packet + sendEnvironmentPacket(node, *data); - // send mixed audio packet - nodeList->sendPacket(std::move(mixPacket), *node); - data->incrementOutgoingMixedAudioSequenceNumber(); - - // send an audio stream stats packet to the client approximately every second + // send stats packet (about every second) static const unsigned int NUM_FRAMES_PER_SEC = (int) ceil(AudioConstants::NETWORK_FRAMES_PER_SEC); if (data->shouldSendStats(frame % NUM_FRAMES_PER_SEC)) { data->sendAudioStreamStatsPackets(node); } - - ++stats.sumListeners; - } -} - -void AudioMixerSlave::writeMixPacket(std::unique_ptr& mixPacket, AudioMixerClientData* data, QByteArray& buffer) { - int mixPacketBytes = sizeof(quint16) + AudioConstants::MAX_CODEC_NAME_LENGTH_ON_WIRE - + AudioConstants::NETWORK_FRAME_BYTES_STEREO; - mixPacket = NLPacket::create(PacketType::MixedAudio, mixPacketBytes); - - // pack sequence number - quint16 sequence = data->getOutgoingSequenceNumber(); - mixPacket->writePrimitive(sequence); - - // write the codec - QString codecInPacket = data->getCodecName(); - mixPacket->writeString(codecInPacket); - - // pack mixed audio samples - mixPacket->write(buffer.constData(), buffer.size()); -} - -void AudioMixerSlave::writeSilentPacket(std::unique_ptr& mixPacket, AudioMixerClientData* data) { - int silentPacketBytes = sizeof(quint16) + sizeof(quint16) + AudioConstants::MAX_CODEC_NAME_LENGTH_ON_WIRE; - mixPacket = NLPacket::create(PacketType::SilentAudioFrame, silentPacketBytes); - - // pack sequence number - quint16 sequence = data->getOutgoingSequenceNumber(); - mixPacket->writePrimitive(sequence); - - // write the codec - QString codecInPacket = data->getCodecName(); - mixPacket->writeString(codecInPacket); - - // pack number of silent audio samples - quint16 numSilentSamples = AudioConstants::NETWORK_FRAME_SAMPLES_STEREO; - mixPacket->writePrimitive(numSilentSamples); -} - -void AudioMixerSlave::sendEnvironmentPacket(const SharedNodePointer& node) { - // Send stream properties - bool hasReverb = false; - float reverbTime, wetLevel; - - auto& reverbSettings = AudioMixer::getReverbSettings(); - auto& audioZones = AudioMixer::getAudioZones(); - - // find reverb properties - for (int i = 0; i < reverbSettings.size(); ++i) { - AudioMixerClientData* data = static_cast(node->getLinkedData()); - glm::vec3 streamPosition = data->getAvatarAudioStream()->getPosition(); - AABox box = audioZones[reverbSettings[i].zone]; - if (box.contains(streamPosition)) { - hasReverb = true; - reverbTime = reverbSettings[i].reverbTime; - wetLevel = reverbSettings[i].wetLevel; - - break; - } - } - - AudioMixerClientData* nodeData = static_cast(node->getLinkedData()); - AvatarAudioStream* stream = nodeData->getAvatarAudioStream(); - bool dataChanged = (stream->hasReverb() != hasReverb) || - (stream->hasReverb() && (stream->getRevebTime() != reverbTime || - stream->getWetLevel() != wetLevel)); - if (dataChanged) { - // Update stream - if (hasReverb) { - stream->setReverb(reverbTime, wetLevel); - } else { - stream->clearReverb(); - } - } - - // Send at change or every so often - float CHANCE_OF_SEND = 0.01f; - bool sendData = dataChanged || (randFloat() < CHANCE_OF_SEND); - - if (sendData) { - auto nodeList = DependencyManager::get(); - - unsigned char bitset = 0; - - int packetSize = sizeof(bitset); - - if (hasReverb) { - packetSize += sizeof(reverbTime) + sizeof(wetLevel); - } - - auto envPacket = NLPacket::create(PacketType::AudioEnvironment, packetSize); - - if (hasReverb) { - setAtBit(bitset, HAS_REVERB_BIT); - } - - envPacket->writePrimitive(bitset); - - if (hasReverb) { - envPacket->writePrimitive(reverbTime); - envPacket->writePrimitive(wetLevel); - } - nodeList->sendPacket(std::move(envPacket), *node); } } diff --git a/assignment-client/src/audio/AudioMixerSlave.h b/assignment-client/src/audio/AudioMixerSlave.h index ccfc3da2af..83906b6f84 100644 --- a/assignment-client/src/audio/AudioMixerSlave.h +++ b/assignment-client/src/audio/AudioMixerSlave.h @@ -28,17 +28,12 @@ class AudioMixerClientData; class AudioMixerSlave { public: // mix and broadcast non-ignored streams to the node - // returns true if a listener mix was broadcast for the node + // returns true if a mixed packet was sent to the node void mix(const SharedNodePointer& node, unsigned int frame); AudioMixerStats stats; private: - void writeMixPacket(std::unique_ptr& mixPacket, AudioMixerClientData* data, QByteArray& buffer); - void writeSilentPacket(std::unique_ptr& mixPacket, AudioMixerClientData* data); - - void sendEnvironmentPacket(const SharedNodePointer& node); - // create mix, returns true if mix has audio bool prepareMix(const SharedNodePointer& node); // add a stream to the mix From e6dfc5204d53de7b51a792f32939d41fe3455285 Mon Sep 17 00:00:00 2001 From: Zach Pomerantz Date: Mon, 28 Nov 2016 17:13:09 -0500 Subject: [PATCH 09/48] expose mutex in LimitedNodeList --- libraries/networking/src/LimitedNodeList.h | 8 +++++--- 1 file changed, 5 insertions(+), 3 deletions(-) diff --git a/libraries/networking/src/LimitedNodeList.h b/libraries/networking/src/LimitedNodeList.h index 598964c2b7..00635e2c4b 100644 --- a/libraries/networking/src/LimitedNodeList.h +++ b/libraries/networking/src/LimitedNodeList.h @@ -174,7 +174,9 @@ public: void sendPeerQueryToIceServer(const HifiSockAddr& iceServerSockAddr, const QUuid& clientID, const QUuid& peerID); SharedNodePointer findNodeWithAddr(const HifiSockAddr& addr); - + + QReadWriteLock& getMutex() { return _nodeMutex; } + template void eachNode(NodeLambda functor) { QReadLocker readLock(&_nodeMutex); @@ -280,7 +282,7 @@ signals: protected slots: void connectedForLocalSocketTest(); void errorTestingLocalSocket(); - + void clientConnectionToSockAddrReset(const HifiSockAddr& sockAddr); protected: @@ -347,7 +349,7 @@ protected: functor(it); } } - + private slots: void flagTimeForConnectionStep(ConnectionStep connectionStep, quint64 timestamp); void possiblyTimeoutSTUNAddressLookup(); From 7a440def876322a0978b669805c86eac47440140 Mon Sep 17 00:00:00 2001 From: Zach Pomerantz Date: Wed, 16 Nov 2016 20:03:08 -0500 Subject: [PATCH 10/48] add AudioMixerSlavePool --- assignment-client/src/audio/AudioMixer.cpp | 23 ++- assignment-client/src/audio/AudioMixer.h | 4 +- .../src/audio/AudioMixerSlavePool.cpp | 149 ++++++++++++++++++ .../src/audio/AudioMixerSlavePool.h | 79 ++++++++++ 4 files changed, 249 insertions(+), 6 deletions(-) create mode 100644 assignment-client/src/audio/AudioMixerSlavePool.cpp create mode 100644 assignment-client/src/audio/AudioMixerSlavePool.h diff --git a/assignment-client/src/audio/AudioMixer.cpp b/assignment-client/src/audio/AudioMixer.cpp index 5a6014aee9..50ef901c2d 100644 --- a/assignment-client/src/audio/AudioMixer.cpp +++ b/assignment-client/src/audio/AudioMixer.cpp @@ -355,12 +355,27 @@ void AudioMixer::start() { while (!_isFinished) { manageLoad(frameTimestamp, framesSinceManagement); - slave.stats.reset(); + { + QReadLocker readLocker(&nodeList->getMutex()); + std::vector nodes; - nodeList->eachNode([&](const SharedNodePointer& node) { _stats.sumStreams += prepareFrame(node, frame); }); - nodeList->eachNode([&](const SharedNodePointer& node) { slave.mix(node, frame); }); + nodeList->eachNode([&](const SharedNodePointer& node) { + // collect the available nodes (to queue them for the slavePool) + nodes.emplace_back(node); - _stats.accumulate(slave.stats); + // prepare frames; pop off any new audio from their streams + _stats.sumStreams += prepareFrame(node, frame); + }); + + // mix across slave threads + slavePool.mix(nodes, frame); + } + + // gather stats + slavePool.each([&](AudioMixerSlave& slave) { + _stats.accumulate(slave.stats); + slave.stats.reset(); + }); ++frame; ++_numStatFrames; diff --git a/assignment-client/src/audio/AudioMixer.h b/assignment-client/src/audio/AudioMixer.h index 0bfcabd2ab..c1811b6655 100644 --- a/assignment-client/src/audio/AudioMixer.h +++ b/assignment-client/src/audio/AudioMixer.h @@ -19,7 +19,7 @@ #include #include "AudioMixerStats.h" -#include "AudioMixerSlave.h" +#include "AudioMixerSlavePool.h" class PositionalAudioStream; class AvatarAudioStream; @@ -88,7 +88,7 @@ private: QString _codecPreferenceOrder; - AudioMixerSlave slave; + AudioMixerSlavePool slavePool; static int _numStaticJitterFrames; // -1 denotes dynamic jitter buffering static float _noiseMutingThreshold; diff --git a/assignment-client/src/audio/AudioMixerSlavePool.cpp b/assignment-client/src/audio/AudioMixerSlavePool.cpp new file mode 100644 index 0000000000..035233c48a --- /dev/null +++ b/assignment-client/src/audio/AudioMixerSlavePool.cpp @@ -0,0 +1,149 @@ +// +// AudioMixerSlavePool.cpp +// assignment-client/src/audio +// +// Created by Zach Pomerantz on 11/16/2016. +// Copyright 2016 High Fidelity, Inc. +// +// Distributed under the Apache License, Version 2.0. +// See the accompanying file LICENSE or http://www.apache.org/licenses/LICENSE-2.0.html +// + +#include +#include + +#include "AudioMixerSlavePool.h" + +AudioMixerSlavePool::~AudioMixerSlavePool() { + { + std::unique_lock lock(_mutex); + wait(lock); + } + setNumThreads(0); +} + +void AudioMixerSlavePool::mix(const std::vector& nodes, unsigned int frame) { + std::unique_lock lock(_mutex); + start(lock, nodes, frame); + wait(lock); +} + +void AudioMixerSlavePool::each(std::function functor) { + std::unique_lock lock(_mutex); + assert(!_running); + + for (auto& slave : _slaves) { + functor(*slave.get()); + } +} + +void AudioMixerSlavePool::start(std::unique_lock& lock, const std::vector& nodes, unsigned int frame) { + assert(!_running); + + // fill the queue + for (auto& node : nodes) { + _queue.emplace(node); + } + + // toggle running state + _frame = frame; + _running = true; + _numStarted = 0; + _numFinished = 0; + + _slaveCondition.notify_all(); +} + +void AudioMixerSlavePool::wait(std::unique_lock& lock) { + if (_running) { + _poolCondition.wait(lock, [&] { + return _numFinished == _numThreads; + }); + } + + assert(_queue.empty()); + + // toggle running state + _running = false; +} + +void AudioMixerSlavePool::slaveWait() { + std::unique_lock lock(_mutex); + + _slaveCondition.wait(lock, [&] { + return _numStarted != _numThreads; + }); + + // toggle running state + ++_numStarted; +} + +void AudioMixerSlavePool::slaveNotify() { + { + std::unique_lock lock(_mutex); + ++_numFinished; + } + _poolCondition.notify_one(); +} + +void AudioMixerSlavePool::setNumThreads(int numThreads) { + std::unique_lock lock(_mutex); + + // ensure slave are not running + assert(!_running); + + // clamp to allowed size + { + // idealThreadCount returns -1 if cores cannot be detected - cast it to a large number + int maxThreads = (int)((unsigned int)QThread::idealThreadCount()); + int clampedThreads = std::min(std::max(1, numThreads), maxThreads); + if (clampedThreads != numThreads) { + qWarning("%s: clamped to %d (was %d)", __FUNCTION__, numThreads, clampedThreads); + numThreads = clampedThreads; + } + } + qDebug("%s: set %d threads", __FUNCTION__, numThreads); + + if (numThreads > _numThreads) { + // start new slaves + for (int i = 0; i < numThreads - _numThreads; ++i) { + auto slave = new AudioMixerSlaveThread(*this); + slave->start(); + _slaves.emplace_back(slave); + } + } else if (numThreads < _numThreads) { + auto extraBegin = _slaves.begin() + _numThreads; + + // stop extra slaves... + auto slave = extraBegin; + while (slave != _slaves.end()) { + (*slave)->stop(); + } + + // ...cycle slaves with empty queue... + start(lock, std::vector(), 0); + wait(lock); + + // ...wait for them to finish... + slave = extraBegin; + while (slave != _slaves.end()) { + (*slave)->wait(); + } + + // ...and delete them + _slaves.erase(extraBegin, _slaves.end()); + } + + _numThreads = _numStarted = _numFinished = numThreads; +} + +void AudioMixerSlaveThread::run() { + while (!_stop) { + _pool.slaveWait(); + SharedNodePointer node; + while (_pool._queue.try_pop(node)) { + mix(node, _pool._frame); + } + _pool.slaveNotify(); + } +} diff --git a/assignment-client/src/audio/AudioMixerSlavePool.h b/assignment-client/src/audio/AudioMixerSlavePool.h new file mode 100644 index 0000000000..194df83c2b --- /dev/null +++ b/assignment-client/src/audio/AudioMixerSlavePool.h @@ -0,0 +1,79 @@ +// +// AudioMixerSlavePool.h +// assignment-client/src/audio +// +// Created by Zach Pomerantz on 11/16/2016. +// Copyright 2016 High Fidelity, Inc. +// +// Distributed under the Apache License, Version 2.0. +// See the accompanying file LICENSE or http://www.apache.org/licenses/LICENSE-2.0.html +// + +#ifndef hifi_AudioMixerSlavePool_h +#define hifi_AudioMixerSlavePool_h + +#include +#include +#include + +#include + +#include + +#include "AudioMixerSlave.h" + +class AudioMixerSlaveThread; + +class AudioMixerSlavePool { + using Queue = tbb::concurrent_queue; + +public: + AudioMixerSlavePool(int numThreads = QThread::idealThreadCount()) { setNumThreads(numThreads); } + ~AudioMixerSlavePool(); + + // mix on slave threads + void mix(const std::vector& nodes, unsigned int frame); + + void each(std::function functor); + + void setNumThreads(int numThreads); + int numThreads() { return _numThreads; } + +private: + void start(std::unique_lock& lock, const std::vector& nodes, unsigned int frame); + void wait(std::unique_lock& lock); + + friend class AudioMixerSlaveThread; + + // wait for pool to start (called by slaves) + void slaveWait(); + // notify that the slave has finished (called by slave) + void slaveNotify(); + + std::vector> _slaves; + Queue _queue; + unsigned int _frame { 0 }; + + std::mutex _mutex; + std::condition_variable _slaveCondition; + std::condition_variable _poolCondition; + int _numThreads { 0 }; + int _numStarted { 0 }; + int _numFinished { 0 }; + bool _running { false }; +}; + +class AudioMixerSlaveThread : public QThread, public AudioMixerSlave { + Q_OBJECT +public: + AudioMixerSlaveThread(AudioMixerSlavePool& pool) : _pool(pool) {} + + void run() override final; + void stop() { _stop = true; } + +private: + AudioMixerSlavePool& _pool; + std::atomic _stop; +}; + +#endif // hifi_AudioMixerSlavePool_h From b4638105e311c6b309980c39dff6c2fb05f19243 Mon Sep 17 00:00:00 2001 From: Zach Pomerantz Date: Mon, 28 Nov 2016 18:40:11 -0500 Subject: [PATCH 11/48] expose audio mixer threads to gui --- assignment-client/src/audio/AudioMixer.cpp | 16 +++++++++++-- assignment-client/src/audio/AudioMixer.h | 2 +- .../resources/describe-settings.json | 23 +++++++++++++++++++ 3 files changed, 38 insertions(+), 3 deletions(-) diff --git a/assignment-client/src/audio/AudioMixer.cpp b/assignment-client/src/audio/AudioMixer.cpp index 50ef901c2d..a114f836b2 100644 --- a/assignment-client/src/audio/AudioMixer.cpp +++ b/assignment-client/src/audio/AudioMixer.cpp @@ -41,6 +41,7 @@ static const float DEFAULT_NOISE_MUTING_THRESHOLD = 0.003f; static const QString AUDIO_MIXER_LOGGING_TARGET_NAME = "audio-mixer"; static const QString AUDIO_ENV_GROUP_KEY = "audio_env"; static const QString AUDIO_BUFFER_GROUP_KEY = "audio_buffer"; +static const QString AUDIO_THREADING_GROUP_KEY = "audio_threading"; int AudioMixer::_numStaticJitterFrames{ -1 }; float AudioMixer::_noiseMutingThreshold{ DEFAULT_NOISE_MUTING_THRESHOLD }; @@ -368,11 +369,11 @@ void AudioMixer::start() { }); // mix across slave threads - slavePool.mix(nodes, frame); + _slavePool.mix(nodes, frame); } // gather stats - slavePool.each([&](AudioMixerSlave& slave) { + _slavePool.each([&](AudioMixerSlave& slave) { _stats.accumulate(slave.stats); slave.stats.reset(); }); @@ -469,6 +470,17 @@ int AudioMixer::prepareFrame(const SharedNodePointer& node, unsigned int frame) } void AudioMixer::parseSettingsObject(const QJsonObject &settingsObject) { + if (settingsObject.contains(AUDIO_THREADING_GROUP_KEY)) { + QJsonObject audioThreadingGroupObject = settingsObject[AUDIO_THREADING_GROUP_KEY].toObject(); + const QString AUTO_THREADS = "auto_threads"; + bool autoThreads = audioThreadingGroupObject[AUTO_THREADS].toBool(); + if (!autoThreads) { + const QString NUM_THREADS = "num_threads"; + int numThreads = audioThreadingGroupObject[NUM_THREADS].toInt(); + _slavePool.setNumThreads(numThreads); + } + } + if (settingsObject.contains(AUDIO_BUFFER_GROUP_KEY)) { QJsonObject audioBufferGroupObject = settingsObject[AUDIO_BUFFER_GROUP_KEY].toObject(); diff --git a/assignment-client/src/audio/AudioMixer.h b/assignment-client/src/audio/AudioMixer.h index c1811b6655..7a4a84e61f 100644 --- a/assignment-client/src/audio/AudioMixer.h +++ b/assignment-client/src/audio/AudioMixer.h @@ -88,7 +88,7 @@ private: QString _codecPreferenceOrder; - AudioMixerSlavePool slavePool; + AudioMixerSlavePool _slavePool; static int _numStaticJitterFrames; // -1 denotes dynamic jitter buffering static float _noiseMutingThreshold; diff --git a/domain-server/resources/describe-settings.json b/domain-server/resources/describe-settings.json index 8cd9136895..2c9bf3ed4a 100644 --- a/domain-server/resources/describe-settings.json +++ b/domain-server/resources/describe-settings.json @@ -978,6 +978,29 @@ } ] }, + { + "name": "audio_threading", + "label": "Audio Threading", + "assignment-types": [0], + "settings": [ + { + "name": "auto_threads", + "label": "Automatically determine thread count", + "type": "checkbox", + "help": "Allow system to determine number of threads (recommended)", + "default": true, + "advanced": true + }, + { + "name": "num_threads", + "label": "Number of Threads", + "help": "Threads to spin up for audio mixing (if not automatically set)", + "placeholder": "1", + "default": "1", + "advanced": true + } + ] + }, { "name": "audio_env", "label": "Audio Environment", From 56a1d57a94f66f3ead577da058974b4f8f7752b5 Mon Sep 17 00:00:00 2001 From: samcake Date: Tue, 29 Nov 2016 11:57:50 -0800 Subject: [PATCH 12/48] removing separate vertex format from the gl45backend to test if it avoids the crash --- .../gpu-gl/src/gpu/gl45/GL45BackendInput.cpp | 106 ++++++++++++++++++ .../src/gpu/gl45/GL45BackendTransform.cpp | 14 +++ 2 files changed, 120 insertions(+) diff --git a/libraries/gpu-gl/src/gpu/gl45/GL45BackendInput.cpp b/libraries/gpu-gl/src/gpu/gl45/GL45BackendInput.cpp index 01bd2d7bce..97f98ecf09 100644 --- a/libraries/gpu-gl/src/gpu/gl45/GL45BackendInput.cpp +++ b/libraries/gpu-gl/src/gpu/gl45/GL45BackendInput.cpp @@ -17,16 +17,26 @@ using namespace gpu::gl45; void GL45Backend::resetInputStage() { Parent::resetInputStage(); +#ifdef GPU_USE_SEPARATE_VERTEX_FORMAT glBindBuffer(GL_ARRAY_BUFFER, 0); for (uint32_t i = 0; i < _input._attributeActivation.size(); i++) { glDisableVertexAttribArray(i); + glVertexAttribFormat(i, 4, GL_FLOAT, GL_FALSE, 0); } for (uint32_t i = 0; i < _input._attribBindingBuffers.size(); i++) { glBindVertexBuffer(i, 0, 0, 0); } +#else + glBindBuffer(GL_ARRAY_BUFFER, 0); + for (uint32_t i = 0; i < _input._attributeActivation.size(); i++) { + glDisableVertexAttribArray(i); + glVertexAttribPointer(i, 4, GL_FLOAT, GL_FALSE, 0, 0); + } +#endif } void GL45Backend::updateInput() { +#ifdef GPU_USE_SEPARATE_VERTEX_FORMAT if (_input._invalidFormat) { InputStageState::ActivationCache newActivation; @@ -116,4 +126,100 @@ void GL45Backend::updateInput() { _input._invalidBuffers.reset(); (void)CHECK_GL_ERROR(); } +#else + if (_input._invalidFormat || _input._invalidBuffers.any()) { + + if (_input._invalidFormat) { + InputStageState::ActivationCache newActivation; + + _stats._ISNumFormatChanges++; + + // Check expected activation + if (_input._format) { + for (auto& it : _input._format->getAttributes()) { + const Stream::Attribute& attrib = (it).second; + uint8_t locationCount = attrib._element.getLocationCount(); + for (int i = 0; i < locationCount; ++i) { + newActivation.set(attrib._slot + i); + } + } + } + + // Manage Activation what was and what is expected now + for (unsigned int i = 0; i < newActivation.size(); i++) { + bool newState = newActivation[i]; + if (newState != _input._attributeActivation[i]) { + + if (newState) { + glEnableVertexAttribArray(i); + } else { + glDisableVertexAttribArray(i); + } + (void)CHECK_GL_ERROR(); + + _input._attributeActivation.flip(i); + } + } + } + + // now we need to bind the buffers and assign the attrib pointers + if (_input._format) { + const Buffers& buffers = _input._buffers; + const Offsets& offsets = _input._bufferOffsets; + const Offsets& strides = _input._bufferStrides; + + const Stream::Format::AttributeMap& attributes = _input._format->getAttributes(); + auto& inputChannels = _input._format->getChannels(); + _stats._ISNumInputBufferChanges++; + + GLuint boundVBO = 0; + for (auto& channelIt : inputChannels) { + const Stream::Format::ChannelMap::value_type::second_type& channel = (channelIt).second; + if ((channelIt).first < buffers.size()) { + int bufferNum = (channelIt).first; + + if (_input._invalidBuffers.test(bufferNum) || _input._invalidFormat) { + // GLuint vbo = gpu::GL41Backend::getBufferID((*buffers[bufferNum])); + GLuint vbo = _input._bufferVBOs[bufferNum]; + if (boundVBO != vbo) { + glBindBuffer(GL_ARRAY_BUFFER, vbo); + (void)CHECK_GL_ERROR(); + boundVBO = vbo; + } + _input._invalidBuffers[bufferNum] = false; + + for (unsigned int i = 0; i < channel._slots.size(); i++) { + const Stream::Attribute& attrib = attributes.at(channel._slots[i]); + GLuint slot = attrib._slot; + GLuint count = attrib._element.getLocationScalarCount(); + uint8_t locationCount = attrib._element.getLocationCount(); + GLenum type = gl::ELEMENT_TYPE_TO_GL[attrib._element.getType()]; + // GLenum perLocationStride = strides[bufferNum]; + GLenum perLocationStride = attrib._element.getLocationSize(); + GLuint stride = (GLuint)strides[bufferNum]; + GLuint pointer = (GLuint)(attrib._offset + offsets[bufferNum]); + GLboolean isNormalized = attrib._element.isNormalized(); + + for (size_t locNum = 0; locNum < locationCount; ++locNum) { + glVertexAttribPointer(slot + (GLuint)locNum, count, type, isNormalized, stride, + reinterpret_cast(pointer + perLocationStride * (GLuint)locNum)); +#ifdef GPU_STEREO_DRAWCALL_INSTANCED + glVertexAttribDivisor(slot + (GLuint)locNum, attrib._frequency * (isStereo() ? 2 : 1)); +#else + glVertexAttribDivisor(slot + (GLuint)locNum, attrib._frequency); +#endif + } + + // TODO: Support properly the IAttrib version + + (void)CHECK_GL_ERROR(); + } + } + } + } + } + // everything format related should be in sync now + _input._invalidFormat = false; + } +#endif } diff --git a/libraries/gpu-gl/src/gpu/gl45/GL45BackendTransform.cpp b/libraries/gpu-gl/src/gpu/gl45/GL45BackendTransform.cpp index 57862eb983..f5f869add5 100644 --- a/libraries/gpu-gl/src/gpu/gl45/GL45BackendTransform.cpp +++ b/libraries/gpu-gl/src/gpu/gl45/GL45BackendTransform.cpp @@ -80,6 +80,7 @@ void GL45Backend::updateTransform(const Batch& batch) { } glVertexAttribI2i(gpu::Stream::DRAW_CALL_INFO, drawCallInfo.index, drawCallInfo.unused); } else { +#ifdef GPU_USE_SEPARATE_VERTEX_FORMAT if (!_transform._enabledDrawcallInfoBuffer) { glEnableVertexAttribArray(gpu::Stream::DRAW_CALL_INFO); // Make sure attrib array is enabled glVertexAttribIFormat(gpu::Stream::DRAW_CALL_INFO, 2, GL_UNSIGNED_SHORT, 0); @@ -95,6 +96,19 @@ void GL45Backend::updateTransform(const Batch& batch) { // so we must provide a stride. // This is in contrast to VertexAttrib*Pointer, where a zero signifies tightly-packed elements. glBindVertexBuffer(gpu::Stream::DRAW_CALL_INFO, _transform._drawCallInfoBuffer, (GLintptr)_transform._drawCallInfoOffsets[batch._currentNamedCall], 2 * sizeof(GLushort)); +#else + if (!_transform._enabledDrawcallInfoBuffer) { + glEnableVertexAttribArray(gpu::Stream::DRAW_CALL_INFO); // Make sure attrib array is enabled + glBindBuffer(GL_ARRAY_BUFFER, _transform._drawCallInfoBuffer); +#ifdef GPU_STEREO_DRAWCALL_INSTANCED + glVertexAttribDivisor(gpu::Stream::DRAW_CALL_INFO, (isStereo() ? 2 : 1)); +#else + glVertexAttribDivisor(gpu::Stream::DRAW_CALL_INFO, 1); +#endif + _transform._enabledDrawcallInfoBuffer = true; + } + glVertexAttribIPointer(gpu::Stream::DRAW_CALL_INFO, 2, GL_UNSIGNED_SHORT, 0, _transform._drawCallInfoOffsets[batch._currentNamedCall]); +#endif } (void)CHECK_GL_ERROR(); From 8a6dcdeb6898e61fb7096e05021e38de93dc2788 Mon Sep 17 00:00:00 2001 From: Zach Pomerantz Date: Wed, 30 Nov 2016 16:37:34 -0500 Subject: [PATCH 13/48] mix audio with single node list read lock --- assignment-client/src/audio/AudioMixer.cpp | 19 ++-- .../src/audio/AudioMixerSlave.cpp | 14 ++- assignment-client/src/audio/AudioMixerSlave.h | 12 ++- .../src/audio/AudioMixerSlavePool.cpp | 91 +++++++++------- .../src/audio/AudioMixerSlavePool.h | 101 +++++++++++------- libraries/networking/src/LimitedNodeList.h | 19 +++- 6 files changed, 159 insertions(+), 97 deletions(-) diff --git a/assignment-client/src/audio/AudioMixer.cpp b/assignment-client/src/audio/AudioMixer.cpp index a114f836b2..a2c4402cb0 100644 --- a/assignment-client/src/audio/AudioMixer.cpp +++ b/assignment-client/src/audio/AudioMixer.cpp @@ -356,21 +356,18 @@ void AudioMixer::start() { while (!_isFinished) { manageLoad(frameTimestamp, framesSinceManagement); - { - QReadLocker readLocker(&nodeList->getMutex()); - std::vector nodes; - - nodeList->eachNode([&](const SharedNodePointer& node) { - // collect the available nodes (to queue them for the slavePool) - nodes.emplace_back(node); - - // prepare frames; pop off any new audio from their streams + // aquire the read-lock in a single thread, to avoid canonical rwlock undefined behaviors + // node removal will acquire a write lock; + // read locks (in slave threads) while a write lock is pending have undefined order in pthread + nodeList->algorithm([&](NodeList::const_iterator cbegin, NodeList::const_iterator cend) { + // prepare frames; pop off any new audio from their streams + std::for_each(cbegin, cend, [&](const SharedNodePointer& node) { _stats.sumStreams += prepareFrame(node, frame); }); // mix across slave threads - _slavePool.mix(nodes, frame); - } + _slavePool.mix(cbegin, cend, frame); + }); // gather stats _slavePool.each([&](AudioMixerSlave& slave) { diff --git a/assignment-client/src/audio/AudioMixerSlave.cpp b/assignment-client/src/audio/AudioMixerSlave.cpp index 6c302f4cbc..24a5e9306a 100644 --- a/assignment-client/src/audio/AudioMixerSlave.cpp +++ b/assignment-client/src/audio/AudioMixerSlave.cpp @@ -9,6 +9,8 @@ // See the accompanying file LICENSE or http://www.apache.org/licenses/LICENSE-2.0.html // +#include + #include #include #include @@ -131,7 +133,13 @@ void sendEnvironmentPacket(const SharedNodePointer& node, AudioMixerClientData& } } -void AudioMixerSlave::mix(const SharedNodePointer& node, unsigned int frame) { +void AudioMixerSlave::configure(ConstIter begin, ConstIter end, unsigned int frame) { + _begin = begin; + _end = end; + _frame = frame; +} + +void AudioMixerSlave::mix(const SharedNodePointer& node) { // check that the node is valid AudioMixerClientData* data = (AudioMixerClientData*)node->getLinkedData(); if (data == nullptr) { @@ -181,7 +189,7 @@ void AudioMixerSlave::mix(const SharedNodePointer& node, unsigned int frame) { // send stats packet (about every second) static const unsigned int NUM_FRAMES_PER_SEC = (int) ceil(AudioConstants::NETWORK_FRAMES_PER_SEC); - if (data->shouldSendStats(frame % NUM_FRAMES_PER_SEC)) { + if (data->shouldSendStats(_frame % NUM_FRAMES_PER_SEC)) { data->sendAudioStreamStatsPackets(node); } } @@ -195,7 +203,7 @@ bool AudioMixerSlave::prepareMix(const SharedNodePointer& node) { memset(_mixedSamples, 0, sizeof(_mixedSamples)); // loop through all other nodes that have sufficient audio to mix - DependencyManager::get()->eachNode([&](const SharedNodePointer& otherNode){ + std::for_each(_begin, _end, [&](const SharedNodePointer& otherNode){ // make sure that we have audio data for this other node // and that it isn't being ignored by our listening node // and that it isn't ignoring our listening node diff --git a/assignment-client/src/audio/AudioMixerSlave.h b/assignment-client/src/audio/AudioMixerSlave.h index 83906b6f84..8be1b697de 100644 --- a/assignment-client/src/audio/AudioMixerSlave.h +++ b/assignment-client/src/audio/AudioMixerSlave.h @@ -17,6 +17,7 @@ #include #include #include +#include #include "AudioMixerStats.h" @@ -27,9 +28,13 @@ class AudioMixerClientData; class AudioMixerSlave { public: + using ConstIter = NodeList::const_iterator; + + void configure(ConstIter begin, ConstIter end, unsigned int frame); + // mix and broadcast non-ignored streams to the node // returns true if a mixed packet was sent to the node - void mix(const SharedNodePointer& node, unsigned int frame); + void mix(const SharedNodePointer& node); AudioMixerStats stats; @@ -48,6 +53,11 @@ private: // mixing buffers float _mixedSamples[AudioConstants::NETWORK_FRAME_SAMPLES_STEREO]; int16_t _clampedSamples[AudioConstants::NETWORK_FRAME_SAMPLES_STEREO]; + + // frame state + ConstIter _begin; + ConstIter _end; + unsigned int _frame { 0 }; }; #endif // hifi_AudioMixerSlave_h diff --git a/assignment-client/src/audio/AudioMixerSlavePool.cpp b/assignment-client/src/audio/AudioMixerSlavePool.cpp index 035233c48a..e0b3ac03ac 100644 --- a/assignment-client/src/audio/AudioMixerSlavePool.cpp +++ b/assignment-client/src/audio/AudioMixerSlavePool.cpp @@ -14,22 +14,59 @@ #include "AudioMixerSlavePool.h" +void AudioMixerSlaveThread::run() { + while (!_stop) { + wait(); + + SharedNodePointer node; + while (try_pop(node)) { + mix(node); + } + + notify(); + } +} + +void AudioMixerSlaveThread::wait() { + Lock lock(_pool._mutex); + + _pool._slaveCondition.wait(lock, [&] { + return _pool._numStarted != _pool._numThreads; + }); + + // toggle running state + ++_pool._numStarted; + configure(_pool._begin, _pool._end, _pool._frame); +} + +void AudioMixerSlaveThread::notify() { + { + Lock lock(_pool._mutex); + ++_pool._numFinished; + } + _pool._poolCondition.notify_one(); +} + +bool AudioMixerSlaveThread::try_pop(SharedNodePointer& node) { + return _pool._queue.try_pop(node); +} + AudioMixerSlavePool::~AudioMixerSlavePool() { { - std::unique_lock lock(_mutex); + Lock lock(_mutex); wait(lock); } setNumThreads(0); } -void AudioMixerSlavePool::mix(const std::vector& nodes, unsigned int frame) { - std::unique_lock lock(_mutex); - start(lock, nodes, frame); +void AudioMixerSlavePool::mix(ConstIter begin, ConstIter end, unsigned int frame) { + Lock lock(_mutex); + start(lock, begin, end, frame); wait(lock); } void AudioMixerSlavePool::each(std::function functor) { - std::unique_lock lock(_mutex); + Lock lock(_mutex); assert(!_running); for (auto& slave : _slaves) { @@ -37,24 +74,26 @@ void AudioMixerSlavePool::each(std::function funct } } -void AudioMixerSlavePool::start(std::unique_lock& lock, const std::vector& nodes, unsigned int frame) { +void AudioMixerSlavePool::start(Lock& lock, ConstIter begin, ConstIter end, unsigned int frame) { assert(!_running); // fill the queue - for (auto& node : nodes) { + std::for_each(begin, end, [&](const SharedNodePointer& node) { _queue.emplace(node); - } + }); // toggle running state _frame = frame; _running = true; _numStarted = 0; _numFinished = 0; + _begin = begin; + _end = end; _slaveCondition.notify_all(); } -void AudioMixerSlavePool::wait(std::unique_lock& lock) { +void AudioMixerSlavePool::wait(Lock& lock) { if (_running) { _poolCondition.wait(lock, [&] { return _numFinished == _numThreads; @@ -67,27 +106,8 @@ void AudioMixerSlavePool::wait(std::unique_lock& lock) { _running = false; } -void AudioMixerSlavePool::slaveWait() { - std::unique_lock lock(_mutex); - - _slaveCondition.wait(lock, [&] { - return _numStarted != _numThreads; - }); - - // toggle running state - ++_numStarted; -} - -void AudioMixerSlavePool::slaveNotify() { - { - std::unique_lock lock(_mutex); - ++_numFinished; - } - _poolCondition.notify_one(); -} - void AudioMixerSlavePool::setNumThreads(int numThreads) { - std::unique_lock lock(_mutex); + Lock lock(_mutex); // ensure slave are not running assert(!_running); @@ -121,7 +141,7 @@ void AudioMixerSlavePool::setNumThreads(int numThreads) { } // ...cycle slaves with empty queue... - start(lock, std::vector(), 0); + start(lock); wait(lock); // ...wait for them to finish... @@ -136,14 +156,3 @@ void AudioMixerSlavePool::setNumThreads(int numThreads) { _numThreads = _numStarted = _numFinished = numThreads; } - -void AudioMixerSlaveThread::run() { - while (!_stop) { - _pool.slaveWait(); - SharedNodePointer node; - while (_pool._queue.try_pop(node)) { - mix(node, _pool._frame); - } - _pool.slaveNotify(); - } -} diff --git a/assignment-client/src/audio/AudioMixerSlavePool.h b/assignment-client/src/audio/AudioMixerSlavePool.h index 194df83c2b..c8eb775f87 100644 --- a/assignment-client/src/audio/AudioMixerSlavePool.h +++ b/assignment-client/src/audio/AudioMixerSlavePool.h @@ -22,49 +22,14 @@ #include "AudioMixerSlave.h" -class AudioMixerSlaveThread; - -class AudioMixerSlavePool { - using Queue = tbb::concurrent_queue; - -public: - AudioMixerSlavePool(int numThreads = QThread::idealThreadCount()) { setNumThreads(numThreads); } - ~AudioMixerSlavePool(); - - // mix on slave threads - void mix(const std::vector& nodes, unsigned int frame); - - void each(std::function functor); - - void setNumThreads(int numThreads); - int numThreads() { return _numThreads; } - -private: - void start(std::unique_lock& lock, const std::vector& nodes, unsigned int frame); - void wait(std::unique_lock& lock); - - friend class AudioMixerSlaveThread; - - // wait for pool to start (called by slaves) - void slaveWait(); - // notify that the slave has finished (called by slave) - void slaveNotify(); - - std::vector> _slaves; - Queue _queue; - unsigned int _frame { 0 }; - - std::mutex _mutex; - std::condition_variable _slaveCondition; - std::condition_variable _poolCondition; - int _numThreads { 0 }; - int _numStarted { 0 }; - int _numFinished { 0 }; - bool _running { false }; -}; +class AudioMixerSlavePool; class AudioMixerSlaveThread : public QThread, public AudioMixerSlave { Q_OBJECT + using ConstIter = NodeList::const_iterator; + using Mutex = std::mutex; + using Lock = std::unique_lock; + public: AudioMixerSlaveThread(AudioMixerSlavePool& pool) : _pool(pool) {} @@ -72,8 +37,64 @@ public: void stop() { _stop = true; } private: + friend class AudioMixerSlavePool; + + void wait(); + void notify(); + bool try_pop(SharedNodePointer& node); + + // frame state AudioMixerSlavePool& _pool; + + // synchronization state std::atomic _stop; }; +class AudioMixerSlavePool { + using Queue = tbb::concurrent_queue; + using Mutex = std::mutex; + using Lock = std::unique_lock; + using ConditionVariable = std::condition_variable; + +public: + using ConstIter = NodeList::const_iterator; + + AudioMixerSlavePool(int numThreads = QThread::idealThreadCount()) { setNumThreads(numThreads); } + ~AudioMixerSlavePool(); + + // mix on slave threads + void mix(ConstIter begin, ConstIter end, unsigned int frame); + + // iterate over all slaves + void each(std::function functor); + + void setNumThreads(int numThreads); + int numThreads() { return _numThreads; } + +private: + void start(Lock& lock, ConstIter begin = ConstIter(), ConstIter end = ConstIter(), unsigned int frame = 0); + void wait(Lock& lock); + + std::vector> _slaves; + + friend void AudioMixerSlaveThread::wait(); + friend void AudioMixerSlaveThread::notify(); + friend bool AudioMixerSlaveThread::try_pop(SharedNodePointer& node); + + // synchronization state + Mutex _mutex; + ConditionVariable _slaveCondition; + ConditionVariable _poolCondition; + int _numThreads { 0 }; + int _numStarted { 0 }; + int _numFinished { 0 }; + bool _running { false }; + + // frame state + Queue _queue; + unsigned int _frame { 0 }; + ConstIter _begin; + ConstIter _end; +}; + #endif // hifi_AudioMixerSlavePool_h diff --git a/libraries/networking/src/LimitedNodeList.h b/libraries/networking/src/LimitedNodeList.h index 00635e2c4b..ef26d87ec7 100644 --- a/libraries/networking/src/LimitedNodeList.h +++ b/libraries/networking/src/LimitedNodeList.h @@ -175,7 +175,24 @@ public: SharedNodePointer findNodeWithAddr(const HifiSockAddr& addr); - QReadWriteLock& getMutex() { return _nodeMutex; } + // for use with algorithm + using value_type = SharedNodePointer; + using const_iterator = std::vector::const_iterator; + + // Cede control of iteration under a single read lock (e.g. for use by thread pools) + // This allows multiple threads (i.e. a thread pool) to share a lock + // without deadlocking when a dying node attempts to acquire a write lock + template + void algorithm(NodeAlgorithmLambda functor) { + QReadLocker readLock(&_nodeMutex); + + std::vector nodes(_nodeHash.size()); + std::transform(_nodeHash.cbegin(), _nodeHash.cend(), nodes.begin(), [](const NodeHash::value_type& it) { + return it.second; + }); + + functor(nodes.cbegin(), nodes.cend()); + } template void eachNode(NodeLambda functor) { From 2c42bb0aa8efc1dca6c307eaa44fd099680f5558 Mon Sep 17 00:00:00 2001 From: Zach Pomerantz Date: Wed, 30 Nov 2016 18:18:36 -0500 Subject: [PATCH 14/48] time audio mixing frames --- assignment-client/src/audio/AudioMixer.cpp | 16 ++++++++++++++++ assignment-client/src/audio/AudioMixer.h | 6 ++++++ 2 files changed, 22 insertions(+) diff --git a/assignment-client/src/audio/AudioMixer.cpp b/assignment-client/src/audio/AudioMixer.cpp index a2c4402cb0..b2257d4ba8 100644 --- a/assignment-client/src/audio/AudioMixer.cpp +++ b/assignment-client/src/audio/AudioMixer.cpp @@ -267,6 +267,15 @@ void AudioMixer::sendStatsPacket() { statsObject["avg_streams_per_frame"] = (float)_stats.sumStreams / (float)_numStatFrames; statsObject["avg_listeners_per_frame"] = (float)_stats.sumListeners / (float)_numStatFrames; + _sumMixTimesIndex = (_sumMixTimesIndex + 1) % MIX_TIMES_TRAILING_SECONDS; + _sumMixTimesTrailing -= _sumMixTimesHistory[_sumMixTimesIndex]; + _sumMixTimesHistory[_sumMixTimesIndex] = _sumMixTimes.count(); + _sumMixTimesTrailing += _sumMixTimesHistory[_sumMixTimesIndex]; + + statsObject["avg_us_per_frame"] = (qint64)(_sumMixTimes.count() / _numStatFrames); + statsObject["avg_us_per_frames_trailing"] = + (qint64)((_sumMixTimesTrailing / MIX_TIMES_TRAILING_SECONDS) / _numStatFrames); + QJsonObject mixStats; mixStats["%_hrtf_mixes"] = percentageForMixStats(_stats.hrtfRenders); mixStats["%_hrtf_silent_mixes"] = percentageForMixStats(_stats.hrtfSilentRenders); @@ -280,6 +289,7 @@ void AudioMixer::sendStatsPacket() { statsObject["mix_stats"] = mixStats; _numStatFrames = 0; + _sumMixTimes = std::chrono::microseconds(0); _stats.reset(); // add stats for each listerner @@ -356,6 +366,9 @@ void AudioMixer::start() { while (!_isFinished) { manageLoad(frameTimestamp, framesSinceManagement); + // start timing the frame + auto startTime = p_high_resolution_clock::now(); + // aquire the read-lock in a single thread, to avoid canonical rwlock undefined behaviors // node removal will acquire a write lock; // read locks (in slave threads) while a write lock is pending have undefined order in pthread @@ -375,6 +388,9 @@ void AudioMixer::start() { slave.stats.reset(); }); + // stop timing the frame + _sumMixTimes += std::chrono::duration_cast(p_high_resolution_clock::now() - startTime); + ++frame; ++_numStatFrames; diff --git a/assignment-client/src/audio/AudioMixer.h b/assignment-client/src/audio/AudioMixer.h index 7a4a84e61f..77bc9ad1fb 100644 --- a/assignment-client/src/audio/AudioMixer.h +++ b/assignment-client/src/audio/AudioMixer.h @@ -90,6 +90,12 @@ private: AudioMixerSlavePool _slavePool; + static const int MIX_TIMES_TRAILING_SECONDS = 10; + std::chrono::microseconds _sumMixTimes { 0 }; + uint64_t _sumMixTimesTrailing { 0 }; + uint64_t _sumMixTimesHistory[MIX_TIMES_TRAILING_SECONDS] { 0 }; + int _sumMixTimesIndex { 0 }; + static int _numStaticJitterFrames; // -1 denotes dynamic jitter buffering static float _noiseMutingThreshold; static float _attenuationPerDoublingInDistance; From 78bc5cf502aa5170b6a92328cbb340ec3e2cdecc Mon Sep 17 00:00:00 2001 From: Zach Pomerantz Date: Thu, 1 Dec 2016 21:25:56 +0000 Subject: [PATCH 15/48] fix audio pool resizing --- .../src/audio/AudioMixerSlavePool.cpp | 28 +++++++++++++------ .../src/audio/AudioMixerSlavePool.h | 2 ++ 2 files changed, 21 insertions(+), 9 deletions(-) diff --git a/assignment-client/src/audio/AudioMixerSlavePool.cpp b/assignment-client/src/audio/AudioMixerSlavePool.cpp index e0b3ac03ac..4208c35de5 100644 --- a/assignment-client/src/audio/AudioMixerSlavePool.cpp +++ b/assignment-client/src/audio/AudioMixerSlavePool.cpp @@ -56,7 +56,7 @@ AudioMixerSlavePool::~AudioMixerSlavePool() { Lock lock(_mutex); wait(lock); } - setNumThreads(0); + resize(0); } void AudioMixerSlavePool::mix(ConstIter begin, ConstIter end, unsigned int frame) { @@ -107,21 +107,30 @@ void AudioMixerSlavePool::wait(Lock& lock) { } void AudioMixerSlavePool::setNumThreads(int numThreads) { - Lock lock(_mutex); - - // ensure slave are not running - assert(!_running); - // clamp to allowed size { - // idealThreadCount returns -1 if cores cannot be detected - cast it to a large number - int maxThreads = (int)((unsigned int)QThread::idealThreadCount()); + int maxThreads = QThread::idealThreadCount(); + if (maxThreads == -1) { + // idealThreadCount returns -1 if cores cannot be detected + maxThreads = std::numeric_limits::max(); + } + int clampedThreads = std::min(std::max(1, numThreads), maxThreads); if (clampedThreads != numThreads) { qWarning("%s: clamped to %d (was %d)", __FUNCTION__, numThreads, clampedThreads); numThreads = clampedThreads; } } + + resize(numThreads); +} + +void AudioMixerSlavePool::resize(int numThreads) { + Lock lock(_mutex); + + // ensure slave are not running + assert(!_running); + qDebug("%s: set %d threads", __FUNCTION__, numThreads); if (numThreads > _numThreads) { @@ -147,7 +156,8 @@ void AudioMixerSlavePool::setNumThreads(int numThreads) { // ...wait for them to finish... slave = extraBegin; while (slave != _slaves.end()) { - (*slave)->wait(); + QThread* thread = reinterpret_cast(slave->get()); + thread->wait(); } // ...and delete them diff --git a/assignment-client/src/audio/AudioMixerSlavePool.h b/assignment-client/src/audio/AudioMixerSlavePool.h index c8eb775f87..00ba5ffec6 100644 --- a/assignment-client/src/audio/AudioMixerSlavePool.h +++ b/assignment-client/src/audio/AudioMixerSlavePool.h @@ -75,6 +75,8 @@ private: void start(Lock& lock, ConstIter begin = ConstIter(), ConstIter end = ConstIter(), unsigned int frame = 0); void wait(Lock& lock); + void resize(int numThreads); + std::vector> _slaves; friend void AudioMixerSlaveThread::wait(); From 800744a139af5839a014e37963eedf55392ddf1b Mon Sep 17 00:00:00 2001 From: Zach Pomerantz Date: Thu, 1 Dec 2016 23:03:37 +0000 Subject: [PATCH 16/48] fix audio pool shutdown and add single threading --- .../src/audio/AudioMixerSlavePool.cpp | 41 +++++++++++++++---- 1 file changed, 32 insertions(+), 9 deletions(-) diff --git a/assignment-client/src/audio/AudioMixerSlavePool.cpp b/assignment-client/src/audio/AudioMixerSlavePool.cpp index 4208c35de5..1a4ee78e54 100644 --- a/assignment-client/src/audio/AudioMixerSlavePool.cpp +++ b/assignment-client/src/audio/AudioMixerSlavePool.cpp @@ -31,7 +31,7 @@ void AudioMixerSlaveThread::wait() { Lock lock(_pool._mutex); _pool._slaveCondition.wait(lock, [&] { - return _pool._numStarted != _pool._numThreads; + return _pool._numStarted != _pool._numThreads; }); // toggle running state @@ -52,26 +52,40 @@ bool AudioMixerSlaveThread::try_pop(SharedNodePointer& node) { } AudioMixerSlavePool::~AudioMixerSlavePool() { - { - Lock lock(_mutex); - wait(lock); - } resize(0); } +#ifdef AUDIO_SINGLE_THREADED +static AudioMixerSlave slave; +#endif + void AudioMixerSlavePool::mix(ConstIter begin, ConstIter end, unsigned int frame) { Lock lock(_mutex); + +#ifdef AUDIO_SINGLE_THREADED + slave.configure(_begin, _end, frame); + std::for_each(begin, end, [&](const SharedNodePointer& node) { + slave.mix(node); + }); + +#else start(lock, begin, end, frame); wait(lock); +#endif + } void AudioMixerSlavePool::each(std::function functor) { Lock lock(_mutex); assert(!_running); +#ifdef AUDIO_SINGLE_THREADED + functor(slave); +#else for (auto& slave : _slaves) { functor(*slave.get()); } +#endif } void AudioMixerSlavePool::start(Lock& lock, ConstIter begin, ConstIter end, unsigned int frame) { @@ -83,12 +97,12 @@ void AudioMixerSlavePool::start(Lock& lock, ConstIter begin, ConstIter end, unsi }); // toggle running state - _frame = frame; _running = true; _numStarted = 0; _numFinished = 0; _begin = begin; _end = end; + _frame = frame; _slaveCondition.notify_all(); } @@ -130,8 +144,12 @@ void AudioMixerSlavePool::resize(int numThreads) { // ensure slave are not running assert(!_running); + assert(_numThreads == _slaves.size()); - qDebug("%s: set %d threads", __FUNCTION__, numThreads); +#ifdef AUDIO_SINGLE_THREADED + qDebug("%s: running single threaded", __FUNCTION__, numThreads); +#else + qDebug("%s: set %d threads (was %d)", __FUNCTION__, numThreads, _numThreads); if (numThreads > _numThreads) { // start new slaves @@ -141,23 +159,25 @@ void AudioMixerSlavePool::resize(int numThreads) { _slaves.emplace_back(slave); } } else if (numThreads < _numThreads) { - auto extraBegin = _slaves.begin() + _numThreads; + auto extraBegin = _slaves.begin() + numThreads; // stop extra slaves... auto slave = extraBegin; while (slave != _slaves.end()) { (*slave)->stop(); + ++slave; } // ...cycle slaves with empty queue... start(lock); - wait(lock); + lock.unlock(); // ...wait for them to finish... slave = extraBegin; while (slave != _slaves.end()) { QThread* thread = reinterpret_cast(slave->get()); thread->wait(); + ++slave; } // ...and delete them @@ -165,4 +185,7 @@ void AudioMixerSlavePool::resize(int numThreads) { } _numThreads = _numStarted = _numFinished = numThreads; +#endif + + assert(_numThreads == _slaves.size()); } From d899391a1ad64bed394ea01a15468873f3f67d58 Mon Sep 17 00:00:00 2001 From: Zach Pomerantz Date: Fri, 2 Dec 2016 03:26:41 +0000 Subject: [PATCH 17/48] fix audio hist array initializer for WIN32 --- assignment-client/src/audio/AudioMixer.h | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/assignment-client/src/audio/AudioMixer.h b/assignment-client/src/audio/AudioMixer.h index 77bc9ad1fb..b76ee7f58c 100644 --- a/assignment-client/src/audio/AudioMixer.h +++ b/assignment-client/src/audio/AudioMixer.h @@ -93,7 +93,7 @@ private: static const int MIX_TIMES_TRAILING_SECONDS = 10; std::chrono::microseconds _sumMixTimes { 0 }; uint64_t _sumMixTimesTrailing { 0 }; - uint64_t _sumMixTimesHistory[MIX_TIMES_TRAILING_SECONDS] { 0 }; + uint64_t _sumMixTimesHistory[MIX_TIMES_TRAILING_SECONDS] {}; int _sumMixTimesIndex { 0 }; static int _numStaticJitterFrames; // -1 denotes dynamic jitter buffering From b0900c5d09d2f560f80412be70ad50d4e4da2371 Mon Sep 17 00:00:00 2001 From: samcake Date: Fri, 2 Dec 2016 17:58:15 -0800 Subject: [PATCH 18/48] Revert "removing separate vertex format from the gl45backend to test if it avoids the crash" This reverts commit 56a1d57a94f66f3ead577da058974b4f8f7752b5. --- .../gpu-gl/src/gpu/gl45/GL45BackendInput.cpp | 106 ------------------ .../src/gpu/gl45/GL45BackendTransform.cpp | 14 --- 2 files changed, 120 deletions(-) diff --git a/libraries/gpu-gl/src/gpu/gl45/GL45BackendInput.cpp b/libraries/gpu-gl/src/gpu/gl45/GL45BackendInput.cpp index 97f98ecf09..01bd2d7bce 100644 --- a/libraries/gpu-gl/src/gpu/gl45/GL45BackendInput.cpp +++ b/libraries/gpu-gl/src/gpu/gl45/GL45BackendInput.cpp @@ -17,26 +17,16 @@ using namespace gpu::gl45; void GL45Backend::resetInputStage() { Parent::resetInputStage(); -#ifdef GPU_USE_SEPARATE_VERTEX_FORMAT glBindBuffer(GL_ARRAY_BUFFER, 0); for (uint32_t i = 0; i < _input._attributeActivation.size(); i++) { glDisableVertexAttribArray(i); - glVertexAttribFormat(i, 4, GL_FLOAT, GL_FALSE, 0); } for (uint32_t i = 0; i < _input._attribBindingBuffers.size(); i++) { glBindVertexBuffer(i, 0, 0, 0); } -#else - glBindBuffer(GL_ARRAY_BUFFER, 0); - for (uint32_t i = 0; i < _input._attributeActivation.size(); i++) { - glDisableVertexAttribArray(i); - glVertexAttribPointer(i, 4, GL_FLOAT, GL_FALSE, 0, 0); - } -#endif } void GL45Backend::updateInput() { -#ifdef GPU_USE_SEPARATE_VERTEX_FORMAT if (_input._invalidFormat) { InputStageState::ActivationCache newActivation; @@ -126,100 +116,4 @@ void GL45Backend::updateInput() { _input._invalidBuffers.reset(); (void)CHECK_GL_ERROR(); } -#else - if (_input._invalidFormat || _input._invalidBuffers.any()) { - - if (_input._invalidFormat) { - InputStageState::ActivationCache newActivation; - - _stats._ISNumFormatChanges++; - - // Check expected activation - if (_input._format) { - for (auto& it : _input._format->getAttributes()) { - const Stream::Attribute& attrib = (it).second; - uint8_t locationCount = attrib._element.getLocationCount(); - for (int i = 0; i < locationCount; ++i) { - newActivation.set(attrib._slot + i); - } - } - } - - // Manage Activation what was and what is expected now - for (unsigned int i = 0; i < newActivation.size(); i++) { - bool newState = newActivation[i]; - if (newState != _input._attributeActivation[i]) { - - if (newState) { - glEnableVertexAttribArray(i); - } else { - glDisableVertexAttribArray(i); - } - (void)CHECK_GL_ERROR(); - - _input._attributeActivation.flip(i); - } - } - } - - // now we need to bind the buffers and assign the attrib pointers - if (_input._format) { - const Buffers& buffers = _input._buffers; - const Offsets& offsets = _input._bufferOffsets; - const Offsets& strides = _input._bufferStrides; - - const Stream::Format::AttributeMap& attributes = _input._format->getAttributes(); - auto& inputChannels = _input._format->getChannels(); - _stats._ISNumInputBufferChanges++; - - GLuint boundVBO = 0; - for (auto& channelIt : inputChannels) { - const Stream::Format::ChannelMap::value_type::second_type& channel = (channelIt).second; - if ((channelIt).first < buffers.size()) { - int bufferNum = (channelIt).first; - - if (_input._invalidBuffers.test(bufferNum) || _input._invalidFormat) { - // GLuint vbo = gpu::GL41Backend::getBufferID((*buffers[bufferNum])); - GLuint vbo = _input._bufferVBOs[bufferNum]; - if (boundVBO != vbo) { - glBindBuffer(GL_ARRAY_BUFFER, vbo); - (void)CHECK_GL_ERROR(); - boundVBO = vbo; - } - _input._invalidBuffers[bufferNum] = false; - - for (unsigned int i = 0; i < channel._slots.size(); i++) { - const Stream::Attribute& attrib = attributes.at(channel._slots[i]); - GLuint slot = attrib._slot; - GLuint count = attrib._element.getLocationScalarCount(); - uint8_t locationCount = attrib._element.getLocationCount(); - GLenum type = gl::ELEMENT_TYPE_TO_GL[attrib._element.getType()]; - // GLenum perLocationStride = strides[bufferNum]; - GLenum perLocationStride = attrib._element.getLocationSize(); - GLuint stride = (GLuint)strides[bufferNum]; - GLuint pointer = (GLuint)(attrib._offset + offsets[bufferNum]); - GLboolean isNormalized = attrib._element.isNormalized(); - - for (size_t locNum = 0; locNum < locationCount; ++locNum) { - glVertexAttribPointer(slot + (GLuint)locNum, count, type, isNormalized, stride, - reinterpret_cast(pointer + perLocationStride * (GLuint)locNum)); -#ifdef GPU_STEREO_DRAWCALL_INSTANCED - glVertexAttribDivisor(slot + (GLuint)locNum, attrib._frequency * (isStereo() ? 2 : 1)); -#else - glVertexAttribDivisor(slot + (GLuint)locNum, attrib._frequency); -#endif - } - - // TODO: Support properly the IAttrib version - - (void)CHECK_GL_ERROR(); - } - } - } - } - } - // everything format related should be in sync now - _input._invalidFormat = false; - } -#endif } diff --git a/libraries/gpu-gl/src/gpu/gl45/GL45BackendTransform.cpp b/libraries/gpu-gl/src/gpu/gl45/GL45BackendTransform.cpp index f5f869add5..57862eb983 100644 --- a/libraries/gpu-gl/src/gpu/gl45/GL45BackendTransform.cpp +++ b/libraries/gpu-gl/src/gpu/gl45/GL45BackendTransform.cpp @@ -80,7 +80,6 @@ void GL45Backend::updateTransform(const Batch& batch) { } glVertexAttribI2i(gpu::Stream::DRAW_CALL_INFO, drawCallInfo.index, drawCallInfo.unused); } else { -#ifdef GPU_USE_SEPARATE_VERTEX_FORMAT if (!_transform._enabledDrawcallInfoBuffer) { glEnableVertexAttribArray(gpu::Stream::DRAW_CALL_INFO); // Make sure attrib array is enabled glVertexAttribIFormat(gpu::Stream::DRAW_CALL_INFO, 2, GL_UNSIGNED_SHORT, 0); @@ -96,19 +95,6 @@ void GL45Backend::updateTransform(const Batch& batch) { // so we must provide a stride. // This is in contrast to VertexAttrib*Pointer, where a zero signifies tightly-packed elements. glBindVertexBuffer(gpu::Stream::DRAW_CALL_INFO, _transform._drawCallInfoBuffer, (GLintptr)_transform._drawCallInfoOffsets[batch._currentNamedCall], 2 * sizeof(GLushort)); -#else - if (!_transform._enabledDrawcallInfoBuffer) { - glEnableVertexAttribArray(gpu::Stream::DRAW_CALL_INFO); // Make sure attrib array is enabled - glBindBuffer(GL_ARRAY_BUFFER, _transform._drawCallInfoBuffer); -#ifdef GPU_STEREO_DRAWCALL_INSTANCED - glVertexAttribDivisor(gpu::Stream::DRAW_CALL_INFO, (isStereo() ? 2 : 1)); -#else - glVertexAttribDivisor(gpu::Stream::DRAW_CALL_INFO, 1); -#endif - _transform._enabledDrawcallInfoBuffer = true; - } - glVertexAttribIPointer(gpu::Stream::DRAW_CALL_INFO, 2, GL_UNSIGNED_SHORT, 0, _transform._drawCallInfoOffsets[batch._currentNamedCall]); -#endif } (void)CHECK_GL_ERROR(); From 2a6e46aa0c54c975014661f7affd449e13de8afe Mon Sep 17 00:00:00 2001 From: Zach Pomerantz Date: Tue, 6 Dec 2016 20:15:39 +0000 Subject: [PATCH 19/48] rm static buffer from AudioMixerSlave --- .../src/audio/AudioMixerSlave.cpp | 30 +++++++++---------- assignment-client/src/audio/AudioMixerSlave.h | 4 +-- 2 files changed, 16 insertions(+), 18 deletions(-) diff --git a/assignment-client/src/audio/AudioMixerSlave.cpp b/assignment-client/src/audio/AudioMixerSlave.cpp index 24a5e9306a..8e6ad17192 100644 --- a/assignment-client/src/audio/AudioMixerSlave.cpp +++ b/assignment-client/src/audio/AudioMixerSlave.cpp @@ -172,7 +172,7 @@ void AudioMixerSlave::mix(const SharedNodePointer& node) { // encode the audio QByteArray encodedBuffer; if (mixHasAudio) { - QByteArray decodedBuffer(reinterpret_cast(_clampedSamples), AudioConstants::NETWORK_FRAME_BYTES_STEREO); + QByteArray decodedBuffer(reinterpret_cast(_bufferSamples), AudioConstants::NETWORK_FRAME_BYTES_STEREO); data->encode(decodedBuffer, encodedBuffer); } else { // time to flush, which resets the shouldFlush until next time we encode something @@ -200,16 +200,16 @@ bool AudioMixerSlave::prepareMix(const SharedNodePointer& node) { AudioMixerClientData* nodeData = static_cast(node->getLinkedData()); // zero out the client mix for this node - memset(_mixedSamples, 0, sizeof(_mixedSamples)); + memset(_mixSamples, 0, sizeof(_mixSamples)); // loop through all other nodes that have sufficient audio to mix std::for_each(_begin, _end, [&](const SharedNodePointer& otherNode){ // make sure that we have audio data for this other node // and that it isn't being ignored by our listening node // and that it isn't ignoring our listening node - if (otherNode->getLinkedData() + AudioMixerClientData* otherData = static_cast(otherNode->getLinkedData()); + if (otherData && !node->isIgnoringNodeWithID(otherNode->getUUID()) && !otherNode->isIgnoringNodeWithID(node->getUUID())) { - AudioMixerClientData* otherData = static_cast(otherNode->getLinkedData()); // check if distance is inside ignore radius if (node->isIgnoreRadiusEnabled() || otherNode->isIgnoreRadiusEnabled()) { @@ -232,12 +232,12 @@ bool AudioMixerSlave::prepareMix(const SharedNodePointer& node) { }); // use the per listner AudioLimiter to render the mixed data... - nodeData->audioLimiter.render(_mixedSamples, _clampedSamples, AudioConstants::NETWORK_FRAME_SAMPLES_PER_CHANNEL); + nodeData->audioLimiter.render(_mixSamples, _bufferSamples, AudioConstants::NETWORK_FRAME_SAMPLES_PER_CHANNEL); // check for silent audio after the peak limitor has converted the samples bool hasAudio = false; for (int i = 0; i < AudioConstants::NETWORK_FRAME_SAMPLES_STEREO; ++i) { - if (_clampedSamples[i] != 0) { + if (_bufferSamples[i] != 0) { hasAudio = true; break; } @@ -306,7 +306,7 @@ void AudioMixerSlave::addStreamToMix(AudioMixerClientData& listenerNodeData, con // this is not done for stereo streams since they do not go through the HRTF static int16_t silentMonoBlock[AudioConstants::NETWORK_FRAME_SAMPLES_PER_CHANNEL] = {}; - hrtf.renderSilent(silentMonoBlock, _mixedSamples, HRTF_DATASET_INDEX, azimuth, distance, gain, + hrtf.renderSilent(silentMonoBlock, _mixSamples, HRTF_DATASET_INDEX, azimuth, distance, gain, AudioConstants::NETWORK_FRAME_SAMPLES_PER_CHANNEL); ++stats.hrtfSilentRenders; @@ -324,15 +324,15 @@ void AudioMixerSlave::addStreamToMix(AudioMixerClientData& listenerNodeData, con // simply apply our calculated gain to each sample if (streamToAdd.isStereo()) { for (int i = 0; i < AudioConstants::NETWORK_FRAME_SAMPLES_STEREO; ++i) { - _mixedSamples[i] += float(streamPopOutput[i] * gain / AudioConstants::MAX_SAMPLE_VALUE); + _mixSamples[i] += float(streamPopOutput[i] * gain / AudioConstants::MAX_SAMPLE_VALUE); } ++stats.manualStereoMixes; } else { for (int i = 0; i < AudioConstants::NETWORK_FRAME_SAMPLES_STEREO; i += 2) { auto monoSample = float(streamPopOutput[i / 2] * gain / AudioConstants::MAX_SAMPLE_VALUE); - _mixedSamples[i] += monoSample; - _mixedSamples[i + 1] += monoSample; + _mixSamples[i] += monoSample; + _mixSamples[i + 1] += monoSample; } ++stats.manualEchoMixes; @@ -344,16 +344,14 @@ void AudioMixerSlave::addStreamToMix(AudioMixerClientData& listenerNodeData, con // get the existing listener-source HRTF object, or create a new one auto& hrtf = listenerNodeData.hrtfForStream(sourceNodeID, streamToAdd.getStreamIdentifier()); - static int16_t streamBlock[AudioConstants::NETWORK_FRAME_SAMPLES_PER_CHANNEL]; - - streamPopOutput.readSamples(streamBlock, AudioConstants::NETWORK_FRAME_SAMPLES_PER_CHANNEL); + streamPopOutput.readSamples(_bufferSamples, AudioConstants::NETWORK_FRAME_SAMPLES_PER_CHANNEL); // if the frame we're about to mix is silent, simply call render silent and move on if (streamToAdd.getLastPopOutputLoudness() == 0.0f) { // silent frame from source // we still need to call renderSilent via the HRTF for mono source - hrtf.renderSilent(streamBlock, _mixedSamples, HRTF_DATASET_INDEX, azimuth, distance, gain, + hrtf.renderSilent(_bufferSamples, _mixSamples, HRTF_DATASET_INDEX, azimuth, distance, gain, AudioConstants::NETWORK_FRAME_SAMPLES_PER_CHANNEL); ++stats.hrtfSilentRenders; @@ -367,7 +365,7 @@ void AudioMixerSlave::addStreamToMix(AudioMixerClientData& listenerNodeData, con // the mixer is struggling so we're going to drop off some streams // we call renderSilent via the HRTF with the actual frame data and a gain of 0.0 - hrtf.renderSilent(streamBlock, _mixedSamples, HRTF_DATASET_INDEX, azimuth, distance, 0.0f, + hrtf.renderSilent(_bufferSamples, _mixSamples, HRTF_DATASET_INDEX, azimuth, distance, 0.0f, AudioConstants::NETWORK_FRAME_SAMPLES_PER_CHANNEL); ++stats.hrtfStruggleRenders; @@ -378,7 +376,7 @@ void AudioMixerSlave::addStreamToMix(AudioMixerClientData& listenerNodeData, con ++stats.hrtfRenders; // mono stream, call the HRTF with our block and calculated azimuth and gain - hrtf.render(streamBlock, _mixedSamples, HRTF_DATASET_INDEX, azimuth, distance, gain, + hrtf.render(_bufferSamples, _mixSamples, HRTF_DATASET_INDEX, azimuth, distance, gain, AudioConstants::NETWORK_FRAME_SAMPLES_PER_CHANNEL); } diff --git a/assignment-client/src/audio/AudioMixerSlave.h b/assignment-client/src/audio/AudioMixerSlave.h index 8be1b697de..c4aabfbb4a 100644 --- a/assignment-client/src/audio/AudioMixerSlave.h +++ b/assignment-client/src/audio/AudioMixerSlave.h @@ -51,8 +51,8 @@ private: const glm::vec3& relativePosition); // mixing buffers - float _mixedSamples[AudioConstants::NETWORK_FRAME_SAMPLES_STEREO]; - int16_t _clampedSamples[AudioConstants::NETWORK_FRAME_SAMPLES_STEREO]; + float _mixSamples[AudioConstants::NETWORK_FRAME_SAMPLES_STEREO]; + int16_t _bufferSamples[AudioConstants::NETWORK_FRAME_SAMPLES_STEREO]; // frame state ConstIter _begin; From 132e9bd64466c52c455c35117338cba9ec443a6a Mon Sep 17 00:00:00 2001 From: Zach Pomerantz Date: Tue, 6 Dec 2016 23:51:06 +0000 Subject: [PATCH 20/48] add timings to AudioMixer --- assignment-client/src/audio/AudioMixer.cpp | 88 +++++++++++++++++----- assignment-client/src/audio/AudioMixer.h | 31 ++++++-- 2 files changed, 96 insertions(+), 23 deletions(-) diff --git a/assignment-client/src/audio/AudioMixer.cpp b/assignment-client/src/audio/AudioMixer.cpp index b2257d4ba8..1d41bc1bfd 100644 --- a/assignment-client/src/audio/AudioMixer.cpp +++ b/assignment-client/src/audio/AudioMixer.cpp @@ -260,6 +260,7 @@ void AudioMixer::sendStatsPacket() { return; } + // general stats statsObject["useDynamicJitterBuffers"] = _numStaticJitterFrames == -1; statsObject["trailing_sleep_percentage"] = _trailingSleepRatio * 100.0f; statsObject["performance_throttling_ratio"] = _performanceThrottlingRatio; @@ -267,16 +268,36 @@ void AudioMixer::sendStatsPacket() { statsObject["avg_streams_per_frame"] = (float)_stats.sumStreams / (float)_numStatFrames; statsObject["avg_listeners_per_frame"] = (float)_stats.sumListeners / (float)_numStatFrames; - _sumMixTimesIndex = (_sumMixTimesIndex + 1) % MIX_TIMES_TRAILING_SECONDS; - _sumMixTimesTrailing -= _sumMixTimesHistory[_sumMixTimesIndex]; - _sumMixTimesHistory[_sumMixTimesIndex] = _sumMixTimes.count(); - _sumMixTimesTrailing += _sumMixTimesHistory[_sumMixTimesIndex]; + // timing stats + QJsonObject timingStats; + uint64_t timing, trailing; - statsObject["avg_us_per_frame"] = (qint64)(_sumMixTimes.count() / _numStatFrames); - statsObject["avg_us_per_frames_trailing"] = - (qint64)((_sumMixTimesTrailing / MIX_TIMES_TRAILING_SECONDS) / _numStatFrames); + _sleepTiming.get(timing, trailing); + timingStats["us_per_sleep"] = (qint64)(timing / _numStatFrames); + timingStats["us_per_sleep_trailing"] = (qint64)(trailing / _numStatFrames); + _frameTiming.get(timing, trailing); + timingStats["us_per_frame"] = (qint64)(timing / _numStatFrames); + timingStats["us_per_frame_trailing"] = (qint64)(trailing / _numStatFrames); + + _prepareTiming.get(timing, trailing); + timingStats["us_per_prepare"] = (qint64)(timing / _numStatFrames); + timingStats["us_per_prepare_trailing"] = (qint64)(trailing / _numStatFrames); + + _mixTiming.get(timing, trailing); + timingStats["us_per_mix"] = (qint64)(timing / _numStatFrames); + timingStats["us_per_mix_trailing"] = (qint64)(trailing / _numStatFrames); + + _eventsTiming.get(timing, trailing); + timingStats["us_per_events"] = (qint64)(timing / _numStatFrames); + timingStats["us_per_events_trailing"] = (qint64)(trailing / _numStatFrames); + + // call it "avg_..." to keep it higher in the display, sorted alphabetically + statsObject["avg_timing_stats"] = timingStats; + + // mix stats QJsonObject mixStats; + mixStats["%_hrtf_mixes"] = percentageForMixStats(_stats.hrtfRenders); mixStats["%_hrtf_silent_mixes"] = percentageForMixStats(_stats.hrtfSilentRenders); mixStats["%_hrtf_struggle_mixes"] = percentageForMixStats(_stats.hrtfStruggleRenders); @@ -289,7 +310,6 @@ void AudioMixer::sendStatsPacket() { statsObject["mix_stats"] = mixStats; _numStatFrames = 0; - _sumMixTimes = std::chrono::microseconds(0); _stats.reset(); // add stats for each listerner @@ -364,22 +384,30 @@ void AudioMixer::start() { unsigned int frame = 1; while (!_isFinished) { - manageLoad(frameTimestamp, framesSinceManagement); + { + auto timer = _sleepTiming.timer(); + manageLoad(frameTimestamp, framesSinceManagement); + } - // start timing the frame - auto startTime = p_high_resolution_clock::now(); + auto timer = _frameTiming.timer(); // aquire the read-lock in a single thread, to avoid canonical rwlock undefined behaviors // node removal will acquire a write lock; // read locks (in slave threads) while a write lock is pending have undefined order in pthread nodeList->algorithm([&](NodeList::const_iterator cbegin, NodeList::const_iterator cend) { // prepare frames; pop off any new audio from their streams - std::for_each(cbegin, cend, [&](const SharedNodePointer& node) { - _stats.sumStreams += prepareFrame(node, frame); - }); + { + auto timer = _prepareTiming.timer(); + std::for_each(cbegin, cend, [&](const SharedNodePointer& node) { + _stats.sumStreams += prepareFrame(node, frame); + }); + } // mix across slave threads - _slavePool.mix(cbegin, cend, frame); + { + auto timer = _mixTiming.timer(); + _slavePool.mix(cbegin, cend, frame); + } }); // gather stats @@ -388,14 +416,13 @@ void AudioMixer::start() { slave.stats.reset(); }); - // stop timing the frame - _sumMixTimes += std::chrono::duration_cast(p_high_resolution_clock::now() - startTime); - ++frame; ++_numStatFrames; // play nice with qt event-looping { + auto timer = _eventsTiming.timer(); + // since we're a while loop we need to yield to qt's event processing QCoreApplication::processEvents(); @@ -695,3 +722,28 @@ void AudioMixer::parseSettingsObject(const QJsonObject &settingsObject) { } } } + +AudioMixer::Timer::Timing::Timing(uint64_t& sum) : _sum(sum) { + _timing = p_high_resolution_clock::now(); +} + +AudioMixer::Timer::Timing::~Timing() { + _sum += std::chrono::duration_cast(p_high_resolution_clock::now() - _timing).count(); +} + +void AudioMixer::Timer::get(uint64_t& timing, uint64_t& trailing) { + // update history + _index = (_index + 1) % TIMER_TRAILING_SECONDS; + uint64_t oldTiming = _history[_index]; + _history[_index] = _sum; + + // update trailing + _trailing -= oldTiming; + _trailing += _sum; + + timing = _sum; + trailing = _trailing / TIMER_TRAILING_SECONDS; + + // reset _sum; + _sum = 0; +} diff --git a/assignment-client/src/audio/AudioMixer.h b/assignment-client/src/audio/AudioMixer.h index b76ee7f58c..59cdec7732 100644 --- a/assignment-client/src/audio/AudioMixer.h +++ b/assignment-client/src/audio/AudioMixer.h @@ -90,11 +90,32 @@ private: AudioMixerSlavePool _slavePool; - static const int MIX_TIMES_TRAILING_SECONDS = 10; - std::chrono::microseconds _sumMixTimes { 0 }; - uint64_t _sumMixTimesTrailing { 0 }; - uint64_t _sumMixTimesHistory[MIX_TIMES_TRAILING_SECONDS] {}; - int _sumMixTimesIndex { 0 }; + class Timer { + public: + class Timing{ + public: + Timing(uint64_t& sum); + ~Timing(); + private: + p_high_resolution_clock::time_point _timing; + uint64_t& _sum; + }; + + Timing timer() { return Timing(_sum); } + void get(uint64_t& timing, uint64_t& trailing); + private: + static const int TIMER_TRAILING_SECONDS = 10; + + uint64_t _sum { 0 }; + uint64_t _trailing { 0 }; + uint64_t _history[TIMER_TRAILING_SECONDS] {}; + int _index { 0 }; + }; + Timer _sleepTiming; + Timer _frameTiming; + Timer _prepareTiming; + Timer _mixTiming; + Timer _eventsTiming; static int _numStaticJitterFrames; // -1 denotes dynamic jitter buffering static float _noiseMutingThreshold; From 557ab43f0fe8e91fc1850c8285a076801d94152a Mon Sep 17 00:00:00 2001 From: Zach Pomerantz Date: Tue, 6 Dec 2016 23:53:16 +0000 Subject: [PATCH 21/48] add a timeout for errant audio slave threads --- assignment-client/src/audio/AudioMixerSlavePool.cpp | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/assignment-client/src/audio/AudioMixerSlavePool.cpp b/assignment-client/src/audio/AudioMixerSlavePool.cpp index 1a4ee78e54..8d3ad19e40 100644 --- a/assignment-client/src/audio/AudioMixerSlavePool.cpp +++ b/assignment-client/src/audio/AudioMixerSlavePool.cpp @@ -176,7 +176,8 @@ void AudioMixerSlavePool::resize(int numThreads) { slave = extraBegin; while (slave != _slaves.end()) { QThread* thread = reinterpret_cast(slave->get()); - thread->wait(); + static const int MAX_THREAD_WAIT_TIME = 10; + thread->wait(MAX_THREAD_WAIT_TIME); ++slave; } From 2fc904b99043956d3861aaf0a705d79138a25617 Mon Sep 17 00:00:00 2001 From: Zach Pomerantz Date: Tue, 6 Dec 2016 23:54:13 +0000 Subject: [PATCH 22/48] fix typos --- assignment-client/src/audio/AudioMixerSlave.cpp | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/assignment-client/src/audio/AudioMixerSlave.cpp b/assignment-client/src/audio/AudioMixerSlave.cpp index 8e6ad17192..09f9d92c71 100644 --- a/assignment-client/src/audio/AudioMixerSlave.cpp +++ b/assignment-client/src/audio/AudioMixerSlave.cpp @@ -231,10 +231,10 @@ bool AudioMixerSlave::prepareMix(const SharedNodePointer& node) { } }); - // use the per listner AudioLimiter to render the mixed data... + // use the per listener AudioLimiter to render the mixed data... nodeData->audioLimiter.render(_mixSamples, _bufferSamples, AudioConstants::NETWORK_FRAME_SAMPLES_PER_CHANNEL); - // check for silent audio after the peak limitor has converted the samples + // check for silent audio after the peak limiter has converted the samples bool hasAudio = false; for (int i = 0; i < AudioConstants::NETWORK_FRAME_SAMPLES_STEREO; ++i) { if (_bufferSamples[i] != 0) { From 259775e163c403801ede1c73d7e01ac28c26547c Mon Sep 17 00:00:00 2001 From: Zach Pomerantz Date: Tue, 6 Dec 2016 23:56:11 +0000 Subject: [PATCH 23/48] clarify lock requirement in audio slave pool --- assignment-client/src/audio/AudioMixerSlavePool.cpp | 3 +++ assignment-client/src/audio/AudioMixerSlavePool.h | 1 + 2 files changed, 4 insertions(+) diff --git a/assignment-client/src/audio/AudioMixerSlavePool.cpp b/assignment-client/src/audio/AudioMixerSlavePool.cpp index 8d3ad19e40..6f0adaf661 100644 --- a/assignment-client/src/audio/AudioMixerSlavePool.cpp +++ b/assignment-client/src/audio/AudioMixerSlavePool.cpp @@ -89,6 +89,7 @@ void AudioMixerSlavePool::each(std::function funct } void AudioMixerSlavePool::start(Lock& lock, ConstIter begin, ConstIter end, unsigned int frame) { + assert(lock.owns_lock()); assert(!_running); // fill the queue @@ -108,6 +109,8 @@ void AudioMixerSlavePool::start(Lock& lock, ConstIter begin, ConstIter end, unsi } void AudioMixerSlavePool::wait(Lock& lock) { + assert(lock.owns_lock()); + if (_running) { _poolCondition.wait(lock, [&] { return _numFinished == _numThreads; diff --git a/assignment-client/src/audio/AudioMixerSlavePool.h b/assignment-client/src/audio/AudioMixerSlavePool.h index 00ba5ffec6..7043d2c50e 100644 --- a/assignment-client/src/audio/AudioMixerSlavePool.h +++ b/assignment-client/src/audio/AudioMixerSlavePool.h @@ -72,6 +72,7 @@ public: int numThreads() { return _numThreads; } private: + // these methods require access to guarded members, so require a lock as argument void start(Lock& lock, ConstIter begin = ConstIter(), ConstIter end = ConstIter(), unsigned int frame = 0); void wait(Lock& lock); From 28960681d9d7fa5206848a78530328c25337d35d Mon Sep 17 00:00:00 2001 From: Zach Pomerantz Date: Tue, 6 Dec 2016 23:57:38 +0000 Subject: [PATCH 24/48] limit audio pool to sane thread count --- assignment-client/src/audio/AudioMixerSlavePool.cpp | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/assignment-client/src/audio/AudioMixerSlavePool.cpp b/assignment-client/src/audio/AudioMixerSlavePool.cpp index 6f0adaf661..7668553662 100644 --- a/assignment-client/src/audio/AudioMixerSlavePool.cpp +++ b/assignment-client/src/audio/AudioMixerSlavePool.cpp @@ -129,7 +129,8 @@ void AudioMixerSlavePool::setNumThreads(int numThreads) { int maxThreads = QThread::idealThreadCount(); if (maxThreads == -1) { // idealThreadCount returns -1 if cores cannot be detected - maxThreads = std::numeric_limits::max(); + static const int MAX_THREADS_IF_UNKNOWN = 4; + maxThreads = MAX_THREADS_IF_UNKNOWN; } int clampedThreads = std::min(std::max(1, numThreads), maxThreads); From b2f995d3df233ddd2143e21e66f6415d5af4d009 Mon Sep 17 00:00:00 2001 From: Zach Pomerantz Date: Wed, 7 Dec 2016 00:00:03 +0000 Subject: [PATCH 25/48] algorithm->nestedEach in LimitedNodeList --- assignment-client/src/audio/AudioMixer.cpp | 5 +---- libraries/networking/src/LimitedNodeList.h | 6 +++--- 2 files changed, 4 insertions(+), 7 deletions(-) diff --git a/assignment-client/src/audio/AudioMixer.cpp b/assignment-client/src/audio/AudioMixer.cpp index 1d41bc1bfd..9608f7c63b 100644 --- a/assignment-client/src/audio/AudioMixer.cpp +++ b/assignment-client/src/audio/AudioMixer.cpp @@ -391,10 +391,7 @@ void AudioMixer::start() { auto timer = _frameTiming.timer(); - // aquire the read-lock in a single thread, to avoid canonical rwlock undefined behaviors - // node removal will acquire a write lock; - // read locks (in slave threads) while a write lock is pending have undefined order in pthread - nodeList->algorithm([&](NodeList::const_iterator cbegin, NodeList::const_iterator cend) { + nodeList->nestedEach([&](NodeList::const_iterator cbegin, NodeList::const_iterator cend) { // prepare frames; pop off any new audio from their streams { auto timer = _prepareTiming.timer(); diff --git a/libraries/networking/src/LimitedNodeList.h b/libraries/networking/src/LimitedNodeList.h index ef26d87ec7..c635744512 100644 --- a/libraries/networking/src/LimitedNodeList.h +++ b/libraries/networking/src/LimitedNodeList.h @@ -175,15 +175,15 @@ public: SharedNodePointer findNodeWithAddr(const HifiSockAddr& addr); - // for use with algorithm using value_type = SharedNodePointer; using const_iterator = std::vector::const_iterator; // Cede control of iteration under a single read lock (e.g. for use by thread pools) + // Use this for nested loops instead of taking nested read locks! // This allows multiple threads (i.e. a thread pool) to share a lock // without deadlocking when a dying node attempts to acquire a write lock - template - void algorithm(NodeAlgorithmLambda functor) { + template + void nestedEach(NestedNodeLambda functor) { QReadLocker readLock(&_nodeMutex); std::vector nodes(_nodeHash.size()); From c4e435a1666d09c8191f71ef8dd66201dddc8ee1 Mon Sep 17 00:00:00 2001 From: Zach Pomerantz Date: Wed, 7 Dec 2016 01:17:34 +0000 Subject: [PATCH 26/48] simplify locks in AudioMixerSlavePool --- .../src/audio/AudioMixerSlavePool.cpp | 100 +++++++----------- .../src/audio/AudioMixerSlavePool.h | 13 +-- 2 files changed, 42 insertions(+), 71 deletions(-) diff --git a/assignment-client/src/audio/AudioMixerSlavePool.cpp b/assignment-client/src/audio/AudioMixerSlavePool.cpp index 7668553662..6d17028de3 100644 --- a/assignment-client/src/audio/AudioMixerSlavePool.cpp +++ b/assignment-client/src/audio/AudioMixerSlavePool.cpp @@ -16,32 +16,34 @@ void AudioMixerSlaveThread::run() { while (!_stop) { - wait(); + wait(); // for the audio pool to notify + // iterate over all available nodes SharedNodePointer node; while (try_pop(node)) { mix(node); } - notify(); + notify(); // the audio pool we are done } } void AudioMixerSlaveThread::wait() { - Lock lock(_pool._mutex); - - _pool._slaveCondition.wait(lock, [&] { - return _pool._numStarted != _pool._numThreads; - }); - - // toggle running state - ++_pool._numStarted; + { + Lock lock(_pool._mutex); + _pool._slaveCondition.wait(lock, [&] { + assert(_pool._numStarted <= _pool._numThreads); + return _pool._numStarted != _pool._numThreads; + }); + ++_pool._numStarted; + } configure(_pool._begin, _pool._end, _pool._frame); } void AudioMixerSlaveThread::notify() { { Lock lock(_pool._mutex); + assert(_pool._numFinished < _pool._numThreads); ++_pool._numFinished; } _pool._poolCondition.notify_one(); @@ -51,34 +53,45 @@ bool AudioMixerSlaveThread::try_pop(SharedNodePointer& node) { return _pool._queue.try_pop(node); } -AudioMixerSlavePool::~AudioMixerSlavePool() { - resize(0); -} - #ifdef AUDIO_SINGLE_THREADED static AudioMixerSlave slave; #endif void AudioMixerSlavePool::mix(ConstIter begin, ConstIter end, unsigned int frame) { - Lock lock(_mutex); + _begin = begin; + _end = end; + _frame = frame; #ifdef AUDIO_SINGLE_THREADED slave.configure(_begin, _end, frame); std::for_each(begin, end, [&](const SharedNodePointer& node) { slave.mix(node); }); - #else - start(lock, begin, end, frame); - wait(lock); -#endif + // fill the queue + std::for_each(_begin, _end, [&](const SharedNodePointer& node) { + _queue.emplace(node); + }); + { + Lock lock(_mutex); + + // mix + _numStarted = _numFinished = 0; + _slaveCondition.notify_all(); + + // wait + _poolCondition.wait(lock, [&] { + assert(_numFinished <= _numThreads); + return _numFinished == _numThreads; + }); + } + assert(_numStarted == _numThreads); + assert(_queue.empty()); +#endif } void AudioMixerSlavePool::each(std::function functor) { - Lock lock(_mutex); - assert(!_running); - #ifdef AUDIO_SINGLE_THREADED functor(slave); #else @@ -88,41 +101,6 @@ void AudioMixerSlavePool::each(std::function funct #endif } -void AudioMixerSlavePool::start(Lock& lock, ConstIter begin, ConstIter end, unsigned int frame) { - assert(lock.owns_lock()); - assert(!_running); - - // fill the queue - std::for_each(begin, end, [&](const SharedNodePointer& node) { - _queue.emplace(node); - }); - - // toggle running state - _running = true; - _numStarted = 0; - _numFinished = 0; - _begin = begin; - _end = end; - _frame = frame; - - _slaveCondition.notify_all(); -} - -void AudioMixerSlavePool::wait(Lock& lock) { - assert(lock.owns_lock()); - - if (_running) { - _poolCondition.wait(lock, [&] { - return _numFinished == _numThreads; - }); - } - - assert(_queue.empty()); - - // toggle running state - _running = false; -} - void AudioMixerSlavePool::setNumThreads(int numThreads) { // clamp to allowed size { @@ -144,10 +122,6 @@ void AudioMixerSlavePool::setNumThreads(int numThreads) { } void AudioMixerSlavePool::resize(int numThreads) { - Lock lock(_mutex); - - // ensure slave are not running - assert(!_running); assert(_numThreads == _slaves.size()); #ifdef AUDIO_SINGLE_THREADED @@ -173,8 +147,8 @@ void AudioMixerSlavePool::resize(int numThreads) { } // ...cycle slaves with empty queue... - start(lock); - lock.unlock(); + _numStarted = _numFinished = 0; + _slaveCondition.notify_all(); // ...wait for them to finish... slave = extraBegin; diff --git a/assignment-client/src/audio/AudioMixerSlavePool.h b/assignment-client/src/audio/AudioMixerSlavePool.h index 7043d2c50e..91f60f5094 100644 --- a/assignment-client/src/audio/AudioMixerSlavePool.h +++ b/assignment-client/src/audio/AudioMixerSlavePool.h @@ -50,6 +50,8 @@ private: std::atomic _stop; }; +// Slave pool for audio mixers +// AudioMixerSlavePool is not thread-safe! It should be instantiated and used from a single thread. class AudioMixerSlavePool { using Queue = tbb::concurrent_queue; using Mutex = std::mutex; @@ -60,7 +62,7 @@ public: using ConstIter = NodeList::const_iterator; AudioMixerSlavePool(int numThreads = QThread::idealThreadCount()) { setNumThreads(numThreads); } - ~AudioMixerSlavePool(); + ~AudioMixerSlavePool() { resize(0); } // mix on slave threads void mix(ConstIter begin, ConstIter end, unsigned int frame); @@ -72,10 +74,6 @@ public: int numThreads() { return _numThreads; } private: - // these methods require access to guarded members, so require a lock as argument - void start(Lock& lock, ConstIter begin = ConstIter(), ConstIter end = ConstIter(), unsigned int frame = 0); - void wait(Lock& lock); - void resize(int numThreads); std::vector> _slaves; @@ -89,9 +87,8 @@ private: ConditionVariable _slaveCondition; ConditionVariable _poolCondition; int _numThreads { 0 }; - int _numStarted { 0 }; - int _numFinished { 0 }; - bool _running { false }; + int _numStarted { 0 }; // guarded by _mutex + int _numFinished { 0 }; // guarded by _mutex // frame state Queue _queue; From 87dd8a46f8e4204d2c3c9bfdd29b7313a4206a51 Mon Sep 17 00:00:00 2001 From: Bradley Austin Davis Date: Tue, 6 Dec 2016 16:09:20 -0800 Subject: [PATCH 27/48] Preserve and restore the GL context when resizing QML surfaces --- libraries/gl/src/gl/GLHelpers.cpp | 12 +++++ libraries/gl/src/gl/GLHelpers.h | 5 ++ libraries/gl/src/gl/OffscreenQmlSurface.cpp | 57 +++++++++++---------- 3 files changed, 46 insertions(+), 28 deletions(-) diff --git a/libraries/gl/src/gl/GLHelpers.cpp b/libraries/gl/src/gl/GLHelpers.cpp index dca1214b32..ab91ca0902 100644 --- a/libraries/gl/src/gl/GLHelpers.cpp +++ b/libraries/gl/src/gl/GLHelpers.cpp @@ -69,3 +69,15 @@ QThread* RENDER_THREAD = nullptr; bool isRenderThread() { return QThread::currentThread() == RENDER_THREAD; } + +namespace gl { + void withSavedContext(const std::function& f) { + // Save the original GL context, because creating a QML surface will create a new context + QOpenGLContext * savedContext = QOpenGLContext::currentContext(); + QSurface * savedSurface = savedContext ? savedContext->surface() : nullptr; + f(); + if (savedContext) { + savedContext->makeCurrent(savedSurface); + } + } +} diff --git a/libraries/gl/src/gl/GLHelpers.h b/libraries/gl/src/gl/GLHelpers.h index daa181467d..84229b97d2 100644 --- a/libraries/gl/src/gl/GLHelpers.h +++ b/libraries/gl/src/gl/GLHelpers.h @@ -10,6 +10,7 @@ #ifndef hifi_GLHelpers_h #define hifi_GLHelpers_h +#include #include // 16 bits of depth precision @@ -34,4 +35,8 @@ int glVersionToInteger(QString glVersion); bool isRenderThread(); +namespace gl { + void withSavedContext(const std::function& f); +} + #endif diff --git a/libraries/gl/src/gl/OffscreenQmlSurface.cpp b/libraries/gl/src/gl/OffscreenQmlSurface.cpp index 06f755c1dd..cde779d101 100644 --- a/libraries/gl/src/gl/OffscreenQmlSurface.cpp +++ b/libraries/gl/src/gl/OffscreenQmlSurface.cpp @@ -467,40 +467,41 @@ void OffscreenQmlSurface::resize(const QSize& newSize_, bool forceResize) { } qCDebug(glLogging) << "Offscreen UI resizing to " << newSize.width() << "x" << newSize.height(); + gl::withSavedContext([&] { + _canvas->makeCurrent(); - _canvas->makeCurrent(); - - // Release hold on the textures of the old size - if (uvec2() != _size) { - // If the most recent texture was unused, we can directly recycle it - if (_latestTextureAndFence.first) { - offscreenTextures.releaseTexture(_latestTextureAndFence); - _latestTextureAndFence = { 0, 0 }; + // Release hold on the textures of the old size + if (uvec2() != _size) { + // If the most recent texture was unused, we can directly recycle it + if (_latestTextureAndFence.first) { + offscreenTextures.releaseTexture(_latestTextureAndFence); + _latestTextureAndFence = { 0, 0 }; + } + offscreenTextures.releaseSize(_size); } - offscreenTextures.releaseSize(_size); - } - _size = newOffscreenSize; + _size = newOffscreenSize; - // Acquire the new texture size - if (uvec2() != _size) { - offscreenTextures.acquireSize(_size); - if (_depthStencil) { - glDeleteRenderbuffers(1, &_depthStencil); - _depthStencil = 0; + // Acquire the new texture size + if (uvec2() != _size) { + offscreenTextures.acquireSize(_size); + if (_depthStencil) { + glDeleteRenderbuffers(1, &_depthStencil); + _depthStencil = 0; + } + glGenRenderbuffers(1, &_depthStencil); + glBindRenderbuffer(GL_RENDERBUFFER, _depthStencil); + glRenderbufferStorage(GL_RENDERBUFFER, GL_DEPTH_COMPONENT16, _size.x, _size.y); + if (!_fbo) { + glGenFramebuffers(1, &_fbo); + } + glBindFramebuffer(GL_DRAW_FRAMEBUFFER, _fbo); + glFramebufferRenderbuffer(GL_DRAW_FRAMEBUFFER, GL_DEPTH_ATTACHMENT, GL_RENDERBUFFER, _depthStencil); + glBindFramebuffer(GL_DRAW_FRAMEBUFFER, 0); } - glGenRenderbuffers(1, &_depthStencil); - glBindRenderbuffer(GL_RENDERBUFFER, _depthStencil); - glRenderbufferStorage(GL_RENDERBUFFER, GL_DEPTH_COMPONENT16, _size.x, _size.y); - if (!_fbo) { - glGenFramebuffers(1, &_fbo); - } - glBindFramebuffer(GL_DRAW_FRAMEBUFFER, _fbo); - glFramebufferRenderbuffer(GL_DRAW_FRAMEBUFFER, GL_DEPTH_ATTACHMENT, GL_RENDERBUFFER, _depthStencil); - glBindFramebuffer(GL_DRAW_FRAMEBUFFER, 0); - } - _canvas->doneCurrent(); + _canvas->doneCurrent(); + }); } QQuickItem* OffscreenQmlSurface::getRootItem() { From d2ed3caf02e47f344e38d6d25e2d4a313154e2e3 Mon Sep 17 00:00:00 2001 From: Zach Pomerantz Date: Wed, 7 Dec 2016 13:40:22 -0500 Subject: [PATCH 28/48] respect audio thread pool size setting --- assignment-client/src/audio/AudioMixer.cpp | 7 +++++-- assignment-client/src/audio/AudioMixerSlavePool.cpp | 2 +- 2 files changed, 6 insertions(+), 3 deletions(-) diff --git a/assignment-client/src/audio/AudioMixer.cpp b/assignment-client/src/audio/AudioMixer.cpp index 9608f7c63b..01715497b1 100644 --- a/assignment-client/src/audio/AudioMixer.cpp +++ b/assignment-client/src/audio/AudioMixer.cpp @@ -512,9 +512,12 @@ void AudioMixer::parseSettingsObject(const QJsonObject &settingsObject) { const QString AUTO_THREADS = "auto_threads"; bool autoThreads = audioThreadingGroupObject[AUTO_THREADS].toBool(); if (!autoThreads) { + bool ok; const QString NUM_THREADS = "num_threads"; - int numThreads = audioThreadingGroupObject[NUM_THREADS].toInt(); - _slavePool.setNumThreads(numThreads); + int numThreads = audioThreadingGroupObject[NUM_THREADS].toString().toInt(&ok); + if (ok) { + _slavePool.setNumThreads(numThreads); + } } } diff --git a/assignment-client/src/audio/AudioMixerSlavePool.cpp b/assignment-client/src/audio/AudioMixerSlavePool.cpp index 6d17028de3..6403ebc4d8 100644 --- a/assignment-client/src/audio/AudioMixerSlavePool.cpp +++ b/assignment-client/src/audio/AudioMixerSlavePool.cpp @@ -113,7 +113,7 @@ void AudioMixerSlavePool::setNumThreads(int numThreads) { int clampedThreads = std::min(std::max(1, numThreads), maxThreads); if (clampedThreads != numThreads) { - qWarning("%s: clamped to %d (was %d)", __FUNCTION__, numThreads, clampedThreads); + qWarning("%s: clamped to %d (was %d)", __FUNCTION__, clampedThreads, numThreads); numThreads = clampedThreads; } } From f7e8d47426278f0f0e4e9fd22aefff079d7cea6f Mon Sep 17 00:00:00 2001 From: Chris Collins Date: Wed, 7 Dec 2016 12:21:13 -0800 Subject: [PATCH 29/48] Revert "Updated the Qt audio plugins" This reverts commit b888ce890c732f416362ea09185add9c8b468e69. --- cmake/externals/wasapi/CMakeLists.txt | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/cmake/externals/wasapi/CMakeLists.txt b/cmake/externals/wasapi/CMakeLists.txt index 7cfca4f3ba..67f47d68fc 100644 --- a/cmake/externals/wasapi/CMakeLists.txt +++ b/cmake/externals/wasapi/CMakeLists.txt @@ -6,8 +6,8 @@ if (WIN32) include(ExternalProject) ExternalProject_Add( ${EXTERNAL_NAME} - URL http://hifi-public.s3.amazonaws.com/dependencies/qtaudio_wasapi5.zip - URL_MD5 0530753e855ffc00232cc969bf1c84a8 + URL http://hifi-public.s3.amazonaws.com/dependencies/qtaudio_wasapi4.zip + URL_MD5 2abde5340a64d387848f12b9536a7e85 CONFIGURE_COMMAND "" BUILD_COMMAND "" INSTALL_COMMAND "" From 0fcb8803b6a28b2dbe4ef13810b50b35decb5187 Mon Sep 17 00:00:00 2001 From: Faye Li Si Fi Date: Wed, 7 Dec 2016 12:27:31 -0800 Subject: [PATCH 30/48] attempt to expose last edited property to script --- libraries/entities/src/EntityItemProperties.cpp | 1 + libraries/entities/src/EntityPropertyFlags.h | 1 + 2 files changed, 2 insertions(+) diff --git a/libraries/entities/src/EntityItemProperties.cpp b/libraries/entities/src/EntityItemProperties.cpp index e20bd81ab3..ed908ffd85 100644 --- a/libraries/entities/src/EntityItemProperties.cpp +++ b/libraries/entities/src/EntityItemProperties.cpp @@ -369,6 +369,7 @@ QScriptValue EntityItemProperties::copyToScriptValue(QScriptEngine* engine, bool COPY_PROPERTY_TO_QSCRIPTVALUE_GETTER_NO_SKIP(ageAsText, formatSecondsElapsed(getAge())); // gettable, but not settable } + COPY_PROPERTY_TO_QSCRIPTVALUE(PROP_LAST_EDITED, lastEdited); COPY_PROPERTY_TO_QSCRIPTVALUE(PROP_LAST_EDITED_BY, lastEditedBy); COPY_PROPERTY_TO_QSCRIPTVALUE(PROP_POSITION, position); COPY_PROPERTY_TO_QSCRIPTVALUE(PROP_DIMENSIONS, dimensions); diff --git a/libraries/entities/src/EntityPropertyFlags.h b/libraries/entities/src/EntityPropertyFlags.h index 127f3d0eea..45eff36338 100644 --- a/libraries/entities/src/EntityPropertyFlags.h +++ b/libraries/entities/src/EntityPropertyFlags.h @@ -182,6 +182,7 @@ enum EntityPropertyList { PROP_LOCAL_ANGULAR_VELOCITY, // only used to convert values to and from scripts PROP_LAST_EDITED_BY, + PROP_LAST_EDITED, //////////////////////////////////////////////////////////////////////////////////////////////////// // ATTENTION: add new properties to end of list just ABOVE this line From 0556661d1362886e6c917882fb98c9e8aff1eb83 Mon Sep 17 00:00:00 2001 From: Faye Li Si Fi Date: Wed, 7 Dec 2016 15:40:21 -0800 Subject: [PATCH 31/48] getting rid of uneccessary flag, fix formatting, and getting it to work yay --- libraries/entities/src/EntityItem.cpp | 1 + libraries/entities/src/EntityItemProperties.cpp | 2 +- libraries/entities/src/EntityPropertyFlags.h | 1 - 3 files changed, 2 insertions(+), 2 deletions(-) diff --git a/libraries/entities/src/EntityItem.cpp b/libraries/entities/src/EntityItem.cpp index 67a7417f9f..13bc1ac41f 100644 --- a/libraries/entities/src/EntityItem.cpp +++ b/libraries/entities/src/EntityItem.cpp @@ -1165,6 +1165,7 @@ EntityItemProperties EntityItem::getProperties(EntityPropertyFlags desiredProper properties._id = getID(); properties._idSet = true; properties._created = _created; + properties._lastEdited = _lastEdited; properties.setClientOnly(_clientOnly); properties.setOwningAvatarID(_owningAvatarID); diff --git a/libraries/entities/src/EntityItemProperties.cpp b/libraries/entities/src/EntityItemProperties.cpp index ed908ffd85..ca3480d3f0 100644 --- a/libraries/entities/src/EntityItemProperties.cpp +++ b/libraries/entities/src/EntityItemProperties.cpp @@ -369,7 +369,7 @@ QScriptValue EntityItemProperties::copyToScriptValue(QScriptEngine* engine, bool COPY_PROPERTY_TO_QSCRIPTVALUE_GETTER_NO_SKIP(ageAsText, formatSecondsElapsed(getAge())); // gettable, but not settable } - COPY_PROPERTY_TO_QSCRIPTVALUE(PROP_LAST_EDITED, lastEdited); + COPY_PROPERTY_TO_QSCRIPTVALUE(PROP_LAST_EDITED_BY, lastEditedBy); COPY_PROPERTY_TO_QSCRIPTVALUE(PROP_POSITION, position); COPY_PROPERTY_TO_QSCRIPTVALUE(PROP_DIMENSIONS, dimensions); diff --git a/libraries/entities/src/EntityPropertyFlags.h b/libraries/entities/src/EntityPropertyFlags.h index 45eff36338..127f3d0eea 100644 --- a/libraries/entities/src/EntityPropertyFlags.h +++ b/libraries/entities/src/EntityPropertyFlags.h @@ -182,7 +182,6 @@ enum EntityPropertyList { PROP_LOCAL_ANGULAR_VELOCITY, // only used to convert values to and from scripts PROP_LAST_EDITED_BY, - PROP_LAST_EDITED, //////////////////////////////////////////////////////////////////////////////////////////////////// // ATTENTION: add new properties to end of list just ABOVE this line From b6798adb5c823ca3963c04eadabeb88949aad66b Mon Sep 17 00:00:00 2001 From: Faye Li Si Fi Date: Wed, 7 Dec 2016 15:40:37 -0800 Subject: [PATCH 32/48] yay --- libraries/entities/src/EntityItemProperties.cpp | 2 ++ 1 file changed, 2 insertions(+) diff --git a/libraries/entities/src/EntityItemProperties.cpp b/libraries/entities/src/EntityItemProperties.cpp index ca3480d3f0..5e7de319cc 100644 --- a/libraries/entities/src/EntityItemProperties.cpp +++ b/libraries/entities/src/EntityItemProperties.cpp @@ -370,6 +370,8 @@ QScriptValue EntityItemProperties::copyToScriptValue(QScriptEngine* engine, bool } + QScriptValue V = convertScriptValue(engine, _lastEdited); + properties.setProperty("lastEdited", V); COPY_PROPERTY_TO_QSCRIPTVALUE(PROP_LAST_EDITED_BY, lastEditedBy); COPY_PROPERTY_TO_QSCRIPTVALUE(PROP_POSITION, position); COPY_PROPERTY_TO_QSCRIPTVALUE(PROP_DIMENSIONS, dimensions); From ce9346f524e23e1fced13753365745344894990c Mon Sep 17 00:00:00 2001 From: Zach Pomerantz Date: Wed, 7 Dec 2016 18:43:02 -0500 Subject: [PATCH 33/48] fix thread cleanup of audio pool --- .../src/audio/AudioMixerSlavePool.cpp | 39 +++++++++++++------ .../src/audio/AudioMixerSlavePool.h | 11 ++---- 2 files changed, 32 insertions(+), 18 deletions(-) diff --git a/assignment-client/src/audio/AudioMixerSlavePool.cpp b/assignment-client/src/audio/AudioMixerSlavePool.cpp index 6403ebc4d8..9bc28ec24a 100644 --- a/assignment-client/src/audio/AudioMixerSlavePool.cpp +++ b/assignment-client/src/audio/AudioMixerSlavePool.cpp @@ -15,8 +15,8 @@ #include "AudioMixerSlavePool.h" void AudioMixerSlaveThread::run() { - while (!_stop) { - wait(); // for the audio pool to notify + while (true) { + wait(); // iterate over all available nodes SharedNodePointer node; @@ -24,7 +24,11 @@ void AudioMixerSlaveThread::run() { mix(node); } - notify(); // the audio pool we are done + bool stopping = _stop; + notify(stopping); + if (stopping) { + return; + } } } @@ -40,11 +44,14 @@ void AudioMixerSlaveThread::wait() { configure(_pool._begin, _pool._end, _pool._frame); } -void AudioMixerSlaveThread::notify() { +void AudioMixerSlaveThread::notify(bool stopping) { { Lock lock(_pool._mutex); assert(_pool._numFinished < _pool._numThreads); ++_pool._numFinished; + if (stopping) { + ++_pool._numStopped; + } } _pool._poolCondition.notify_one(); } @@ -139,18 +146,28 @@ void AudioMixerSlavePool::resize(int numThreads) { } else if (numThreads < _numThreads) { auto extraBegin = _slaves.begin() + numThreads; - // stop extra slaves... + // mark slaves to stop... auto slave = extraBegin; while (slave != _slaves.end()) { - (*slave)->stop(); + (*slave)->_stop = true; ++slave; } - // ...cycle slaves with empty queue... - _numStarted = _numFinished = 0; - _slaveCondition.notify_all(); + // ...cycle them until they do stop... + { + Lock lock(_mutex); + _numStopped = 0; + while (_numStopped != (_numThreads - numThreads)) { + _numStarted = _numFinished = _numStopped; + _slaveCondition.notify_all(); + _poolCondition.wait(lock, [&] { + assert(_numFinished <= _numThreads); + return _numFinished == _numThreads; + }); + } + } - // ...wait for them to finish... + // ...wait for threads to finish... slave = extraBegin; while (slave != _slaves.end()) { QThread* thread = reinterpret_cast(slave->get()); @@ -159,7 +176,7 @@ void AudioMixerSlavePool::resize(int numThreads) { ++slave; } - // ...and delete them + // ...and erase them _slaves.erase(extraBegin, _slaves.end()); } diff --git a/assignment-client/src/audio/AudioMixerSlavePool.h b/assignment-client/src/audio/AudioMixerSlavePool.h index 91f60f5094..33b9062b99 100644 --- a/assignment-client/src/audio/AudioMixerSlavePool.h +++ b/assignment-client/src/audio/AudioMixerSlavePool.h @@ -34,20 +34,16 @@ public: AudioMixerSlaveThread(AudioMixerSlavePool& pool) : _pool(pool) {} void run() override final; - void stop() { _stop = true; } private: friend class AudioMixerSlavePool; void wait(); - void notify(); + void notify(bool stopping); bool try_pop(SharedNodePointer& node); - // frame state AudioMixerSlavePool& _pool; - - // synchronization state - std::atomic _stop; + bool _stop; }; // Slave pool for audio mixers @@ -79,7 +75,7 @@ private: std::vector> _slaves; friend void AudioMixerSlaveThread::wait(); - friend void AudioMixerSlaveThread::notify(); + friend void AudioMixerSlaveThread::notify(bool stopping); friend bool AudioMixerSlaveThread::try_pop(SharedNodePointer& node); // synchronization state @@ -89,6 +85,7 @@ private: int _numThreads { 0 }; int _numStarted { 0 }; // guarded by _mutex int _numFinished { 0 }; // guarded by _mutex + int _numStopped { 0 }; // guarded by _mutex // frame state Queue _queue; From f5b4fac46d8e14b14cb02433f9a9687be0c7b51d Mon Sep 17 00:00:00 2001 From: Zach Pomerantz Date: Wed, 7 Dec 2016 18:47:33 -0500 Subject: [PATCH 34/48] set default audio threads to 1 --- domain-server/resources/describe-settings.json | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/domain-server/resources/describe-settings.json b/domain-server/resources/describe-settings.json index 2c9bf3ed4a..ceafedb3dc 100644 --- a/domain-server/resources/describe-settings.json +++ b/domain-server/resources/describe-settings.json @@ -988,7 +988,7 @@ "label": "Automatically determine thread count", "type": "checkbox", "help": "Allow system to determine number of threads (recommended)", - "default": true, + "default": false, "advanced": true }, { From 1fde68c86e5deb57d4a6f1476a03b271ea075511 Mon Sep 17 00:00:00 2001 From: Faye Li Si Fi Date: Wed, 7 Dec 2016 16:10:03 -0800 Subject: [PATCH 35/48] better code style --- libraries/entities/src/EntityItemProperties.cpp | 4 +--- 1 file changed, 1 insertion(+), 3 deletions(-) diff --git a/libraries/entities/src/EntityItemProperties.cpp b/libraries/entities/src/EntityItemProperties.cpp index 5e7de319cc..6ac39a19f9 100644 --- a/libraries/entities/src/EntityItemProperties.cpp +++ b/libraries/entities/src/EntityItemProperties.cpp @@ -369,9 +369,7 @@ QScriptValue EntityItemProperties::copyToScriptValue(QScriptEngine* engine, bool COPY_PROPERTY_TO_QSCRIPTVALUE_GETTER_NO_SKIP(ageAsText, formatSecondsElapsed(getAge())); // gettable, but not settable } - - QScriptValue V = convertScriptValue(engine, _lastEdited); - properties.setProperty("lastEdited", V); + properties.setProperty("lastEdited", convertScriptValue(engine, _lastEdited)); COPY_PROPERTY_TO_QSCRIPTVALUE(PROP_LAST_EDITED_BY, lastEditedBy); COPY_PROPERTY_TO_QSCRIPTVALUE(PROP_POSITION, position); COPY_PROPERTY_TO_QSCRIPTVALUE(PROP_DIMENSIONS, dimensions); From 7b0b7770513933f68d64ea19aa00a58654c807bc Mon Sep 17 00:00:00 2001 From: Atlante45 Date: Wed, 7 Dec 2016 17:29:54 -0800 Subject: [PATCH 36/48] Honor reorient and quit request from OVR --- interface/src/Application.cpp | 3 ++- libraries/plugins/src/plugins/DisplayPlugin.h | 3 ++- .../oculus/src/OculusBaseDisplayPlugin.cpp | 9 +++++++ plugins/oculus/src/OculusHelpers.cpp | 26 ++++++++++++++++--- plugins/oculus/src/OculusHelpers.h | 4 +++ 5 files changed, 40 insertions(+), 5 deletions(-) diff --git a/interface/src/Application.cpp b/interface/src/Application.cpp index c6150dcc53..b9b3b4ff82 100644 --- a/interface/src/Application.cpp +++ b/interface/src/Application.cpp @@ -4750,7 +4750,7 @@ void Application::resetSensors(bool andReload) { DependencyManager::get()->reset(); getActiveDisplayPlugin()->resetSensors(); _overlayConductor.centerUI(); - getMyAvatar()->reset(andReload); + getMyAvatar()->reset(true, andReload); QMetaObject::invokeMethod(DependencyManager::get().data(), "reset", Qt::QueuedConnection); } @@ -5735,6 +5735,7 @@ void Application::updateDisplayMode() { QObject::connect(displayPlugin.get(), &DisplayPlugin::recommendedFramebufferSizeChanged, [this](const QSize & size) { resizeGL(); }); + QObject::connect(displayPlugin.get(), &DisplayPlugin::resetSensorsRequested, this, &Application::requestReset); first = false; } diff --git a/libraries/plugins/src/plugins/DisplayPlugin.h b/libraries/plugins/src/plugins/DisplayPlugin.h index c87669c99a..6b55d8ed64 100644 --- a/libraries/plugins/src/plugins/DisplayPlugin.h +++ b/libraries/plugins/src/plugins/DisplayPlugin.h @@ -205,7 +205,8 @@ public: signals: - void recommendedFramebufferSizeChanged(const QSize & size); + void recommendedFramebufferSizeChanged(const QSize& size); + void resetSensorsRequested(); protected: void incrementPresentCount(); diff --git a/plugins/oculus/src/OculusBaseDisplayPlugin.cpp b/plugins/oculus/src/OculusBaseDisplayPlugin.cpp index 7209104a85..5db5840f43 100644 --- a/plugins/oculus/src/OculusBaseDisplayPlugin.cpp +++ b/plugins/oculus/src/OculusBaseDisplayPlugin.cpp @@ -21,6 +21,15 @@ void OculusBaseDisplayPlugin::resetSensors() { } bool OculusBaseDisplayPlugin::beginFrameRender(uint32_t frameIndex) { + handleOVREvents(); + if (quitRequested()) { + QMetaObject::invokeMethod(qApp, "quit"); + return false; + } + if (reorientRequested()) { + emit resetSensorsRequested(); + } + _currentRenderFrameInfo = FrameInfo(); _currentRenderFrameInfo.sensorSampleTime = ovr_GetTimeInSeconds(); _currentRenderFrameInfo.predictedDisplayTime = ovr_GetPredictedDisplayTime(_session, frameIndex); diff --git a/plugins/oculus/src/OculusHelpers.cpp b/plugins/oculus/src/OculusHelpers.cpp index 5fbc0db7d1..feb0896fa4 100644 --- a/plugins/oculus/src/OculusHelpers.cpp +++ b/plugins/oculus/src/OculusHelpers.cpp @@ -20,15 +20,15 @@ #include #include -using Mutex = std::mutex; -using Lock = std::unique_lock; - Q_DECLARE_LOGGING_CATEGORY(oculus) Q_LOGGING_CATEGORY(oculus, "hifi.plugins.oculus") static std::atomic refCount { 0 }; static ovrSession session { nullptr }; +static bool _quitRequested { false }; +static bool _reorientRequested { false }; + inline ovrErrorInfo getError() { ovrErrorInfo error; ovr_GetLastErrorInfo(&error); @@ -116,6 +116,26 @@ void releaseOculusSession() { #endif } +void handleOVREvents() { + if (!session) { + return; + } + + ovrSessionStatus status; + if (!OVR_SUCCESS(ovr_GetSessionStatus(session, &status))) { + return; + } + + _quitRequested = status.ShouldQuit; + _reorientRequested = status.ShouldRecenter; +} + +bool quitRequested() { + return _quitRequested; +} +bool reorientRequested() { + return _reorientRequested; +} controller::Pose ovrControllerPoseToHandPose( ovrHandType hand, diff --git a/plugins/oculus/src/OculusHelpers.h b/plugins/oculus/src/OculusHelpers.h index e55dde1034..8244add84e 100644 --- a/plugins/oculus/src/OculusHelpers.h +++ b/plugins/oculus/src/OculusHelpers.h @@ -20,6 +20,10 @@ bool oculusAvailable(); ovrSession acquireOculusSession(); void releaseOculusSession(); +void handleOVREvents(); +bool quitRequested(); +bool reorientRequested(); + // Convenience method for looping over each eye with a lambda template inline void ovr_for_each_eye(Function function) { From 6e0ad5a7ed360705640abadb1b4fa0b8575160ca Mon Sep 17 00:00:00 2001 From: Zach Pomerantz Date: Thu, 8 Dec 2016 01:52:58 +0000 Subject: [PATCH 37/48] initialize audio threads --- assignment-client/src/audio/AudioMixerSlavePool.h | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/assignment-client/src/audio/AudioMixerSlavePool.h b/assignment-client/src/audio/AudioMixerSlavePool.h index 33b9062b99..e8781950f3 100644 --- a/assignment-client/src/audio/AudioMixerSlavePool.h +++ b/assignment-client/src/audio/AudioMixerSlavePool.h @@ -43,7 +43,7 @@ private: bool try_pop(SharedNodePointer& node); AudioMixerSlavePool& _pool; - bool _stop; + bool _stop { false }; }; // Slave pool for audio mixers From 3508e717532e09a8237c8060903843df0e8658e7 Mon Sep 17 00:00:00 2001 From: Zach Pomerantz Date: Thu, 8 Dec 2016 01:53:25 +0000 Subject: [PATCH 38/48] avoid pessimizing audio slave lock contention --- .../src/audio/AudioMixerSlavePool.cpp | 25 +++++++++---------- 1 file changed, 12 insertions(+), 13 deletions(-) diff --git a/assignment-client/src/audio/AudioMixerSlavePool.cpp b/assignment-client/src/audio/AudioMixerSlavePool.cpp index 9bc28ec24a..ca6ab7960f 100644 --- a/assignment-client/src/audio/AudioMixerSlavePool.cpp +++ b/assignment-client/src/audio/AudioMixerSlavePool.cpp @@ -27,7 +27,7 @@ void AudioMixerSlaveThread::run() { bool stopping = _stop; notify(stopping); if (stopping) { - return; + break; } } } @@ -80,14 +80,13 @@ void AudioMixerSlavePool::mix(ConstIter begin, ConstIter end, unsigned int frame _queue.emplace(node); }); + // mix + _numStarted = _numFinished = 0; + _slaveCondition.notify_all(); + + // wait { Lock lock(_mutex); - - // mix - _numStarted = _numFinished = 0; - _slaveCondition.notify_all(); - - // wait _poolCondition.wait(lock, [&] { assert(_numFinished <= _numThreads); return _numFinished == _numThreads; @@ -154,12 +153,12 @@ void AudioMixerSlavePool::resize(int numThreads) { } // ...cycle them until they do stop... - { - Lock lock(_mutex); - _numStopped = 0; - while (_numStopped != (_numThreads - numThreads)) { - _numStarted = _numFinished = _numStopped; - _slaveCondition.notify_all(); + _numStopped = 0; + while (_numStopped != (_numThreads - numThreads)) { + _numStarted = _numFinished = _numStopped; + _slaveCondition.notify_all(); + { + Lock lock(_mutex); _poolCondition.wait(lock, [&] { assert(_numFinished <= _numThreads); return _numFinished == _numThreads; From 3fcd45a23625ae8d6dabf27e1fc610c816cded6b Mon Sep 17 00:00:00 2001 From: Bradley Austin Davis Date: Thu, 8 Dec 2016 10:59:14 -0800 Subject: [PATCH 39/48] Optimized method for transforming axis aligned bounding boxes --- libraries/shared/src/AABox.cpp | 52 +++++++++------------------------- 1 file changed, 13 insertions(+), 39 deletions(-) diff --git a/libraries/shared/src/AABox.cpp b/libraries/shared/src/AABox.cpp index a3afa220c9..82b815b006 100644 --- a/libraries/shared/src/AABox.cpp +++ b/libraries/shared/src/AABox.cpp @@ -568,46 +568,20 @@ void AABox::transform(const Transform& transform) { translate(transform.getTranslation()); } +// Logic based on http://clb.demon.fi/MathGeoLib/nightly/docs/AABB.cpp_code.html#471 void AABox::transform(const glm::mat4& matrix) { - auto minimum = _corner; - auto maximum = _corner + _scale; + auto halfSize = _scale * 0.5f; + auto center = _corner + halfSize; + halfSize = abs(halfSize); + auto newCenter = transformPoint(matrix, center); - glm::vec3 bottomLeftNear(minimum.x, minimum.y, minimum.z); - glm::vec3 bottomRightNear(maximum.x, minimum.y, minimum.z); - glm::vec3 bottomLeftFar(minimum.x, minimum.y, maximum.z); - glm::vec3 bottomRightFar(maximum.x, minimum.y, maximum.z); - glm::vec3 topLeftNear(minimum.x, maximum.y, minimum.z); - glm::vec3 topRightNear(maximum.x, maximum.y, minimum.z); - glm::vec3 topLeftFar(minimum.x, maximum.y, maximum.z); - glm::vec3 topRightFar(maximum.x, maximum.y, maximum.z); + auto mm = glm::transpose(glm::mat3(matrix)); + vec3 newDir = vec3( + glm::dot(glm::abs(vec3(mm[0])), halfSize), + glm::dot(glm::abs(vec3(mm[1])), halfSize), + glm::dot(glm::abs(vec3(mm[2])), halfSize) + ); - glm::vec3 bottomLeftNearTransformed = transformPoint(matrix, bottomLeftNear); - glm::vec3 bottomRightNearTransformed = transformPoint(matrix, bottomRightNear); - glm::vec3 bottomLeftFarTransformed = transformPoint(matrix, bottomLeftFar); - glm::vec3 bottomRightFarTransformed = transformPoint(matrix, bottomRightFar); - glm::vec3 topLeftNearTransformed = transformPoint(matrix, topLeftNear); - glm::vec3 topRightNearTransformed = transformPoint(matrix, topRightNear); - glm::vec3 topLeftFarTransformed = transformPoint(matrix, topLeftFar); - glm::vec3 topRightFarTransformed = transformPoint(matrix, topRightFar); - - minimum = glm::min(bottomLeftNearTransformed, - glm::min(bottomRightNearTransformed, - glm::min(bottomLeftFarTransformed, - glm::min(bottomRightFarTransformed, - glm::min(topLeftNearTransformed, - glm::min(topRightNearTransformed, - glm::min(topLeftFarTransformed, - topRightFarTransformed))))))); - - maximum = glm::max(bottomLeftNearTransformed, - glm::max(bottomRightNearTransformed, - glm::max(bottomLeftFarTransformed, - glm::max(bottomRightFarTransformed, - glm::max(topLeftNearTransformed, - glm::max(topRightNearTransformed, - glm::max(topLeftFarTransformed, - topRightFarTransformed))))))); - - _corner = minimum; - _scale = maximum - minimum; + _corner = newCenter - newDir; + _scale = newDir * 2.0f; } From eafbeb6d4416790e4632913ff60f7faf3c8a54dd Mon Sep 17 00:00:00 2001 From: Zach Pomerantz Date: Thu, 8 Dec 2016 14:10:00 -0500 Subject: [PATCH 40/48] Revert "avoid pessimizing audio slave lock contention" This reverts commit 3508e717532e09a8237c8060903843df0e8658e7. --- .../src/audio/AudioMixerSlavePool.cpp | 25 ++++++++++--------- 1 file changed, 13 insertions(+), 12 deletions(-) diff --git a/assignment-client/src/audio/AudioMixerSlavePool.cpp b/assignment-client/src/audio/AudioMixerSlavePool.cpp index ca6ab7960f..9bc28ec24a 100644 --- a/assignment-client/src/audio/AudioMixerSlavePool.cpp +++ b/assignment-client/src/audio/AudioMixerSlavePool.cpp @@ -27,7 +27,7 @@ void AudioMixerSlaveThread::run() { bool stopping = _stop; notify(stopping); if (stopping) { - break; + return; } } } @@ -80,13 +80,14 @@ void AudioMixerSlavePool::mix(ConstIter begin, ConstIter end, unsigned int frame _queue.emplace(node); }); - // mix - _numStarted = _numFinished = 0; - _slaveCondition.notify_all(); - - // wait { Lock lock(_mutex); + + // mix + _numStarted = _numFinished = 0; + _slaveCondition.notify_all(); + + // wait _poolCondition.wait(lock, [&] { assert(_numFinished <= _numThreads); return _numFinished == _numThreads; @@ -153,12 +154,12 @@ void AudioMixerSlavePool::resize(int numThreads) { } // ...cycle them until they do stop... - _numStopped = 0; - while (_numStopped != (_numThreads - numThreads)) { - _numStarted = _numFinished = _numStopped; - _slaveCondition.notify_all(); - { - Lock lock(_mutex); + { + Lock lock(_mutex); + _numStopped = 0; + while (_numStopped != (_numThreads - numThreads)) { + _numStarted = _numFinished = _numStopped; + _slaveCondition.notify_all(); _poolCondition.wait(lock, [&] { assert(_numFinished <= _numThreads); return _numFinished == _numThreads; From 5969c1eb06fabd1a282b8986507eb4e1fa00318f Mon Sep 17 00:00:00 2001 From: Zach Pomerantz Date: Thu, 8 Dec 2016 14:12:16 -0500 Subject: [PATCH 41/48] ensure all accesses of audio pool num are guarded --- .../src/audio/AudioMixerSlavePool.cpp | 28 +++++++++---------- 1 file changed, 14 insertions(+), 14 deletions(-) diff --git a/assignment-client/src/audio/AudioMixerSlavePool.cpp b/assignment-client/src/audio/AudioMixerSlavePool.cpp index 9bc28ec24a..6446092448 100644 --- a/assignment-client/src/audio/AudioMixerSlavePool.cpp +++ b/assignment-client/src/audio/AudioMixerSlavePool.cpp @@ -92,8 +92,10 @@ void AudioMixerSlavePool::mix(ConstIter begin, ConstIter end, unsigned int frame assert(_numFinished <= _numThreads); return _numFinished == _numThreads; }); + + assert(_numStarted == _numThreads); } - assert(_numStarted == _numThreads); + assert(_queue.empty()); #endif } @@ -136,6 +138,8 @@ void AudioMixerSlavePool::resize(int numThreads) { #else qDebug("%s: set %d threads (was %d)", __FUNCTION__, numThreads, _numThreads); + Lock lock(_mutex); + if (numThreads > _numThreads) { // start new slaves for (int i = 0; i < numThreads - _numThreads; ++i) { @@ -154,17 +158,14 @@ void AudioMixerSlavePool::resize(int numThreads) { } // ...cycle them until they do stop... - { - Lock lock(_mutex); - _numStopped = 0; - while (_numStopped != (_numThreads - numThreads)) { - _numStarted = _numFinished = _numStopped; - _slaveCondition.notify_all(); - _poolCondition.wait(lock, [&] { - assert(_numFinished <= _numThreads); - return _numFinished == _numThreads; - }); - } + _numStopped = 0; + while (_numStopped != (_numThreads - numThreads)) { + _numStarted = _numFinished = _numStopped; + _slaveCondition.notify_all(); + _poolCondition.wait(lock, [&] { + assert(_numFinished <= _numThreads); + return _numFinished == _numThreads; + }); } // ...wait for threads to finish... @@ -181,7 +182,6 @@ void AudioMixerSlavePool::resize(int numThreads) { } _numThreads = _numStarted = _numFinished = numThreads; -#endif - assert(_numThreads == _slaves.size()); +#endif } From 1644c2eeb43ab7d2a01fe275655c5ec8e06dc0e3 Mon Sep 17 00:00:00 2001 From: Faye Li Si Fi Date: Thu, 8 Dec 2016 11:17:38 -0800 Subject: [PATCH 42/48] added unit test --- .../developer/tests/unit_tests/entityUnitTests.js | 12 ++++++++++++ 1 file changed, 12 insertions(+) diff --git a/scripts/developer/tests/unit_tests/entityUnitTests.js b/scripts/developer/tests/unit_tests/entityUnitTests.js index ad1606c4e9..033a484663 100644 --- a/scripts/developer/tests/unit_tests/entityUnitTests.js +++ b/scripts/developer/tests/unit_tests/entityUnitTests.js @@ -50,4 +50,16 @@ describe('Entity', function() { var props = Entities.getEntityProperties(boxEntity); expect(Math.round(props.position.z)).toEqual(Math.round(newPos.z)); }); + + it("\'s last edited property working correctly", function() { + var props = Entities.getEntityProperties(boxEntity); + expect(props.lastEdited).toBeDefined(); + expect(props.lastEdited).not.toBe(0); + var prevLastEdited = props.lastEdited; + + // Now we make an edit to the entity, which should update its last edited time + Entities.editEntity(boxEntity, {color: {red: 0, green: 255, blue: 0}}); + props = Entities.getEntityProperties(boxEntity); + expect(props.lastEdited).toBeGreaterThan(prevLastEdited); + }); }); \ No newline at end of file From d851278acddef7bf52224b6092045582476a43a2 Mon Sep 17 00:00:00 2001 From: samcake Date: Thu, 8 Dec 2016 15:04:57 -0800 Subject: [PATCH 43/48] FIx the highlight issue, the gloss 2 is actually roughnees to the power of 4 --- libraries/render-utils/src/local_lights_shading.slf | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/libraries/render-utils/src/local_lights_shading.slf b/libraries/render-utils/src/local_lights_shading.slf index d3542fcdfa..0b884d8bc0 100644 --- a/libraries/render-utils/src/local_lights_shading.slf +++ b/libraries/render-utils/src/local_lights_shading.slf @@ -85,7 +85,7 @@ void main(void) { vec3 fragEyeDir = normalize(fragEyeVector.xyz); // COmpute the rougness into gloss2 once: - float fragGloss2 = pow(frag.roughness + 0.001, 2.0); + float fragGloss2 = pow(frag.roughness + 0.001, 4.0); bool withScattering = (frag.scattering * isScatteringEnabled() > 0.0); int numLightTouching = 0; From aa1d90108cf5c04a87bf92fa3b0059f99c0a8b22 Mon Sep 17 00:00:00 2001 From: Atlante45 Date: Thu, 8 Dec 2016 14:12:43 -0800 Subject: [PATCH 44/48] No updater command line argument --- interface/src/Application.cpp | 11 ++++++----- 1 file changed, 6 insertions(+), 5 deletions(-) diff --git a/interface/src/Application.cpp b/interface/src/Application.cpp index c6150dcc53..4951e4d0ec 100644 --- a/interface/src/Application.cpp +++ b/interface/src/Application.cpp @@ -591,8 +591,10 @@ Application::Application(int& argc, char** argv, QElapsedTimer& startupTimer, bo qCDebug(interfaceapp) << "[VERSION] We will use DEVELOPMENT global services."; #endif - - bool wantsSandboxRunning = shouldRunServer(); + + static const QString NO_UPDATER_ARG = "--no-updater"; + static const bool noUpdater = arguments().indexOf(NO_UPDATER_ARG) != -1; + static const bool wantsSandboxRunning = shouldRunServer(); static bool determinedSandboxState = false; static bool sandboxIsRunning = false; SandboxUtils sandboxUtils; @@ -602,11 +604,10 @@ Application::Application(int& argc, char** argv, QElapsedTimer& startupTimer, bo qCDebug(interfaceapp) << "Home sandbox appears to be running....."; determinedSandboxState = true; sandboxIsRunning = true; - }, [&, wantsSandboxRunning]() { + }, [&]() { qCDebug(interfaceapp) << "Home sandbox does not appear to be running...."; if (wantsSandboxRunning) { QString contentPath = getRunServerPath(); - bool noUpdater = SteamClient::isRunning(); SandboxUtils::runLocalSandbox(contentPath, true, RUNNING_MARKER_FILENAME, noUpdater); sandboxIsRunning = true; } @@ -1128,7 +1129,7 @@ Application::Application(int& argc, char** argv, QElapsedTimer& startupTimer, bo #endif // If launched from Steam, let it handle updates - if (!SteamClient::isRunning()) { + if (!noUpdater) { auto applicationUpdater = DependencyManager::get(); connect(applicationUpdater.data(), &AutoUpdater::newVersionIsAvailable, dialogsManager.data(), &DialogsManager::showUpdateDialog); applicationUpdater->checkForUpdate(); From 7a975709491b18c6c263f65443cbe25173c907b8 Mon Sep 17 00:00:00 2001 From: sam Date: Fri, 9 Dec 2016 02:12:31 -0800 Subject: [PATCH 45/48] Debugging the amd shader???? --- libraries/gpu-gl/src/gpu/gl/GLShader.cpp | 2 +- .../render-utils/src/LightClusterGrid.slh | 10 +++++----- .../src/LightClusterGrid_shared.slh | 16 ++++++++-------- libraries/render-utils/src/LightClusters.cpp | 18 +++++++++--------- libraries/render-utils/src/LightClusters.h | 17 ++++++++++------- .../src/lightClusters_drawClusterContent.slf | 2 +- .../lightClusters_drawClusterFromDepth.slf | 2 +- .../src/local_lights_drawOutline.slf | 2 +- .../render-utils/src/local_lights_shading.slf | 19 +++++++++++++++---- 9 files changed, 51 insertions(+), 37 deletions(-) diff --git a/libraries/gpu-gl/src/gpu/gl/GLShader.cpp b/libraries/gpu-gl/src/gpu/gl/GLShader.cpp index 5020ad38a4..1208dd9486 100644 --- a/libraries/gpu-gl/src/gpu/gl/GLShader.cpp +++ b/libraries/gpu-gl/src/gpu/gl/GLShader.cpp @@ -32,7 +32,7 @@ GLShader::~GLShader() { // GLSL version static const std::string glslVersion { - "#version 410 core" + "#version 450 core" }; // Shader domain diff --git a/libraries/render-utils/src/LightClusterGrid.slh b/libraries/render-utils/src/LightClusterGrid.slh index f4ec35a75c..c0e067ca3e 100644 --- a/libraries/render-utils/src/LightClusterGrid.slh +++ b/libraries/render-utils/src/LightClusterGrid.slh @@ -13,15 +13,15 @@ struct FrustumGrid { + mat4 eyeToGridProj; + mat4 worldToEyeMat; + mat4 eyeToWorldMat; float frustumNear; float rangeNear; float rangeFar; float frustumFar; - ivec3 dims; - float spare; - mat4 eyeToGridProj; - mat4 worldToEyeMat; - mat4 eyeToWorldMat; + vec4 dims; + // float spare; }; uniform frustumGridBuffer { diff --git a/libraries/render-utils/src/LightClusterGrid_shared.slh b/libraries/render-utils/src/LightClusterGrid_shared.slh index ed0ed8d04d..70d4ba30ff 100644 --- a/libraries/render-utils/src/LightClusterGrid_shared.slh +++ b/libraries/render-utils/src/LightClusterGrid_shared.slh @@ -73,14 +73,14 @@ vec3 frustumGrid_eyeToVolume(vec3 epos, mat4 projection, float rangeNear, float int frustumGrid_numClusters() { - return frustumGrid.dims.x * frustumGrid.dims.y * (frustumGrid.dims.z + 1); + return int(frustumGrid.dims.x) * int(frustumGrid.dims.y) * (int(frustumGrid.dims.z) + 1); } int frustumGrid_clusterToIndex(ivec3 pos) { - return pos.x + (pos.y + pos.z * frustumGrid.dims.y) * frustumGrid.dims.x; + return int(pos.x + (pos.y + pos.z * int(frustumGrid.dims.y)) * int(frustumGrid.dims.x)); } ivec3 frustumGrid_indexToCluster(int index) { - ivec3 summedDims = ivec3(frustumGrid.dims.x * frustumGrid.dims.y, frustumGrid.dims.x, 1); + ivec3 summedDims = ivec3(int(frustumGrid.dims.x) * int(frustumGrid.dims.y), int(frustumGrid.dims.x), 1); int layer = index / summedDims.x; int offsetInLayer = index % summedDims.x; ivec3 clusterPos = ivec3(offsetInLayer % summedDims.y, offsetInLayer / summedDims.y, layer); @@ -92,7 +92,7 @@ vec3 frustumGrid_clusterPosToEye(vec3 clusterPos) { vec3 cvpos = clusterPos; - vec3 volumePos = frustumGrid_gridToVolume(cvpos, frustumGrid.dims); + vec3 volumePos = frustumGrid_gridToVolume(cvpos, ivec3(frustumGrid.dims.x, frustumGrid.dims.y, frustumGrid.dims.z)); vec3 eyePos = frustumGrid_volumeToEye(volumePos, frustumGrid.eyeToGridProj, frustumGrid.rangeNear, frustumGrid.rangeFar); @@ -115,7 +115,7 @@ int frustumGrid_eyeDepthToClusterLayer(float eyeZ) { float volumeZ = frustumGrid_eyeToVolumeDepth(eyeZ, frustumGrid.rangeNear, frustumGrid.rangeFar); - float gridZ = frustumGrid_volumeToGridDepth(volumeZ, frustumGrid.dims); + float gridZ = frustumGrid_volumeToGridDepth(volumeZ, ivec3(frustumGrid.dims.x, frustumGrid.dims.y, frustumGrid.dims.z)); if (gridZ >= frustumGrid.dims.z) { gridZ = frustumGrid.dims.z; @@ -137,7 +137,7 @@ ivec3 frustumGrid_eyeToClusterPos(vec3 eyePos) { vec3 volumePos = frustumGrid_eyeToVolume(eyePos, frustumGrid.eyeToGridProj, frustumGrid.rangeNear, frustumGrid.rangeFar); - vec3 gridPos = frustumGrid_volumeToGrid(volumePos, frustumGrid.dims); + vec3 gridPos = frustumGrid_volumeToGrid(volumePos, ivec3(frustumGrid.dims.x, frustumGrid.dims.y, frustumGrid.dims.z)); if (gridPos.z >= frustumGrid.dims.z) { gridPos.z = frustumGrid.dims.z; @@ -149,7 +149,7 @@ ivec3 frustumGrid_eyeToClusterPos(vec3 eyePos) { int frustumGrid_eyeToClusterDirH(vec3 eyeDir) { if (eyeDir.z >= 0.0f) { - return (eyeDir.x > 0 ? frustumGrid.dims.x : -1); + return (eyeDir.x > 0 ? int(frustumGrid.dims.x) : -1); } float eyeDepth = -eyeDir.z; @@ -163,7 +163,7 @@ int frustumGrid_eyeToClusterDirH(vec3 eyeDir) { int frustumGrid_eyeToClusterDirV(vec3 eyeDir) { if (eyeDir.z >= 0.0f) { - return (eyeDir.y > 0 ? frustumGrid.dims.y : -1); + return (eyeDir.y > 0 ? int(frustumGrid.dims.y) : -1); } float eyeDepth = -eyeDir.z; diff --git a/libraries/render-utils/src/LightClusters.cpp b/libraries/render-utils/src/LightClusters.cpp index d3a384f1df..8dc95da65b 100644 --- a/libraries/render-utils/src/LightClusters.cpp +++ b/libraries/render-utils/src/LightClusters.cpp @@ -27,21 +27,21 @@ enum LightClusterGridShader_MapSlot { DEFERRED_BUFFER_LINEAR_DEPTH_UNIT = 0, - DEFERRED_BUFFER_COLOR_UNIT, - DEFERRED_BUFFER_NORMAL_UNIT, - DEFERRED_BUFFER_EMISSIVE_UNIT, - DEFERRED_BUFFER_DEPTH_UNIT, + DEFERRED_BUFFER_COLOR_UNIT = 1, + DEFERRED_BUFFER_NORMAL_UNIT = 2, + DEFERRED_BUFFER_EMISSIVE_UNIT = 3, + DEFERRED_BUFFER_DEPTH_UNIT = 4, }; enum LightClusterGridShader_BufferSlot { LIGHT_CLUSTER_GRID_FRUSTUM_GRID_SLOT = 0, - DEFERRED_FRAME_TRANSFORM_BUFFER_SLOT, - CAMERA_CORRECTION_BUFFER_SLOT, + DEFERRED_FRAME_TRANSFORM_BUFFER_SLOT =1, + CAMERA_CORRECTION_BUFFER_SLOT = 2, LIGHT_GPU_SLOT = render::ShapePipeline::Slot::LIGHT, - LIGHT_INDEX_GPU_SLOT, + LIGHT_INDEX_GPU_SLOT = 5, - LIGHT_CLUSTER_GRID_CLUSTER_GRID_SLOT, - LIGHT_CLUSTER_GRID_CLUSTER_CONTENT_SLOT, + LIGHT_CLUSTER_GRID_CLUSTER_GRID_SLOT = 6, + LIGHT_CLUSTER_GRID_CLUSTER_CONTENT_SLOT = 7, }; FrustumGrid::FrustumGrid(const FrustumGrid& source) : diff --git a/libraries/render-utils/src/LightClusters.h b/libraries/render-utils/src/LightClusters.h index d27a6e1d92..0839f774b0 100644 --- a/libraries/render-utils/src/LightClusters.h +++ b/libraries/render-utils/src/LightClusters.h @@ -21,18 +21,21 @@ class FrustumGrid { public: - float frustumNear { 0.1f }; - float rangeNear { 0.1f }; - float rangeFar { 200.0f }; - float frustumFar { 10000.0f }; - - glm::ivec3 dims { 1, 1, 1 }; - float spare; glm::mat4 eyeToGridProj; glm::mat4 worldToEyeMat; glm::mat4 eyeToWorldMat; + float frustumNear { 0.1f }; + float rangeNear { 0.1f }; + float rangeFar { 200.0f }; + float frustumFar { 10000.0f }; + // glm::ivec4 dims { 1, 1, 1, 1 }; + + glm::ivec3 dims { 1, 1, 1 }; + float spare; + + FrustumGrid() = default; FrustumGrid(const FrustumGrid& source); diff --git a/libraries/render-utils/src/lightClusters_drawClusterContent.slf b/libraries/render-utils/src/lightClusters_drawClusterContent.slf index 447f8bd634..4eee9fc33a 100644 --- a/libraries/render-utils/src/lightClusters_drawClusterContent.slf +++ b/libraries/render-utils/src/lightClusters_drawClusterContent.slf @@ -46,7 +46,7 @@ void main(void) { int clusterOffset = cluster.z; - ivec3 dims = frustumGrid.dims.xyz; + ivec3 dims = ivec3(frustumGrid.dims.x, frustumGrid.dims.y, frustumGrid.dims.z); dims.z +=1; ivec3 summedDims = ivec3(dims.x * dims.y, dims.x, 1); diff --git a/libraries/render-utils/src/lightClusters_drawClusterFromDepth.slf b/libraries/render-utils/src/lightClusters_drawClusterFromDepth.slf index ee2e6e0ccc..83bd76cecb 100644 --- a/libraries/render-utils/src/lightClusters_drawClusterFromDepth.slf +++ b/libraries/render-utils/src/lightClusters_drawClusterFromDepth.slf @@ -42,7 +42,7 @@ void main(void) { float numLightsScale = clamp(numLights * 0.05, 0.01, 1.0); - ivec3 dims = frustumGrid.dims.xyz; + ivec3 dims = ivec3(frustumGrid.dims.x, frustumGrid.dims.y, frustumGrid.dims.z); dims.z +=1; ivec3 summedDims = ivec3(dims.x * dims.y, dims.x, 1); diff --git a/libraries/render-utils/src/local_lights_drawOutline.slf b/libraries/render-utils/src/local_lights_drawOutline.slf index 3aa210a241..2cd211c4b6 100644 --- a/libraries/render-utils/src/local_lights_drawOutline.slf +++ b/libraries/render-utils/src/local_lights_drawOutline.slf @@ -65,7 +65,7 @@ void main(void) { } int lightClusterOffset = cluster.z; - ivec3 dims = frustumGrid.dims.xyz; + ivec3 dims = ivec3(frustumGrid.dims.x, frustumGrid.dims.y, frustumGrid.dims.z); if (clusterPos.x < 0 || clusterPos.x >= dims.x) { discard; } diff --git a/libraries/render-utils/src/local_lights_shading.slf b/libraries/render-utils/src/local_lights_shading.slf index 0b884d8bc0..43461c13ee 100644 --- a/libraries/render-utils/src/local_lights_shading.slf +++ b/libraries/render-utils/src/local_lights_shading.slf @@ -34,7 +34,6 @@ in vec2 _texCoord0; out vec4 _fragColor; void main(void) { - // Grab the fragment data from the uv vec2 texCoord = _texCoord0.st; @@ -49,19 +48,31 @@ void main(void) { // Frag pos in world mat4 invViewMat = getViewInverse(); vec4 fragPos = invViewMat * fragPosition; - + // From frag world pos find the cluster vec4 clusterEyePos = frustumGrid_worldToEye(fragPos); ivec3 clusterPos = frustumGrid_eyeToClusterPos(clusterEyePos.xyz); + vec3 volumePos = frustumGrid_eyeToVolume(clusterEyePos.xyz, frustumGrid.eyeToGridProj, frustumGrid.rangeNear, frustumGrid.rangeFar); + ivec3 dims = ivec3(frustumGrid.dims.x, frustumGrid.dims.y, frustumGrid.dims.z); + + vec3 gridPos = frustumGrid_volumeToGrid(volumePos, dims); + + _fragColor = vec4(fract(volumePos.xyz), 1.0); + // _fragColor = vec4(fract(gridPos / vec3(dims)), 1.0); + + return; + ivec3 cluster = clusterGrid_getCluster(frustumGrid_clusterToIndex(clusterPos)); + int numLights = cluster.x + cluster.y; if (numLights <= 0) { discard; } int lightClusterOffset = cluster.z; + // ivec3 dims = ivec3(frustumGrid.dims.x, frustumGrid.dims.y, frustumGrid.dims.z); + //ivec3 dims = frustumGrid.dims.xyz; - ivec3 dims = frustumGrid.dims.xyz; if (clusterPos.x < 0 || clusterPos.x >= dims.x) { discard; } @@ -84,7 +95,7 @@ void main(void) { vec4 fragEyeVector = invViewMat * vec4(-frag.position.xyz, 0.0); vec3 fragEyeDir = normalize(fragEyeVector.xyz); - // COmpute the rougness into gloss2 once: + // Compute the rougness into gloss2 once: float fragGloss2 = pow(frag.roughness + 0.001, 4.0); bool withScattering = (frag.scattering * isScatteringEnabled() > 0.0); From 914fc7fca8b23d237aef9908f4b320fa29603f39 Mon Sep 17 00:00:00 2001 From: sam Date: Fri, 9 Dec 2016 02:58:09 -0800 Subject: [PATCH 46/48] rewind all the unecessary changes and do just the minimum layout(std140) and accessing the big arrays in vec4s to fix the amd bug --- .../render-utils/src/LightClusterGrid.slh | 24 +++++++++++-------- .../src/LightClusterGrid_shared.slh | 16 ++++++------- libraries/render-utils/src/LightClusters.h | 9 +++---- .../render-utils/src/local_lights_shading.slf | 5 ---- 4 files changed, 25 insertions(+), 29 deletions(-) diff --git a/libraries/render-utils/src/LightClusterGrid.slh b/libraries/render-utils/src/LightClusterGrid.slh index c0e067ca3e..722d37814d 100644 --- a/libraries/render-utils/src/LightClusterGrid.slh +++ b/libraries/render-utils/src/LightClusterGrid.slh @@ -13,18 +13,18 @@ struct FrustumGrid { - mat4 eyeToGridProj; - mat4 worldToEyeMat; - mat4 eyeToWorldMat; float frustumNear; float rangeNear; float rangeFar; float frustumFar; - vec4 dims; - // float spare; + ivec3 dims; + float spare; + mat4 eyeToGridProj; + mat4 worldToEyeMat; + mat4 eyeToWorldMat; }; -uniform frustumGridBuffer { +layout(std140) uniform frustumGridBuffer { FrustumGrid frustumGrid; }; @@ -51,16 +51,20 @@ float projection_getFar(mat4 projection) { #define GRID_INDEX_TYPE ivec4 #define GRID_FETCH_BUFFER(i) i / 4][i % 4 <@else@> -#define GRID_NUM_ELEMENTS 16384 +#define GRID_NUM_ELEMENTS 4096 +#define GRID_INDEX_TYPE ivec4 +#define GRID_FETCH_BUFFER(i) i / 4][i % 4 + + <@endif@> -uniform clusterGridBuffer { +layout(std140) uniform clusterGridBuffer { GRID_INDEX_TYPE _clusterGridTable[GRID_NUM_ELEMENTS]; }; -uniform clusterContentBuffer { +layout(std140) uniform clusterContentBuffer { GRID_INDEX_TYPE _clusterGridContent[GRID_NUM_ELEMENTS]; }; diff --git a/libraries/render-utils/src/LightClusterGrid_shared.slh b/libraries/render-utils/src/LightClusterGrid_shared.slh index 70d4ba30ff..ed0ed8d04d 100644 --- a/libraries/render-utils/src/LightClusterGrid_shared.slh +++ b/libraries/render-utils/src/LightClusterGrid_shared.slh @@ -73,14 +73,14 @@ vec3 frustumGrid_eyeToVolume(vec3 epos, mat4 projection, float rangeNear, float int frustumGrid_numClusters() { - return int(frustumGrid.dims.x) * int(frustumGrid.dims.y) * (int(frustumGrid.dims.z) + 1); + return frustumGrid.dims.x * frustumGrid.dims.y * (frustumGrid.dims.z + 1); } int frustumGrid_clusterToIndex(ivec3 pos) { - return int(pos.x + (pos.y + pos.z * int(frustumGrid.dims.y)) * int(frustumGrid.dims.x)); + return pos.x + (pos.y + pos.z * frustumGrid.dims.y) * frustumGrid.dims.x; } ivec3 frustumGrid_indexToCluster(int index) { - ivec3 summedDims = ivec3(int(frustumGrid.dims.x) * int(frustumGrid.dims.y), int(frustumGrid.dims.x), 1); + ivec3 summedDims = ivec3(frustumGrid.dims.x * frustumGrid.dims.y, frustumGrid.dims.x, 1); int layer = index / summedDims.x; int offsetInLayer = index % summedDims.x; ivec3 clusterPos = ivec3(offsetInLayer % summedDims.y, offsetInLayer / summedDims.y, layer); @@ -92,7 +92,7 @@ vec3 frustumGrid_clusterPosToEye(vec3 clusterPos) { vec3 cvpos = clusterPos; - vec3 volumePos = frustumGrid_gridToVolume(cvpos, ivec3(frustumGrid.dims.x, frustumGrid.dims.y, frustumGrid.dims.z)); + vec3 volumePos = frustumGrid_gridToVolume(cvpos, frustumGrid.dims); vec3 eyePos = frustumGrid_volumeToEye(volumePos, frustumGrid.eyeToGridProj, frustumGrid.rangeNear, frustumGrid.rangeFar); @@ -115,7 +115,7 @@ int frustumGrid_eyeDepthToClusterLayer(float eyeZ) { float volumeZ = frustumGrid_eyeToVolumeDepth(eyeZ, frustumGrid.rangeNear, frustumGrid.rangeFar); - float gridZ = frustumGrid_volumeToGridDepth(volumeZ, ivec3(frustumGrid.dims.x, frustumGrid.dims.y, frustumGrid.dims.z)); + float gridZ = frustumGrid_volumeToGridDepth(volumeZ, frustumGrid.dims); if (gridZ >= frustumGrid.dims.z) { gridZ = frustumGrid.dims.z; @@ -137,7 +137,7 @@ ivec3 frustumGrid_eyeToClusterPos(vec3 eyePos) { vec3 volumePos = frustumGrid_eyeToVolume(eyePos, frustumGrid.eyeToGridProj, frustumGrid.rangeNear, frustumGrid.rangeFar); - vec3 gridPos = frustumGrid_volumeToGrid(volumePos, ivec3(frustumGrid.dims.x, frustumGrid.dims.y, frustumGrid.dims.z)); + vec3 gridPos = frustumGrid_volumeToGrid(volumePos, frustumGrid.dims); if (gridPos.z >= frustumGrid.dims.z) { gridPos.z = frustumGrid.dims.z; @@ -149,7 +149,7 @@ ivec3 frustumGrid_eyeToClusterPos(vec3 eyePos) { int frustumGrid_eyeToClusterDirH(vec3 eyeDir) { if (eyeDir.z >= 0.0f) { - return (eyeDir.x > 0 ? int(frustumGrid.dims.x) : -1); + return (eyeDir.x > 0 ? frustumGrid.dims.x : -1); } float eyeDepth = -eyeDir.z; @@ -163,7 +163,7 @@ int frustumGrid_eyeToClusterDirH(vec3 eyeDir) { int frustumGrid_eyeToClusterDirV(vec3 eyeDir) { if (eyeDir.z >= 0.0f) { - return (eyeDir.y > 0 ? int(frustumGrid.dims.y) : -1); + return (eyeDir.y > 0 ? frustumGrid.dims.y : -1); } float eyeDepth = -eyeDir.z; diff --git a/libraries/render-utils/src/LightClusters.h b/libraries/render-utils/src/LightClusters.h index 0839f774b0..d27a6e1d92 100644 --- a/libraries/render-utils/src/LightClusters.h +++ b/libraries/render-utils/src/LightClusters.h @@ -21,20 +21,17 @@ class FrustumGrid { public: - - glm::mat4 eyeToGridProj; - glm::mat4 worldToEyeMat; - glm::mat4 eyeToWorldMat; - float frustumNear { 0.1f }; float rangeNear { 0.1f }; float rangeFar { 200.0f }; float frustumFar { 10000.0f }; - // glm::ivec4 dims { 1, 1, 1, 1 }; glm::ivec3 dims { 1, 1, 1 }; float spare; + glm::mat4 eyeToGridProj; + glm::mat4 worldToEyeMat; + glm::mat4 eyeToWorldMat; FrustumGrid() = default; FrustumGrid(const FrustumGrid& source); diff --git a/libraries/render-utils/src/local_lights_shading.slf b/libraries/render-utils/src/local_lights_shading.slf index 43461c13ee..4f5459e0ef 100644 --- a/libraries/render-utils/src/local_lights_shading.slf +++ b/libraries/render-utils/src/local_lights_shading.slf @@ -58,11 +58,6 @@ void main(void) { vec3 gridPos = frustumGrid_volumeToGrid(volumePos, dims); - _fragColor = vec4(fract(volumePos.xyz), 1.0); - // _fragColor = vec4(fract(gridPos / vec3(dims)), 1.0); - - return; - ivec3 cluster = clusterGrid_getCluster(frustumGrid_clusterToIndex(clusterPos)); int numLights = cluster.x + cluster.y; From 2427a16d6e351fc7ad66acd7ff24f9a4aaa70790 Mon Sep 17 00:00:00 2001 From: sam Date: Fri, 9 Dec 2016 03:05:25 -0800 Subject: [PATCH 47/48] clean up and less changes --- libraries/gpu-gl/src/gpu/gl/GLShader.cpp | 2 +- .../src/lightClusters_drawClusterContent.slf | 2 +- .../src/lightClusters_drawClusterFromDepth.slf | 2 +- libraries/render-utils/src/local_lights_drawOutline.slf | 2 +- libraries/render-utils/src/local_lights_shading.slf | 9 +-------- 5 files changed, 5 insertions(+), 12 deletions(-) diff --git a/libraries/gpu-gl/src/gpu/gl/GLShader.cpp b/libraries/gpu-gl/src/gpu/gl/GLShader.cpp index 1208dd9486..5020ad38a4 100644 --- a/libraries/gpu-gl/src/gpu/gl/GLShader.cpp +++ b/libraries/gpu-gl/src/gpu/gl/GLShader.cpp @@ -32,7 +32,7 @@ GLShader::~GLShader() { // GLSL version static const std::string glslVersion { - "#version 450 core" + "#version 410 core" }; // Shader domain diff --git a/libraries/render-utils/src/lightClusters_drawClusterContent.slf b/libraries/render-utils/src/lightClusters_drawClusterContent.slf index 4eee9fc33a..447f8bd634 100644 --- a/libraries/render-utils/src/lightClusters_drawClusterContent.slf +++ b/libraries/render-utils/src/lightClusters_drawClusterContent.slf @@ -46,7 +46,7 @@ void main(void) { int clusterOffset = cluster.z; - ivec3 dims = ivec3(frustumGrid.dims.x, frustumGrid.dims.y, frustumGrid.dims.z); + ivec3 dims = frustumGrid.dims.xyz; dims.z +=1; ivec3 summedDims = ivec3(dims.x * dims.y, dims.x, 1); diff --git a/libraries/render-utils/src/lightClusters_drawClusterFromDepth.slf b/libraries/render-utils/src/lightClusters_drawClusterFromDepth.slf index 83bd76cecb..ee2e6e0ccc 100644 --- a/libraries/render-utils/src/lightClusters_drawClusterFromDepth.slf +++ b/libraries/render-utils/src/lightClusters_drawClusterFromDepth.slf @@ -42,7 +42,7 @@ void main(void) { float numLightsScale = clamp(numLights * 0.05, 0.01, 1.0); - ivec3 dims = ivec3(frustumGrid.dims.x, frustumGrid.dims.y, frustumGrid.dims.z); + ivec3 dims = frustumGrid.dims.xyz; dims.z +=1; ivec3 summedDims = ivec3(dims.x * dims.y, dims.x, 1); diff --git a/libraries/render-utils/src/local_lights_drawOutline.slf b/libraries/render-utils/src/local_lights_drawOutline.slf index 2cd211c4b6..3aa210a241 100644 --- a/libraries/render-utils/src/local_lights_drawOutline.slf +++ b/libraries/render-utils/src/local_lights_drawOutline.slf @@ -65,7 +65,7 @@ void main(void) { } int lightClusterOffset = cluster.z; - ivec3 dims = ivec3(frustumGrid.dims.x, frustumGrid.dims.y, frustumGrid.dims.z); + ivec3 dims = frustumGrid.dims.xyz; if (clusterPos.x < 0 || clusterPos.x >= dims.x) { discard; } diff --git a/libraries/render-utils/src/local_lights_shading.slf b/libraries/render-utils/src/local_lights_shading.slf index 4f5459e0ef..a4e28a9757 100644 --- a/libraries/render-utils/src/local_lights_shading.slf +++ b/libraries/render-utils/src/local_lights_shading.slf @@ -53,21 +53,14 @@ void main(void) { vec4 clusterEyePos = frustumGrid_worldToEye(fragPos); ivec3 clusterPos = frustumGrid_eyeToClusterPos(clusterEyePos.xyz); - vec3 volumePos = frustumGrid_eyeToVolume(clusterEyePos.xyz, frustumGrid.eyeToGridProj, frustumGrid.rangeNear, frustumGrid.rangeFar); - ivec3 dims = ivec3(frustumGrid.dims.x, frustumGrid.dims.y, frustumGrid.dims.z); - - vec3 gridPos = frustumGrid_volumeToGrid(volumePos, dims); - ivec3 cluster = clusterGrid_getCluster(frustumGrid_clusterToIndex(clusterPos)); - int numLights = cluster.x + cluster.y; if (numLights <= 0) { discard; } int lightClusterOffset = cluster.z; - // ivec3 dims = ivec3(frustumGrid.dims.x, frustumGrid.dims.y, frustumGrid.dims.z); - //ivec3 dims = frustumGrid.dims.xyz; + ivec3 dims = frustumGrid.dims.xyz; if (clusterPos.x < 0 || clusterPos.x >= dims.x) { discard; } From 5def039bd46b42f852fea300ac059476194478cf Mon Sep 17 00:00:00 2001 From: Atlante45 Date: Fri, 9 Dec 2016 14:55:19 -0800 Subject: [PATCH 48/48] Swap start/back xbox controller mapping to conform to Oculus standard --- interface/resources/controllers/xbox.json | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/interface/resources/controllers/xbox.json b/interface/resources/controllers/xbox.json index 51690dda30..0b4a992fa7 100644 --- a/interface/resources/controllers/xbox.json +++ b/interface/resources/controllers/xbox.json @@ -44,8 +44,8 @@ { "from": "GamePad.RB", "to": "Standard.RB" }, { "from": "GamePad.RS", "to": "Standard.RS" }, - { "from": "GamePad.Start", "to": "Actions.CycleCamera" }, - { "from": "GamePad.Back", "to": "Standard.Start" }, + { "from": "GamePad.Start", "to": "Standard.Start" }, + { "from": "GamePad.Back", "to": "Actions.CycleCamera" }, { "from": "GamePad.DU", "to": "Standard.DU" }, { "from": "GamePad.DD", "to": "Standard.DD" },