From e2ff738fc3e37b431046de65f6adb93a97336541 Mon Sep 17 00:00:00 2001 From: Zach Pomerantz Date: Thu, 8 Sep 2016 17:57:41 -0700 Subject: [PATCH 1/4] strip unused audio settings - remove parameters on the jitter buffer algorithm, these are now static - remove parameters on the output starve detection algorithm, now static - move interface audio settings to the Developer menu --- assignment-client/src/Agent.cpp | 7 +- assignment-client/src/audio/AudioMixer.cpp | 97 +++++---- assignment-client/src/audio/AudioMixer.h | 4 +- .../src/audio/AudioMixerClientData.cpp | 4 +- .../src/audio/AvatarAudioStream.cpp | 6 +- .../src/audio/AvatarAudioStream.h | 2 +- .../resources/describe-settings.json | 52 ++--- domain-server/resources/web/css/style.css | 2 +- .../resources/web/settings/index.shtml | 3 +- .../resources/web/settings/js/settings.js | 10 +- interface/src/Menu.cpp | 12 +- interface/src/ui/PreferencesDialog.cpp | 86 ++------ libraries/audio-client/src/AudioClient.cpp | 77 ++++--- libraries/audio-client/src/AudioClient.h | 23 +-- libraries/audio/src/AudioLogging.cpp | 1 + libraries/audio/src/InboundAudioStream.cpp | 191 ++++++------------ libraries/audio/src/InboundAudioStream.h | 177 ++++------------ libraries/audio/src/InjectedAudioStream.cpp | 9 +- libraries/audio/src/InjectedAudioStream.h | 2 +- libraries/audio/src/MixedAudioStream.cpp | 6 +- libraries/audio/src/MixedAudioStream.h | 2 +- .../audio/src/MixedProcessedAudioStream.cpp | 6 +- .../audio/src/MixedProcessedAudioStream.h | 2 +- libraries/audio/src/PositionalAudioStream.cpp | 8 +- libraries/audio/src/PositionalAudioStream.h | 2 +- libraries/shared/src/SettingHandle.h | 38 +++- 26 files changed, 294 insertions(+), 535 deletions(-) diff --git a/assignment-client/src/Agent.cpp b/assignment-client/src/Agent.cpp index 350d359412..c6a2a3d5e8 100644 --- a/assignment-client/src/Agent.cpp +++ b/assignment-client/src/Agent.cpp @@ -48,11 +48,8 @@ static const int RECEIVED_AUDIO_STREAM_CAPACITY_FRAMES = 10; Agent::Agent(ReceivedMessage& message) : ThreadedAssignment(message), _entityEditSender(), - _receivedAudioStream(AudioConstants::NETWORK_FRAME_SAMPLES_STEREO, RECEIVED_AUDIO_STREAM_CAPACITY_FRAMES, - InboundAudioStream::Settings(0, false, RECEIVED_AUDIO_STREAM_CAPACITY_FRAMES, false, - DEFAULT_WINDOW_STARVE_THRESHOLD, DEFAULT_WINDOW_SECONDS_FOR_DESIRED_CALC_ON_TOO_MANY_STARVES, - DEFAULT_WINDOW_SECONDS_FOR_DESIRED_REDUCTION, false)) -{ + _receivedAudioStream(AudioConstants::NETWORK_FRAME_SAMPLES_STEREO, + RECEIVED_AUDIO_STREAM_CAPACITY_FRAMES, RECEIVED_AUDIO_STREAM_CAPACITY_FRAMES) { DependencyManager::get()->setPacketSender(&_entityEditSender); ResourceManager::init(); diff --git a/assignment-client/src/audio/AudioMixer.cpp b/assignment-client/src/audio/AudioMixer.cpp index eabb4955d9..bf211047f8 100644 --- a/assignment-client/src/audio/AudioMixer.cpp +++ b/assignment-client/src/audio/AudioMixer.cpp @@ -61,15 +61,14 @@ #include "AudioMixer.h" -const float LOUDNESS_TO_DISTANCE_RATIO = 0.00001f; -const float DEFAULT_ATTENUATION_PER_DOUBLING_IN_DISTANCE = 0.5f; // attenuation = -6dB * log2(distance) -const float DEFAULT_NOISE_MUTING_THRESHOLD = 0.003f; -const QString AUDIO_MIXER_LOGGING_TARGET_NAME = "audio-mixer"; -const QString AUDIO_ENV_GROUP_KEY = "audio_env"; -const QString AUDIO_BUFFER_GROUP_KEY = "audio_buffer"; - -InboundAudioStream::Settings AudioMixer::_streamSettings; +static const float LOUDNESS_TO_DISTANCE_RATIO = 0.00001f; +static const float DEFAULT_ATTENUATION_PER_DOUBLING_IN_DISTANCE = 0.5f; // attenuation = -6dB * log2(distance) +static const float DEFAULT_NOISE_MUTING_THRESHOLD = 0.003f; +static const QString AUDIO_MIXER_LOGGING_TARGET_NAME = "audio-mixer"; +static const QString AUDIO_ENV_GROUP_KEY = "audio_env"; +static const QString AUDIO_BUFFER_GROUP_KEY = "audio_buffer"; +int AudioMixer::_numStaticJitterFrames{ -1 }; bool AudioMixer::_enableFilter = true; bool AudioMixer::shouldMute(float quietestFrame) { @@ -269,7 +268,7 @@ void AudioMixer::addStreamToMixForListeningNodeWithStream(AudioMixerClientData& if (!streamToAdd.lastPopSucceeded()) { bool forceSilentBlock = true; - if (_streamSettings._repetitionWithFade && !streamToAdd.getLastPopOutput().isNull()) { + if (!streamToAdd.getLastPopOutput().isNull()) { // reptition with fade is enabled, and we do have a valid previous frame to repeat // so we mix the previously-mixed block @@ -641,7 +640,7 @@ QString AudioMixer::percentageForMixStats(int counter) { void AudioMixer::sendStatsPacket() { static QJsonObject statsObject; - statsObject["useDynamicJitterBuffers"] = _streamSettings._dynamicJitterBuffers; + statsObject["useDynamicJitterBuffers"] = _numStaticJitterFrames == -1; statsObject["trailing_sleep_percentage"] = _trailingSleepRatio * 100.0f; statsObject["performance_throttling_ratio"] = _performanceThrottlingRatio; @@ -899,66 +898,66 @@ void AudioMixer::broadcastMixes() { void AudioMixer::parseSettingsObject(const QJsonObject &settingsObject) { if (settingsObject.contains(AUDIO_BUFFER_GROUP_KEY)) { QJsonObject audioBufferGroupObject = settingsObject[AUDIO_BUFFER_GROUP_KEY].toObject(); + qDebug() << audioBufferGroupObject; // check the payload to see if we have asked for dynamicJitterBuffer support const QString DYNAMIC_JITTER_BUFFER_JSON_KEY = "dynamic_jitter_buffer"; - _streamSettings._dynamicJitterBuffers = audioBufferGroupObject[DYNAMIC_JITTER_BUFFER_JSON_KEY].toBool(); - if (_streamSettings._dynamicJitterBuffers) { - qDebug() << "Enable dynamic jitter buffers."; + bool enableDynamicJitterBuffer = audioBufferGroupObject[DYNAMIC_JITTER_BUFFER_JSON_KEY].toBool(); + if (enableDynamicJitterBuffer) { + qDebug() << "Enabling dynamic jitter buffers."; + + bool ok; + const QString DESIRED_JITTER_BUFFER_FRAMES_KEY = "static_desired_jitter_buffer_frames"; + _numStaticJitterFrames = audioBufferGroupObject[DESIRED_JITTER_BUFFER_FRAMES_KEY].toString().toInt(&ok); + if (!ok) { + _numStaticJitterFrames = InboundAudioStream::DEFAULT_STATIC_JITTER_FRAMES; + } + qDebug() << "Static desired jitter buffer frames:" << _numStaticJitterFrames; } else { - qDebug() << "Dynamic jitter buffers disabled."; + qDebug() << "Disabling dynamic jitter buffers."; + _numStaticJitterFrames = -1; } + // check for deprecated audio settings + auto deprecationNotice = [](const QString& setting, const QString& value) { + qInfo().nospace() << "[DEPRECATION NOTICE] " << setting << "(" << value << ") has been deprecated, and has no effect"; + }; bool ok; - const QString DESIRED_JITTER_BUFFER_FRAMES_KEY = "static_desired_jitter_buffer_frames"; - _streamSettings._staticDesiredJitterBufferFrames = audioBufferGroupObject[DESIRED_JITTER_BUFFER_FRAMES_KEY].toString().toInt(&ok); - if (!ok) { - _streamSettings._staticDesiredJitterBufferFrames = DEFAULT_STATIC_DESIRED_JITTER_BUFFER_FRAMES; - } - qDebug() << "Static desired jitter buffer frames:" << _streamSettings._staticDesiredJitterBufferFrames; const QString MAX_FRAMES_OVER_DESIRED_JSON_KEY = "max_frames_over_desired"; - _streamSettings._maxFramesOverDesired = audioBufferGroupObject[MAX_FRAMES_OVER_DESIRED_JSON_KEY].toString().toInt(&ok); - if (!ok) { - _streamSettings._maxFramesOverDesired = DEFAULT_MAX_FRAMES_OVER_DESIRED; - } - qDebug() << "Max frames over desired:" << _streamSettings._maxFramesOverDesired; - - const QString USE_STDEV_FOR_DESIRED_CALC_JSON_KEY = "use_stdev_for_desired_calc"; - _streamSettings._useStDevForJitterCalc = audioBufferGroupObject[USE_STDEV_FOR_DESIRED_CALC_JSON_KEY].toBool(); - if (_streamSettings._useStDevForJitterCalc) { - qDebug() << "Using stdev method for jitter calc if dynamic jitter buffers enabled"; - } else { - qDebug() << "Using max-gap method for jitter calc if dynamic jitter buffers enabled"; + int maxFramesOverDesired = audioBufferGroupObject[MAX_FRAMES_OVER_DESIRED_JSON_KEY].toString().toInt(&ok); + if (ok && maxFramesOverDesired != InboundAudioStream::MAX_FRAMES_OVER_DESIRED) { + deprecationNotice(MAX_FRAMES_OVER_DESIRED_JSON_KEY, QString::number(maxFramesOverDesired)); } const QString WINDOW_STARVE_THRESHOLD_JSON_KEY = "window_starve_threshold"; - _streamSettings._windowStarveThreshold = audioBufferGroupObject[WINDOW_STARVE_THRESHOLD_JSON_KEY].toString().toInt(&ok); - if (!ok) { - _streamSettings._windowStarveThreshold = DEFAULT_WINDOW_STARVE_THRESHOLD; + int windowStarveThreshold = audioBufferGroupObject[WINDOW_STARVE_THRESHOLD_JSON_KEY].toString().toInt(&ok); + if (ok && windowStarveThreshold != InboundAudioStream::WINDOW_STARVE_THRESHOLD) { + deprecationNotice(WINDOW_STARVE_THRESHOLD_JSON_KEY, QString::number(windowStarveThreshold)); } - qDebug() << "Window A starve threshold:" << _streamSettings._windowStarveThreshold; const QString WINDOW_SECONDS_FOR_DESIRED_CALC_ON_TOO_MANY_STARVES_JSON_KEY = "window_seconds_for_desired_calc_on_too_many_starves"; - _streamSettings._windowSecondsForDesiredCalcOnTooManyStarves = audioBufferGroupObject[WINDOW_SECONDS_FOR_DESIRED_CALC_ON_TOO_MANY_STARVES_JSON_KEY].toString().toInt(&ok); - if (!ok) { - _streamSettings._windowSecondsForDesiredCalcOnTooManyStarves = DEFAULT_WINDOW_SECONDS_FOR_DESIRED_CALC_ON_TOO_MANY_STARVES; + int windowSecondsForDesiredCalcOnTooManyStarves = audioBufferGroupObject[WINDOW_SECONDS_FOR_DESIRED_CALC_ON_TOO_MANY_STARVES_JSON_KEY].toString().toInt(&ok); + if (ok && windowSecondsForDesiredCalcOnTooManyStarves != InboundAudioStream::WINDOW_SECONDS_FOR_DESIRED_CALC_ON_TOO_MANY_STARVES) { + deprecationNotice(WINDOW_SECONDS_FOR_DESIRED_CALC_ON_TOO_MANY_STARVES_JSON_KEY, QString::number(windowSecondsForDesiredCalcOnTooManyStarves)); } - qDebug() << "Window A length:" << _streamSettings._windowSecondsForDesiredCalcOnTooManyStarves << "seconds"; const QString WINDOW_SECONDS_FOR_DESIRED_REDUCTION_JSON_KEY = "window_seconds_for_desired_reduction"; - _streamSettings._windowSecondsForDesiredReduction = audioBufferGroupObject[WINDOW_SECONDS_FOR_DESIRED_REDUCTION_JSON_KEY].toString().toInt(&ok); - if (!ok) { - _streamSettings._windowSecondsForDesiredReduction = DEFAULT_WINDOW_SECONDS_FOR_DESIRED_REDUCTION; + int windowSecondsForDesiredReduction = audioBufferGroupObject[WINDOW_SECONDS_FOR_DESIRED_REDUCTION_JSON_KEY].toString().toInt(&ok); + if (ok && windowSecondsForDesiredReduction != InboundAudioStream::WINDOW_SECONDS_FOR_DESIRED_REDUCTION) { + deprecationNotice(WINDOW_SECONDS_FOR_DESIRED_REDUCTION_JSON_KEY, QString::number(windowSecondsForDesiredReduction)); + } + + const QString USE_STDEV_FOR_JITTER_JSON_KEY = "use_stdev_for_desired_calc"; + bool useStDevForJitterCalc = audioBufferGroupObject[USE_STDEV_FOR_JITTER_JSON_KEY].toBool(); + if (useStDevForJitterCalc != InboundAudioStream::USE_STDEV_FOR_JITTER) { + deprecationNotice(USE_STDEV_FOR_JITTER_JSON_KEY, useStDevForJitterCalc ? "true" : "false"); } - qDebug() << "Window B length:" << _streamSettings._windowSecondsForDesiredReduction << "seconds"; const QString REPETITION_WITH_FADE_JSON_KEY = "repetition_with_fade"; - _streamSettings._repetitionWithFade = audioBufferGroupObject[REPETITION_WITH_FADE_JSON_KEY].toBool(); - if (_streamSettings._repetitionWithFade) { - qDebug() << "Repetition with fade enabled"; - } else { - qDebug() << "Repetition with fade disabled"; + bool repetitionWithFade = audioBufferGroupObject[REPETITION_WITH_FADE_JSON_KEY].toBool(); + if (repetitionWithFade != InboundAudioStream::REPETITION_WITH_FADE) { + deprecationNotice(REPETITION_WITH_FADE_JSON_KEY, repetitionWithFade ? "true" : "false"); } } diff --git a/assignment-client/src/audio/AudioMixer.h b/assignment-client/src/audio/AudioMixer.h index 7e3ef8a69b..bccac529c1 100644 --- a/assignment-client/src/audio/AudioMixer.h +++ b/assignment-client/src/audio/AudioMixer.h @@ -39,7 +39,7 @@ public slots: void sendStatsPacket() override; - static const InboundAudioStream::Settings& getStreamSettings() { return _streamSettings; } + static int getStaticJitterFrames() { return _numStaticJitterFrames; } private slots: void broadcastMixes(); @@ -112,7 +112,7 @@ private: }; QVector _zoneReverbSettings; - static InboundAudioStream::Settings _streamSettings; + static int _numStaticJitterFrames; // -1 denotes dynamic jitter buffering static bool _enableFilter; }; diff --git a/assignment-client/src/audio/AudioMixerClientData.cpp b/assignment-client/src/audio/AudioMixerClientData.cpp index 42d385c1f6..290a361118 100644 --- a/assignment-client/src/audio/AudioMixerClientData.cpp +++ b/assignment-client/src/audio/AudioMixerClientData.cpp @@ -109,7 +109,7 @@ int AudioMixerClientData::parseData(ReceivedMessage& message) { bool isStereo = channelFlag == 1; - auto avatarAudioStream = new AvatarAudioStream(isStereo, AudioMixer::getStreamSettings()); + auto avatarAudioStream = new AvatarAudioStream(isStereo, AudioMixer::getStaticJitterFrames()); avatarAudioStream->setupCodec(_codec, _selectedCodecName, AudioConstants::MONO); qDebug() << "creating new AvatarAudioStream... codec:" << _selectedCodecName; @@ -143,7 +143,7 @@ int AudioMixerClientData::parseData(ReceivedMessage& message) { if (streamIt == _audioStreams.end()) { // we don't have this injected stream yet, so add it - auto injectorStream = new InjectedAudioStream(streamIdentifier, isStereo, AudioMixer::getStreamSettings()); + auto injectorStream = new InjectedAudioStream(streamIdentifier, isStereo, AudioMixer::getStaticJitterFrames()); #if INJECTORS_SUPPORT_CODECS injectorStream->setupCodec(_codec, _selectedCodecName, isStereo ? AudioConstants::STEREO : AudioConstants::MONO); diff --git a/assignment-client/src/audio/AvatarAudioStream.cpp b/assignment-client/src/audio/AvatarAudioStream.cpp index da995c999d..1e0c3ed9e6 100644 --- a/assignment-client/src/audio/AvatarAudioStream.cpp +++ b/assignment-client/src/audio/AvatarAudioStream.cpp @@ -13,10 +13,8 @@ #include "AvatarAudioStream.h" -AvatarAudioStream::AvatarAudioStream(bool isStereo, const InboundAudioStream::Settings& settings) : - PositionalAudioStream(PositionalAudioStream::Microphone, isStereo, settings) -{ -} +AvatarAudioStream::AvatarAudioStream(bool isStereo, int numStaticJitterFrames) : + PositionalAudioStream(PositionalAudioStream::Microphone, isStereo, numStaticJitterFrames) {} int AvatarAudioStream::parseStreamProperties(PacketType type, const QByteArray& packetAfterSeqNum, int& numAudioSamples) { int readBytes = 0; diff --git a/assignment-client/src/audio/AvatarAudioStream.h b/assignment-client/src/audio/AvatarAudioStream.h index d2e1137ae6..497e522922 100644 --- a/assignment-client/src/audio/AvatarAudioStream.h +++ b/assignment-client/src/audio/AvatarAudioStream.h @@ -18,7 +18,7 @@ class AvatarAudioStream : public PositionalAudioStream { public: - AvatarAudioStream(bool isStereo, const InboundAudioStream::Settings& settings); + AvatarAudioStream(bool isStereo, int numStaticJitterFrames = -1); private: // disallow copying of AvatarAudioStream objects diff --git a/domain-server/resources/describe-settings.json b/domain-server/resources/describe-settings.json index c9d7ea77d5..63ad0c5899 100644 --- a/domain-server/resources/describe-settings.json +++ b/domain-server/resources/describe-settings.json @@ -1025,65 +1025,41 @@ "name": "dynamic_jitter_buffer", "type": "checkbox", "label": "Dynamic Jitter Buffers", - "help": "Dynamically buffer client audio based on perceived jitter in packet receipt timing", + "help": "Dynamically buffer inbound audio streams based on perceived jitter in packet receipt timing.", "default": true, "advanced": true }, { "name": "static_desired_jitter_buffer_frames", "label": "Static Desired Jitter Buffer Frames", - "help": "If dynamic jitter buffers is disabled, this determines the target number of frames maintained by the AudioMixer's jitter buffers", + "help": "If dynamic jitter buffers is disabled, this determines the size of the jitter buffers of inbound audio streams in the mixer. Higher numbers introduce more latency.", "placeholder": "1", "default": "1", "advanced": true }, { - "name": "max_frames_over_desired", - "label": "Max Frames Over Desired", - "help": "The highest number of frames an AudioMixer's ringbuffer can exceed the desired jitter buffer frames by", - "placeholder": "10", - "default": "10", - "advanced": true + "name": "max_frames_over_desired", + "deprecated": true }, { - "name": "use_stdev_for_desired_calc", - "type": "checkbox", - "label": "Stdev for Desired Jitter Frames Calc", - "help": "Use Philip's method (stdev of timegaps) to calculate desired jitter frames (otherwise Fred's max timegap method is used)", - "default": false, - "advanced": true + "name": "window_starve_threshold", + "deprecated": true }, { - "name": "window_starve_threshold", - "label": "Window Starve Threshold", - "help": "If this many starves occur in an N-second window (N is the number in the next field), then the desired jitter frames will be re-evaluated using Window A.", - "placeholder": "3", - "default": "3", - "advanced": true + "name": "window_seconds_for_desired_calc_on_too_many_starves", + "deprecated": true }, { - "name": "window_seconds_for_desired_calc_on_too_many_starves", - "label": "Timegaps Window (A) Seconds", - "help": "Window A contains a history of timegaps. Its max timegap is used to re-evaluate the desired jitter frames when too many starves occur within it.", - "placeholder": "50", - "default": "50", - "advanced": true + "name": "window_seconds_for_desired_reduction", + "deprecated": true }, { - "name": "window_seconds_for_desired_reduction", - "label": "Timegaps Window (B) Seconds", - "help": "Window B contains a history of timegaps. Its max timegap is used as a ceiling for the desired jitter frames value.", - "placeholder": "10", - "default": "10", - "advanced": true + "name": "use_stdev_for_desired_calc", + "deprecated": true }, { - "name": "repetition_with_fade", - "type": "checkbox", - "label": "Repetition with Fade", - "help": "Dropped frames and mixing during starves repeat the last frame, eventually fading to silence", - "default": false, - "advanced": true + "name": "repetition_with_fade", + "deprecated": true } ] }, diff --git a/domain-server/resources/web/css/style.css b/domain-server/resources/web/css/style.css index ef967a47bf..ad426671a4 100644 --- a/domain-server/resources/web/css/style.css +++ b/domain-server/resources/web/css/style.css @@ -75,7 +75,7 @@ span.port { color: #666666; } -.advanced-setting { +.advanced-setting, .deprecated-setting { display: none; } diff --git a/domain-server/resources/web/settings/index.shtml b/domain-server/resources/web/settings/index.shtml index 802038d806..7af21fe84b 100644 --- a/domain-server/resources/web/settings/index.shtml +++ b/domain-server/resources/web/settings/index.shtml @@ -40,7 +40,8 @@