From 7b8f61609935486944e4f69deb5c0d0795a9563c Mon Sep 17 00:00:00 2001 From: Zach Pomerantz Date: Thu, 8 Sep 2016 17:20:40 -0700 Subject: [PATCH 1/2] add audiostream (jitter) logging --- libraries/audio/src/AudioLogging.cpp | 6 ++++++ libraries/audio/src/AudioLogging.h | 1 + libraries/audio/src/InboundAudioStream.cpp | 11 +++++++++++ 3 files changed, 18 insertions(+) diff --git a/libraries/audio/src/AudioLogging.cpp b/libraries/audio/src/AudioLogging.cpp index f01b8ce846..9bb44f5be6 100644 --- a/libraries/audio/src/AudioLogging.cpp +++ b/libraries/audio/src/AudioLogging.cpp @@ -12,3 +12,9 @@ #include "AudioLogging.h" Q_LOGGING_CATEGORY(audio, "hifi.audio") + +#if DEV_BUILD || PR_BUILD +Q_LOGGING_CATEGORY(audiostream, "hifi.audio-stream", QtDebugMsg) +#else +Q_LOGGING_CATEGORY(audiostream, "hifi.audio-stream", QtInfoMsg) +#endif diff --git a/libraries/audio/src/AudioLogging.h b/libraries/audio/src/AudioLogging.h index c15e53be41..71e22e952c 100644 --- a/libraries/audio/src/AudioLogging.h +++ b/libraries/audio/src/AudioLogging.h @@ -15,5 +15,6 @@ #include Q_DECLARE_LOGGING_CATEGORY(audio) +Q_DECLARE_LOGGING_CATEGORY(audiostream) #endif // hifi_AudioLogging_h diff --git a/libraries/audio/src/InboundAudioStream.cpp b/libraries/audio/src/InboundAudioStream.cpp index 6b79879bb7..7acefd30b8 100644 --- a/libraries/audio/src/InboundAudioStream.cpp +++ b/libraries/audio/src/InboundAudioStream.cpp @@ -16,6 +16,7 @@ #include #include "InboundAudioStream.h" +#include "AudioLogging.h" const int STARVE_HISTORY_CAPACITY = 50; @@ -174,6 +175,9 @@ int InboundAudioStream::parseData(ReceivedMessage& message) { _currentJitterBufferFrames = 0; _oldFramesDropped += framesToDrop; + + qCDebug(audiostream, "Dropped %d frames", framesToDrop); + qCDebug(audiostream, "Resetted current jitter frames"); } framesAvailableChanged(); @@ -228,6 +232,9 @@ int InboundAudioStream::writeDroppableSilentSamples(int silentSamples) { _currentJitterBufferFrames -= numSilentFramesToDrop; _silentFramesDropped += numSilentFramesToDrop; + qCDebug(audiostream, "Dropped %d silent frames", numSilentFramesToDrop); + qCDebug(audiostream, "Set current jitter frames to %d", _currentJitterBufferFrames); + _framesAvailableStat.reset(); } @@ -308,6 +315,8 @@ void InboundAudioStream::framesAvailableChanged() { if (_framesAvailableStat.getElapsedUsecs() >= FRAMES_AVAILABLE_STAT_WINDOW_USECS) { _currentJitterBufferFrames = (int)ceil(_framesAvailableStat.getAverage()); + qCDebug(audiostream, "Set current jitter frames to %d", _currentJitterBufferFrames); + _framesAvailableStat.reset(); } } @@ -355,6 +364,7 @@ void InboundAudioStream::setToStarved() { // make sure _desiredJitterBufferFrames does not become lower here if (calculatedJitterBufferFrames >= _desiredJitterBufferFrames) { _desiredJitterBufferFrames = calculatedJitterBufferFrames; + qCDebug(audiostream, "Set desired jitter frames to %d", _desiredJitterBufferFrames); } } } @@ -444,6 +454,7 @@ void InboundAudioStream::packetReceivedUpdateTimingStats() { / (float)AudioConstants::NETWORK_FRAME_USECS); if (calculatedJitterBufferFrames < _desiredJitterBufferFrames) { _desiredJitterBufferFrames = calculatedJitterBufferFrames; + qCDebug(audiostream, "Set desired jitter frames to %d", _desiredJitterBufferFrames); } _timeGapStatsForDesiredReduction.clearNewStatsAvailableFlag(); } From e785c0cdf801bffd2a007e53ca1e716921749eee Mon Sep 17 00:00:00 2001 From: Zach Pomerantz Date: Thu, 8 Sep 2016 17:23:30 -0700 Subject: [PATCH 2/2] add test for audio jitter --- interface/src/ui/PreferencesDialog.cpp | 11 ++++++ libraries/audio-client/src/AudioClient.cpp | 41 ++++++++++++++++++++++ libraries/audio-client/src/AudioClient.h | 25 +++++++++++++ 3 files changed, 77 insertions(+) diff --git a/interface/src/ui/PreferencesDialog.cpp b/interface/src/ui/PreferencesDialog.cpp index 7fdafc9bda..c2f15a2ba1 100644 --- a/interface/src/ui/PreferencesDialog.cpp +++ b/interface/src/ui/PreferencesDialog.cpp @@ -277,6 +277,17 @@ void setupPreferences() { preference->setStep(1); preferences->addPreference(preference); } +#if DEV_BUILD || PR_BUILD + { + auto getter = []()->float { return DependencyManager::get()->getGateThreshold(); }; + auto setter = [](float value) { return DependencyManager::get()->setGateThreshold(value); }; + auto preference = new SpinnerPreference(AUDIO, "Debug gate threshold", getter, setter); + preference->setMin(1); + preference->setMax((float)100); + preference->setStep(1); + preferences->addPreference(preference); + } +#endif { auto getter = []()->float { return qApp->getMaxOctreePacketsPerSecond(); }; diff --git a/libraries/audio-client/src/AudioClient.cpp b/libraries/audio-client/src/AudioClient.cpp index 7fdda5a118..f567e50a9f 100644 --- a/libraries/audio-client/src/AudioClient.cpp +++ b/libraries/audio-client/src/AudioClient.cpp @@ -55,6 +55,8 @@ static const int RECEIVED_AUDIO_STREAM_CAPACITY_FRAMES = 100; static const auto DEFAULT_POSITION_GETTER = []{ return Vectors::ZERO; }; static const auto DEFAULT_ORIENTATION_GETTER = [] { return Quaternions::IDENTITY; }; +static const int DEFAULT_AUDIO_OUTPUT_GATE_THRESHOLD = 1; + Setting::Handle dynamicJitterBuffers("dynamicJitterBuffers", DEFAULT_DYNAMIC_JITTER_BUFFERS); Setting::Handle maxFramesOverDesired("maxFramesOverDesired", DEFAULT_MAX_FRAMES_OVER_DESIRED); Setting::Handle staticDesiredJitterBufferFrames("staticDesiredJitterBufferFrames", @@ -99,6 +101,8 @@ private: AudioClient::AudioClient() : AbstractAudioInterface(), + _gateThreshold("audioOutputGateThreshold", DEFAULT_AUDIO_OUTPUT_GATE_THRESHOLD), + _gate(this, _gateThreshold.get()), _audioInput(NULL), _desiredInputFormat(), _inputFormat(), @@ -540,11 +544,48 @@ void AudioClient::handleAudioDataPacket(QSharedPointer message) emit receivedFirstPacket(); } +#if DEV_BUILD || PR_BUILD + _gate.insert(message); +#else // Audio output must exist and be correctly set up if we're going to process received audio _receivedAudioStream.parseData(*message); +#endif } } +AudioClient::Gate::Gate(AudioClient* audioClient, int threshold) : + _audioClient(audioClient), + _threshold(threshold) {} + +void AudioClient::Gate::setThreshold(int threshold) { + flush(); + _threshold = std::max(threshold, 1); +} + +void AudioClient::Gate::insert(QSharedPointer message) { + // Short-circuit for normal behavior + if (_threshold == 1) { + _audioClient->_receivedAudioStream.parseData(*message); + return; + } + + _queue.push(message); + _index++; + + if (_index % _threshold == 0) { + flush(); + } +} + +void AudioClient::Gate::flush() { + while (!_queue.empty()) { + _audioClient->_receivedAudioStream.parseData(*_queue.front()); + _queue.pop(); + } + _index = 0; +} + + void AudioClient::handleNoisyMutePacket(QSharedPointer message) { if (!_muted) { toggleMute(); diff --git a/libraries/audio-client/src/AudioClient.h b/libraries/audio-client/src/AudioClient.h index b9e662bf85..ba6b98e1bd 100644 --- a/libraries/audio-client/src/AudioClient.h +++ b/libraries/audio-client/src/AudioClient.h @@ -16,6 +16,7 @@ #include #include #include +#include #include #include @@ -134,6 +135,9 @@ public: int getOutputStarveDetectionThreshold() { return _outputStarveDetectionThreshold.get(); } void setOutputStarveDetectionThreshold(int threshold) { _outputStarveDetectionThreshold.set(threshold); } + int getGateThreshold() { return _gate.getThreshold(); } + void setGateThreshold(int threshold) { _gate.setThreshold(threshold); } + void setPositionGetter(AudioPositionGetter positionGetter) { _positionGetter = positionGetter; } void setOrientationGetter(AudioOrientationGetter orientationGetter) { _orientationGetter = orientationGetter; } @@ -227,6 +231,27 @@ private: float azimuthForSource(const glm::vec3& relativePosition); float gainForSource(float distance, float volume); + class Gate { + public: + Gate(AudioClient* audioClient, int threshold); + + int getThreshold() { return _threshold; } + void setThreshold(int threshold); + + void insert(QSharedPointer message); + + private: + void flush(); + + AudioClient* _audioClient; + std::queue> _queue; + int _index{ 0 }; + int _threshold; + }; + + Setting::Handle _gateThreshold; + Gate _gate; + Mutex _injectorsMutex; QByteArray firstInputFrame; QAudioInput* _audioInput;