Merge pull request #4056 from huffman/audio-buffer-starvation-detection

Audio buffer starvation detection
This commit is contained in:
Brad Hefta-Gaub 2015-01-07 11:18:34 -08:00
commit 8ad21d61c4
5 changed files with 477 additions and 44 deletions

View file

@ -81,6 +81,12 @@ Audio::Audio() :
_noiseSourceEnabled(false),
_toneSourceEnabled(true),
_outgoingAvatarAudioSequenceNumber(0),
_outputBufferSizeFrames(DEFAULT_AUDIO_OUTPUT_BUFFER_SIZE_FRAMES),
_outputStarveDetectionStartTimeMsec(0),
_outputStarveDetectionCount(0),
_outputStarveDetectionEnabled(true),
_outputStarveDetectionPeriodMsec(DEFAULT_AUDIO_OUTPUT_STARVE_DETECTION_PERIOD),
_outputStarveDetectionThreshold(DEFAULT_AUDIO_OUTPUT_STARVE_DETECTION_THRESHOLD),
_audioOutputIODevice(_receivedAudioStream, this),
_stats(&_receivedAudioStream),
_inputGate()
@ -947,10 +953,24 @@ bool Audio::switchInputToAudioDevice(const QAudioDeviceInfo& inputDeviceInfo) {
void Audio::outputNotify() {
int recentUnfulfilled = _audioOutputIODevice.getRecentUnfulfilledReads();
if (recentUnfulfilled > 0) {
qDebug() << "WARNING --- WE HAD at least:" << recentUnfulfilled << "recently unfulfilled readData() calls";
if (_outputStarveDetectionEnabled) {
quint64 now = usecTimestampNow() / 1000;
quint64 dt = now - _outputStarveDetectionStartTimeMsec;
if (dt > _outputStarveDetectionPeriodMsec) {
_outputStarveDetectionStartTimeMsec = now;
_outputStarveDetectionCount = 0;
} else {
_outputStarveDetectionCount += recentUnfulfilled;
if (_outputStarveDetectionCount > _outputStarveDetectionThreshold) {
int newOutputBufferSizeFrames = _outputBufferSizeFrames + 1;
qDebug() << "Starve detection threshold met, increasing buffer size to " << newOutputBufferSizeFrames;
setOutputBufferSize(newOutputBufferSizeFrames);
// TODO: Ryan Huffman -- add code here to increase the AUDIO_OUTPUT_BUFFER_SIZE_FRAMES... this code only
// runs in cases where the audio device requested data samples, and ran dry because we couldn't fulfill the request
_outputStarveDetectionStartTimeMsec = now;
_outputStarveDetectionCount = 0;
}
}
}
}
}
@ -978,13 +998,12 @@ bool Audio::switchOutputToAudioDevice(const QAudioDeviceInfo& outputDeviceInfo)
outputFormatChanged();
const int AUDIO_OUTPUT_BUFFER_SIZE_FRAMES = 3;
// setup our general output device for audio-mixer audio
_audioOutput = new QAudioOutput(outputDeviceInfo, _outputFormat, this);
_audioOutput->setBufferSize(_outputBufferSizeFrames * _outputFrameSize * sizeof(int16_t));
connect(_audioOutput, &QAudioOutput::notify, this, &Audio::outputNotify);
_audioOutput->setBufferSize(AUDIO_OUTPUT_BUFFER_SIZE_FRAMES * _outputFrameSize * sizeof(int16_t));
qDebug() << "Output Buffer capacity in frames: " << _audioOutput->bufferSize() / sizeof(int16_t) / (float)_outputFrameSize;
_audioOutputIODevice.start();
@ -1002,6 +1021,21 @@ bool Audio::switchOutputToAudioDevice(const QAudioDeviceInfo& outputDeviceInfo)
return supportedFormat;
}
void Audio::setOutputBufferSize(int numFrames) {
numFrames = std::min(std::max(numFrames, MIN_AUDIO_OUTPUT_BUFFER_SIZE_FRAMES), MAX_AUDIO_OUTPUT_BUFFER_SIZE_FRAMES);
if (numFrames != _outputBufferSizeFrames) {
qDebug() << "Audio output buffer size (frames): " << numFrames;
_outputBufferSizeFrames = numFrames;
if (_audioOutput) {
// The buffer size can't be adjusted after QAudioOutput::start() has been called, so
// recreate the device by switching to the default.
QAudioDeviceInfo outputDeviceInfo = defaultAudioDeviceForMode(QAudio::AudioOutput);
switchOutputToAudioDevice(outputDeviceInfo);
}
}
}
// The following constant is operating system dependent due to differences in
// the way input audio is handled. The audio input buffer size is inversely
// proportional to the accelerator ratio.

View file

@ -61,6 +61,13 @@ extern "C" {
static const int NUM_AUDIO_CHANNELS = 2;
static const int DEFAULT_AUDIO_OUTPUT_BUFFER_SIZE_FRAMES = 3;
static const int MIN_AUDIO_OUTPUT_BUFFER_SIZE_FRAMES = 1;
static const int MAX_AUDIO_OUTPUT_BUFFER_SIZE_FRAMES = 20;
static const int DEFAULT_AUDIO_OUTPUT_STARVE_DETECTION_ENABLED = true;
static const int DEFAULT_AUDIO_OUTPUT_STARVE_DETECTION_THRESHOLD = 3;
static const int DEFAULT_AUDIO_OUTPUT_STARVE_DETECTION_PERIOD = 10 * 1000; // 10 Seconds
class QAudioInput;
class QAudioOutput;
class QIODevice;
@ -108,7 +115,18 @@ public:
float getAudioOutputMsecsUnplayed() const;
void setRecorder(RecorderPointer recorder) { _recorder = recorder; }
int getOutputBufferSize() { return _outputBufferSizeFrames; }
bool getOutputStarveDetectionEnabled() { return _outputStarveDetectionEnabled; }
void setOutputStarveDetectionEnabled(bool enabled) { _outputStarveDetectionEnabled = enabled; }
int getOutputStarveDetectionPeriod() { return _outputStarveDetectionPeriodMsec; }
void setOutputStarveDetectionPeriod(int msecs) { _outputStarveDetectionPeriodMsec = msecs; }
int getOutputStarveDetectionThreshold() { return _outputStarveDetectionThreshold; }
void setOutputStarveDetectionThreshold(int threshold) { _outputStarveDetectionThreshold = threshold; }
static const float CALLBACK_ACCELERATOR_RATIO;
public slots:
@ -137,6 +155,8 @@ public slots:
void processReceivedSamples(const QByteArray& inputBuffer, QByteArray& outputBuffer);
void sendMuteEnvironmentPacket();
void setOutputBufferSize(int numFrames);
virtual bool outputLocalInjector(bool isStereo, qreal volume, AudioInjector* injector);
bool switchInputToAudioDevice(const QString& inputDeviceName);
@ -184,6 +204,14 @@ private:
QString _inputAudioDeviceName;
QString _outputAudioDeviceName;
int _outputBufferSizeFrames;
bool _outputStarveDetectionEnabled;
quint64 _outputStarveDetectionStartTimeMsec;
int _outputStarveDetectionCount;
int _outputStarveDetectionPeriodMsec;
int _outputStarveDetectionThreshold; // Maximum number of starves per _outputStarveDetectionPeriod before increasing buffer size
StDev _stdev;
QElapsedTimer _timeSinceLastReceived;

View file

@ -41,6 +41,7 @@
#include "Application.h"
#include "AccountManager.h"
#include "Audio.h"
#include "audio/AudioIOStatsRenderer.h"
#include "audio/AudioScope.h"
#include "devices/Faceshift.h"
@ -642,6 +643,13 @@ void Menu::loadSettings(QSettings* settings) {
_receivedAudioStreamSettings._windowSecondsForDesiredCalcOnTooManyStarves = settings->value("windowSecondsForDesiredCalcOnTooManyStarves", DEFAULT_WINDOW_SECONDS_FOR_DESIRED_CALC_ON_TOO_MANY_STARVES).toInt();
_receivedAudioStreamSettings._windowSecondsForDesiredReduction = settings->value("windowSecondsForDesiredReduction", DEFAULT_WINDOW_SECONDS_FOR_DESIRED_REDUCTION).toInt();
_receivedAudioStreamSettings._repetitionWithFade = settings->value("repetitionWithFade", DEFAULT_REPETITION_WITH_FADE).toBool();
QSharedPointer<Audio> audio = DependencyManager::get<Audio>();
audio->setOutputStarveDetectionEnabled(settings->value("audioOutputStarveDetectionEnabled", DEFAULT_AUDIO_OUTPUT_STARVE_DETECTION_ENABLED).toBool());
audio->setOutputStarveDetectionThreshold(settings->value("audioOutputStarveDetectionThreshold", DEFAULT_AUDIO_OUTPUT_STARVE_DETECTION_THRESHOLD).toInt());
audio->setOutputStarveDetectionPeriod(settings->value("audioOutputStarveDetectionPeriod", DEFAULT_AUDIO_OUTPUT_STARVE_DETECTION_PERIOD).toInt());
int bufferSize = settings->value("audioOutputBufferSize", DEFAULT_AUDIO_OUTPUT_BUFFER_SIZE_FRAMES).toInt();
QMetaObject::invokeMethod(audio.data(), "setOutputBufferSize", Q_ARG(int, bufferSize));
_fieldOfView = loadSetting(settings, "fieldOfView", DEFAULT_FIELD_OF_VIEW_DEGREES);
_realWorldFieldOfView = loadSetting(settings, "realWorldFieldOfView", DEFAULT_REAL_WORLD_FIELD_OF_VIEW_DEGREES);
@ -701,6 +709,12 @@ void Menu::saveSettings(QSettings* settings) {
settings->setValue("windowSecondsForDesiredReduction", _receivedAudioStreamSettings._windowSecondsForDesiredReduction);
settings->setValue("repetitionWithFade", _receivedAudioStreamSettings._repetitionWithFade);
QSharedPointer<Audio> audio = DependencyManager::get<Audio>();
settings->setValue("audioOutputStarveDetectionEnabled", audio->getOutputStarveDetectionEnabled());
settings->setValue("audioOutputStarveDetectionThreshold", audio->getOutputStarveDetectionThreshold());
settings->setValue("audioOutputStarveDetectionPeriod", audio->getOutputStarveDetectionPeriod());
settings->setValue("audioOutputBufferSize", audio->getOutputBufferSize());
settings->setValue("fieldOfView", _fieldOfView);
settings->setValue("faceshiftEyeDeflection", _faceshiftEyeDeflection);
settings->setValue("faceshiftHostname", _faceshiftHostname);

View file

@ -12,6 +12,7 @@
#include <QFileDialog>
#include "Application.h"
#include "Audio.h"
#include "MainWindow.h"
#include "Menu.h"
#include "ModelsBrowser.h"
@ -28,6 +29,9 @@ PreferencesDialog::PreferencesDialog() :
ui.setupUi(this);
loadPreferences();
ui.outputBufferSizeSpinner->setMinimum(MIN_AUDIO_OUTPUT_BUFFER_SIZE_FRAMES);
ui.outputBufferSizeSpinner->setMaximum(MAX_AUDIO_OUTPUT_BUFFER_SIZE_FRAMES);
connect(ui.buttonBrowseHead, &QPushButton::clicked, this, &PreferencesDialog::openHeadModelBrowser);
connect(ui.buttonBrowseBody, &QPushButton::clicked, this, &PreferencesDialog::openBodyModelBrowser);
connect(ui.buttonBrowseLocation, &QPushButton::clicked, this, &PreferencesDialog::openSnapshotLocationBrowser);
@ -136,6 +140,13 @@ void PreferencesDialog::loadPreferences() {
ui.windowSecondsForDesiredReductionSpin->setValue(streamSettings._windowSecondsForDesiredReduction);
ui.repetitionWithFadeCheckBox->setChecked(streamSettings._repetitionWithFade);
QSharedPointer<Audio> audio = DependencyManager::get<Audio>();
ui.outputBufferSizeSpinner->setValue(audio->getOutputBufferSize());
ui.outputStarveDetectionCheckBox->setChecked(audio->getOutputStarveDetectionEnabled());
ui.outputStarveDetectionThresholdSpinner->setValue(audio->getOutputStarveDetectionThreshold());
ui.outputStarveDetectionPeriodSpinner->setValue(audio->getOutputStarveDetectionPeriod());
ui.realWorldFieldOfViewSpin->setValue(menuInstance->getRealWorldFieldOfView());
ui.fieldOfViewSpin->setValue(menuInstance->getFieldOfView());
@ -244,7 +255,14 @@ void PreferencesDialog::savePreferences() {
streamSettings._repetitionWithFade = ui.repetitionWithFadeCheckBox->isChecked();
Menu::getInstance()->setReceivedAudioStreamSettings(streamSettings);
DependencyManager::get<Audio>()->setReceivedAudioStreamSettings(streamSettings);
QSharedPointer<Audio> audio = DependencyManager::get<Audio>();
QMetaObject::invokeMethod(audio.data(), "setOutputBufferSize", Q_ARG(int, ui.outputBufferSizeSpinner->value()));
audio->setOutputStarveDetectionEnabled(ui.outputStarveDetectionCheckBox->isChecked());
audio->setOutputStarveDetectionThreshold(ui.outputStarveDetectionThresholdSpinner->value());
audio->setOutputStarveDetectionPeriod(ui.outputStarveDetectionPeriodSpinner->value());
Application::getInstance()->resizeGL(glCanvas->width(), glCanvas->height());

View file

@ -59,9 +59,9 @@
<property name="geometry">
<rect>
<x>0</x>
<y>0</y>
<width>500</width>
<height>1386</height>
<y>-825</y>
<width>485</width>
<height>1611</height>
</rect>
</property>
<layout class="QVBoxLayout" name="verticalLayout_2">
@ -1126,6 +1126,185 @@
</item>
</layout>
</item>
<item>
<layout class="QHBoxLayout" name="horizontalLayout_6">
<property name="spacing">
<number>0</number>
</property>
<property name="topMargin">
<number>7</number>
</property>
<property name="rightMargin">
<number>0</number>
</property>
<property name="bottomMargin">
<number>7</number>
</property>
<item>
<widget class="QLabel" name="label_5">
<property name="font">
<font>
<family>Arial</family>
</font>
</property>
<property name="text">
<string>Faceshift eye detection</string>
</property>
<property name="indent">
<number>0</number>
</property>
<property name="buddy">
<cstring>faceshiftEyeDeflectionSider</cstring>
</property>
</widget>
</item>
<item>
<spacer name="horizontalSpacer_6">
<property name="font">
<font>
<family>Arial</family>
</font>
</property>
<property name="orientation">
<enum>Qt::Horizontal</enum>
</property>
<property name="sizeHint" stdset="0">
<size>
<width>40</width>
<height>20</height>
</size>
</property>
</spacer>
</item>
<item>
<widget class="QSlider" name="faceshiftEyeDeflectionSider">
<property name="sizePolicy">
<sizepolicy hsizetype="Fixed" vsizetype="Fixed">
<horstretch>0</horstretch>
<verstretch>0</verstretch>
</sizepolicy>
</property>
<property name="minimumSize">
<size>
<width>130</width>
<height>0</height>
</size>
</property>
<property name="font">
<font>
<family>Arial</family>
</font>
</property>
<property name="orientation">
<enum>Qt::Horizontal</enum>
</property>
</widget>
</item>
</layout>
</item>
<item>
<layout class="QHBoxLayout" name="horizontalLayout_999">
<property name="spacing">
<number>0</number>
</property>
<property name="topMargin">
<number>7</number>
</property>
<property name="rightMargin">
<number>0</number>
</property>
<property name="bottomMargin">
<number>7</number>
</property>
<item>
<widget class="QLabel" name="label_999">
<property name="font">
<font>
<family>Arial</family>
</font>
</property>
<property name="text">
<string>Faceshift hostname</string>
</property>
<property name="indent">
<number>0</number>
</property>
<property name="buddy">
<cstring>faceshiftHostnameEdit</cstring>
</property>
</widget>
</item>
<item>
<spacer name="horizontalSpacer_999">
<property name="font">
<font>
<family>Arial</family>
</font>
</property>
<property name="orientation">
<enum>Qt::Horizontal</enum>
</property>
<property name="sizeHint" stdset="0">
<size>
<width>0</width>
<height>0</height>
</size>
</property>
</spacer>
</item>
<item>
<widget class="QLineEdit" name="faceshiftHostnameEdit">
<property name="font">
<font>
<family>Arial</family>
</font>
</property>
<property name="layoutDirection">
<enum>Qt::LeftToRight</enum>
</property>
<property name="styleSheet">
<string notr="true"/>
</property>
<property name="placeholderText">
<string>localhost</string>
</property>
</widget>
</item>
</layout>
</item>
<item>
<widget class="QLabel" name="voxelsTitleLabel_2">
<property name="sizePolicy">
<sizepolicy hsizetype="Preferred" vsizetype="Minimum">
<horstretch>0</horstretch>
<verstretch>0</verstretch>
</sizepolicy>
</property>
<property name="minimumSize">
<size>
<width>0</width>
<height>40</height>
</size>
</property>
<property name="font">
<font>
<family>Arial</family>
<pointsize>18</pointsize>
<weight>75</weight>
<bold>true</bold>
</font>
</property>
<property name="styleSheet">
<string notr="true">color:#29967e</string>
</property>
<property name="text">
<string>Audio</string>
</property>
<property name="alignment">
<set>Qt::AlignBottom|Qt::AlignLeading|Qt::AlignLeft</set>
</property>
</widget>
</item>
<item>
<widget class="QCheckBox" name="dynamicJitterBuffersCheckBox">
<property name="sizePolicy">
@ -1651,7 +1830,7 @@
</widget>
</item>
<item>
<layout class="QHBoxLayout" name="horizontalLayout_6">
<layout class="QHBoxLayout" name="horizontalLayout_23">
<property name="spacing">
<number>0</number>
</property>
@ -1664,26 +1843,26 @@
<property name="bottomMargin">
<number>7</number>
</property>
<item>
<widget class="QLabel" name="label_5">
<item alignment="Qt::AlignLeft">
<widget class="QLabel" name="label_20">
<property name="font">
<font>
<family>Arial</family>
</font>
</property>
<property name="text">
<string>Faceshift eye detection</string>
<string>Output Buffer Size (Frames)</string>
</property>
<property name="indent">
<number>0</number>
</property>
<property name="buddy">
<cstring>faceshiftEyeDeflectionSider</cstring>
<cstring>windowSecondsForDesiredReductionSpin</cstring>
</property>
</widget>
</item>
<item>
<spacer name="horizontalSpacer_6">
<spacer name="horizontalSpacer_25">
<property name="font">
<font>
<family>Arial</family>
@ -1701,7 +1880,7 @@
</spacer>
</item>
<item>
<widget class="QSlider" name="faceshiftEyeDeflectionSider">
<widget class="QSpinBox" name="outputBufferSizeSpinner">
<property name="sizePolicy">
<sizepolicy hsizetype="Fixed" vsizetype="Fixed">
<horstretch>0</horstretch>
@ -1710,24 +1889,72 @@
</property>
<property name="minimumSize">
<size>
<width>130</width>
<width>100</width>
<height>0</height>
</size>
</property>
<property name="maximumSize">
<size>
<width>70</width>
<height>16777215</height>
</size>
</property>
<property name="font">
<font>
<family>Arial</family>
</font>
</property>
<property name="orientation">
<enum>Qt::Horizontal</enum>
<property name="minimum">
<number>1</number>
</property>
<property name="maximum">
<number>20</number>
</property>
<property name="value">
<number>1</number>
</property>
</widget>
</item>
</layout>
</item>
<item>
<layout class="QHBoxLayout" name="horizontalLayout_999">
<widget class="QCheckBox" name="outputStarveDetectionCheckBox">
<property name="sizePolicy">
<sizepolicy hsizetype="Minimum" vsizetype="Fixed">
<horstretch>0</horstretch>
<verstretch>0</verstretch>
</sizepolicy>
</property>
<property name="minimumSize">
<size>
<width>32</width>
<height>40</height>
</size>
</property>
<property name="baseSize">
<size>
<width>0</width>
<height>0</height>
</size>
</property>
<property name="font">
<font>
<family>Arial</family>
</font>
</property>
<property name="text">
<string>Output Starve Detection (Automatic Buffer Size Increase)</string>
</property>
<property name="iconSize">
<size>
<width>32</width>
<height>32</height>
</size>
</property>
</widget>
</item>
<item>
<layout class="QHBoxLayout" name="horizontalLayout_27">
<property name="spacing">
<number>0</number>
</property>
@ -1740,26 +1967,26 @@
<property name="bottomMargin">
<number>7</number>
</property>
<item>
<widget class="QLabel" name="label_999">
<item alignment="Qt::AlignLeft">
<widget class="QLabel" name="label_24">
<property name="font">
<font>
<family>Arial</family>
</font>
</property>
<property name="text">
<string>Faceshift hostname</string>
<string>Output Starve Detection Threshold</string>
</property>
<property name="indent">
<number>0</number>
</property>
<property name="buddy">
<cstring>faceshiftHostnameEdit</cstring>
<cstring>windowSecondsForDesiredReductionSpin</cstring>
</property>
</widget>
</item>
<item>
<spacer name="horizontalSpacer_999">
<spacer name="horizontalSpacer_29">
<property name="font">
<font>
<family>Arial</family>
@ -1768,28 +1995,140 @@
<property name="orientation">
<enum>Qt::Horizontal</enum>
</property>
<property name="sizeHint" stdset="0">
<size>
<width>40</width>
<height>20</height>
</size>
</property>
</spacer>
</item>
<item>
<widget class="QLineEdit" name="faceshiftHostnameEdit">
<property name="font">
<font>
<family>Arial</family>
</font>
</property>
<property name="layoutDirection">
<enum>Qt::LeftToRight</enum>
</property>
<property name="styleSheet">
<string notr="true"/>
</property>
<property name="placeholderText">
<string>localhost</string>
</property>
</widget>
<widget class="QSpinBox" name="outputStarveDetectionThresholdSpinner">
<property name="sizePolicy">
<sizepolicy hsizetype="Fixed" vsizetype="Fixed">
<horstretch>0</horstretch>
<verstretch>0</verstretch>
</sizepolicy>
</property>
<property name="minimumSize">
<size>
<width>100</width>
<height>0</height>
</size>
</property>
<property name="maximumSize">
<size>
<width>70</width>
<height>16777215</height>
</size>
</property>
<property name="font">
<font>
<family>Arial</family>
</font>
</property>
<property name="minimum">
<number>1</number>
</property>
<property name="maximum">
<number>500</number>
</property>
<property name="value">
<number>1</number>
</property>
</widget>
</item>
</layout>
</item>
</item>
<item>
<layout class="QHBoxLayout" name="horizontalLayout_25">
<property name="spacing">
<number>0</number>
</property>
<property name="topMargin">
<number>7</number>
</property>
<property name="rightMargin">
<number>0</number>
</property>
<property name="bottomMargin">
<number>7</number>
</property>
<item alignment="Qt::AlignLeft">
<widget class="QLabel" name="label_22">
<property name="font">
<font>
<family>Arial</family>
</font>
</property>
<property name="text">
<string>Output Starve Detection Period (ms)</string>
</property>
<property name="indent">
<number>0</number>
</property>
<property name="buddy">
<cstring>windowSecondsForDesiredReductionSpin</cstring>
</property>
</widget>
</item>
<item>
<spacer name="horizontalSpacer_27">
<property name="font">
<font>
<family>Arial</family>
</font>
</property>
<property name="orientation">
<enum>Qt::Horizontal</enum>
</property>
<property name="sizeHint" stdset="0">
<size>
<width>40</width>
<height>20</height>
</size>
</property>
</spacer>
</item>
<item>
<widget class="QSpinBox" name="outputStarveDetectionPeriodSpinner">
<property name="sizePolicy">
<sizepolicy hsizetype="Fixed" vsizetype="Fixed">
<horstretch>0</horstretch>
<verstretch>0</verstretch>
</sizepolicy>
</property>
<property name="minimumSize">
<size>
<width>100</width>
<height>0</height>
</size>
</property>
<property name="maximumSize">
<size>
<width>70</width>
<height>16777215</height>
</size>
</property>
<property name="font">
<font>
<family>Arial</family>
</font>
</property>
<property name="minimum">
<number>1</number>
</property>
<property name="maximum">
<number>999999999</number>
</property>
<property name="value">
<number>1</number>
</property>
</widget>
</item>
</layout>
</item>
<item>
<widget class="QLabel" name="voxelsTitleLabel">
<property name="sizePolicy">