From 002f8c736f47eff5f84828fba24cee7c29771bce Mon Sep 17 00:00:00 2001 From: Philip Rosedale Date: Mon, 17 Jun 2013 18:30:02 -0700 Subject: [PATCH 1/4] Added adjustable audio jitter buffer (in preferences), and simple lowPassFilter (for pert testing compared to LPF) --- interface/src/Application.cpp | 20 ++++++++-- interface/src/Application.h | 4 ++ interface/src/Audio.cpp | 69 +++++++++++++++++++++++++++-------- interface/src/Audio.h | 12 +++--- interface/src/Head.cpp | 4 +- 5 files changed, 81 insertions(+), 28 deletions(-) diff --git a/interface/src/Application.cpp b/interface/src/Application.cpp index 1acad36d3b..feffb3a453 100644 --- a/interface/src/Application.cpp +++ b/interface/src/Application.cpp @@ -155,7 +155,7 @@ Application::Application(int& argc, char** argv, timeval &startup_time) : _oculusProgram(0), _oculusDistortionScale(1.25), #ifndef _WIN32 - _audio(&_audioScope), + _audio(&_audioScope, 0), #endif _stopNetworkReceiveThread(false), _packetCount(0), @@ -889,7 +889,13 @@ void Application::editPreferences() { QDoubleSpinBox* leanScale = new QDoubleSpinBox(); leanScale->setValue(_myAvatar.getLeanScale()); form->addRow("Lean Scale:", leanScale); - + + QSpinBox* audioJitterBufferSamples = new QSpinBox(); + audioJitterBufferSamples->setMaximum(10000); + audioJitterBufferSamples->setMinimum(-10000); + audioJitterBufferSamples->setValue(_audioJitterBufferSamples); + form->addRow("Audio Jitter Buffer Samples:", audioJitterBufferSamples); + QDialogButtonBox* buttons = new QDialogButtonBox(QDialogButtonBox::Ok | QDialogButtonBox::Cancel); dialog.connect(buttons, SIGNAL(accepted()), SLOT(accept())); dialog.connect(buttons, SIGNAL(rejected()), SLOT(reject())); @@ -903,7 +909,9 @@ void Application::editPreferences() { sendAvatarVoxelURLMessage(url); _headCameraPitchYawScale = headCameraPitchYawScale->value(); + _audioJitterBufferSamples = audioJitterBufferSamples->value(); _myAvatar.setLeanScale(leanScale->value()); + _audio.setJitterBufferSamples(audioJitterBufferSamples->value()); } void Application::pair() { @@ -1341,6 +1349,8 @@ void Application::initMenu() { debugMenu->addAction("Wants Res-In", this, SLOT(setWantsResIn(bool)))->setCheckable(true); debugMenu->addAction("Wants Monochrome", this, SLOT(setWantsMonochrome(bool)))->setCheckable(true); debugMenu->addAction("Wants View Delta Sending", this, SLOT(setWantsDelta(bool)))->setCheckable(true); + (_shouldLowPassFilter = debugMenu->addAction("Test: LowPass filter"))->setCheckable(true); + QMenu* settingsMenu = menuBar->addMenu("Settings"); (_settingsAutosave = settingsMenu->addAction("Autosave"))->setCheckable(true); @@ -1420,6 +1430,9 @@ void Application::init() { gettimeofday(&_lastTimeIdle, NULL); loadSettings(); + _audio.setJitterBufferSamples(_audioJitterBufferSamples); + printLog("Loaded settings.\n"); + sendAvatarVoxelURLMessage(_myAvatar.getVoxels()->getVoxelURL()); } @@ -2661,7 +2674,7 @@ void Application::loadSettings(QSettings* settings) { } _headCameraPitchYawScale = loadSetting(settings, "headCameraPitchYawScale", 0.0f); - + _audioJitterBufferSamples = loadSetting(settings, "audioJitterBufferSamples", 0); settings->beginGroup("View Frustum Offset Camera"); // in case settings is corrupt or missing loadSetting() will check for NaN _viewFrustumOffsetYaw = loadSetting(settings, "viewFrustumOffsetYaw" , 0.0f); @@ -2682,6 +2695,7 @@ void Application::saveSettings(QSettings* settings) { } settings->setValue("headCameraPitchYawScale", _headCameraPitchYawScale); + settings->setValue("audioJitterBufferSamples", _audioJitterBufferSamples); settings->beginGroup("View Frustum Offset Camera"); settings->setValue("viewFrustumOffsetYaw", _viewFrustumOffsetYaw); settings->setValue("viewFrustumOffsetPitch", _viewFrustumOffsetPitch); diff --git a/interface/src/Application.h b/interface/src/Application.h index 1d50b5065f..c13ecf23eb 100644 --- a/interface/src/Application.h +++ b/interface/src/Application.h @@ -76,6 +76,7 @@ public: QSettings* getSettings() { return _settings; } Environment* getEnvironment() { return &_environment; } bool shouldEchoAudio() { return _echoAudioMode->isChecked(); } + bool shouldLowPassFilter() { return _shouldLowPassFilter->isChecked(); } QNetworkAccessManager* getNetworkAccessManager() { return _networkAccessManager; } @@ -176,6 +177,7 @@ private: QAction* _lookingInMirror; // Are we currently rendering one's own head as if in mirror? QAction* _echoAudioMode; // Are we asking the mixer to echo back our audio? + QAction* _shouldLowPassFilter; // Use test lowpass filter QAction* _gyroLook; // Whether to allow the gyro data from head to move your view QAction* _renderAvatarBalls; // Switch between voxels and joints/balls for avatar render QAction* _mouseLook; // Whether the have the mouse near edge of screen move your view @@ -257,6 +259,8 @@ private: int _headMouseX, _headMouseY; float _headCameraPitchYawScale; + int _audioJitterBufferSamples; // Number of extra samples to wait before starting audio playback + HandControl _handControl; int _mouseX; diff --git a/interface/src/Audio.cpp b/interface/src/Audio.cpp index 1f1f9cc335..ad6ccae23c 100644 --- a/interface/src/Audio.cpp +++ b/interface/src/Audio.cpp @@ -25,6 +25,9 @@ #include "Util.h" #include "Log.h" +// Uncomment the following definition to test audio device latency by copying output to input +//#define TEST_AUDIO_LOOPBACK + const int NUM_AUDIO_CHANNELS = 2; const int PACKET_LENGTH_BYTES = 1024; @@ -41,13 +44,8 @@ const float FLANGE_BASE_RATE = 4; const float MAX_FLANGE_SAMPLE_WEIGHT = 0.50; const float MIN_FLANGE_INTENSITY = 0.25; -const float JITTER_BUFFER_LENGTH_MSECS = 12; -const short JITTER_BUFFER_SAMPLES = JITTER_BUFFER_LENGTH_MSECS * - NUM_AUDIO_CHANNELS * (SAMPLE_RATE / 1000.0); - const float AUDIO_CALLBACK_MSECS = (float)BUFFER_LENGTH_SAMPLES_PER_CHANNEL / (float)SAMPLE_RATE * 1000.0; - const int AGENT_LOOPBACK_MODIFIER = 307; int numStarves = 0; @@ -84,6 +82,13 @@ int audioCallback (const void* inputBuffer, int16_t* outputLeft = ((int16_t**) outputBuffer)[0]; int16_t* outputRight = ((int16_t**) outputBuffer)[1]; + // LowPass filter test + if (Application::getInstance()->shouldLowPassFilter()) { + parentAudio->lowPassFilter(inputLeft); + memcpy(parentAudio->_echoInputSamples, inputLeft, + PACKET_LENGTH_SAMPLES_PER_CHANNEL * sizeof(int16_t)); + } + // Add Procedural effects to input samples parentAudio->addProceduralSounds(inputLeft, BUFFER_LENGTH_SAMPLES_PER_CHANNEL); @@ -168,9 +173,10 @@ int audioCallback (const void* inputBuffer, if (ringBuffer->getEndOfLastWrite()) { - if (!ringBuffer->isStarted() && ringBuffer->diffLastWriteNextOutput() < PACKET_LENGTH_SAMPLES + JITTER_BUFFER_SAMPLES) { + if (!ringBuffer->isStarted() && ringBuffer->diffLastWriteNextOutput() < (PACKET_LENGTH_SAMPLES + parentAudio->_jitterBufferSamples)) { // printLog("Held back, buffer has %d of %d samples required.\n", -// ringBuffer->diffLastWriteNextOutput(), PACKET_LENGTH_SAMPLES + JITTER_BUFFER_SAMPLES); +// ringBuffer->diffLastWriteNextOutput(), +// PACKET_LENGTH_SAMPLES + parentAudio->_jitterBufferSamples); } else if (ringBuffer->diffLastWriteNextOutput() < PACKET_LENGTH_SAMPLES) { ringBuffer->setStarted(false); @@ -182,7 +188,9 @@ int audioCallback (const void* inputBuffer, } else { if (!ringBuffer->isStarted()) { ringBuffer->setStarted(true); - // printLog("starting playback %3.1f msecs delayed \n", (usecTimestampNow() - usecTimestamp(&firstPlaybackTimer))/1000.0); + //printLog("starting playback %3.1f msecs delayed, jitter buffer = %d \n", + // (usecTimestampNow() - usecTimestamp(&parentAudio->_firstPlaybackTime))/1000.0, + // parentAudio->_jitterBufferSamples); } else { // printLog("pushing buffer\n"); } @@ -248,9 +256,13 @@ int audioCallback (const void* inputBuffer, } } } - +#ifndef TEST_AUDIO_LOOPBACK outputLeft[s] = leftSample; outputRight[s] = rightSample; +#else + outputLeft[s] = inputLeft[s]; + outputRight[s] = inputLeft[s]; +#endif } ringBuffer->setNextOutput(ringBuffer->getNextOutput() + PACKET_LENGTH_SAMPLES); @@ -282,15 +294,13 @@ void outputPortAudioError(PaError error) { } } -Audio::Audio(Oscilloscope* scope) : +Audio::Audio(Oscilloscope* scope, int16_t initialJitterBufferSamples) : _stream(NULL), _ringBuffer(true), _scope(scope), _averagedLatency(0.0), _measuredJitter(0), - _jitterBufferLengthMsecs(12.0), - _jitterBufferSamples(_jitterBufferLengthMsecs * - NUM_AUDIO_CHANNELS * (SAMPLE_RATE / 1000.0)), + _jitterBufferSamples(initialJitterBufferSamples), _wasStarved(0), _lastInputLoudness(0), _lastVelocity(0), @@ -304,12 +314,20 @@ Audio::Audio(Oscilloscope* scope) : _isGatheringEchoOutputFrames(false) { outputPortAudioError(Pa_Initialize()); + + // NOTE: Portaudio documentation is unclear as to whether it is safe to specify the + // number of frames per buffer explicitly versus setting this value to zero. + // Possible source of latency that we need to investigate further. + // + unsigned long FRAMES_PER_BUFFER = BUFFER_LENGTH_SAMPLES_PER_CHANNEL; + //unsigned long FRAMES_PER_BUFFER = 0; + outputPortAudioError(Pa_OpenDefaultStream(&_stream, 2, 2, (paInt16 | paNonInterleaved), SAMPLE_RATE, - BUFFER_LENGTH_SAMPLES_PER_CHANNEL, + FRAMES_PER_BUFFER, audioCallback, (void*) this)); @@ -533,9 +551,28 @@ void Audio::render(int screenWidth, int screenHeight) { sprintf(out,"%3.1f\n", _measuredJitter); drawtext(startX + jitterPels - 5, topY-10, 0.10, 0, 1, 0, out, 0,1,1); - sprintf(out, "%3.1fms\n", JITTER_BUFFER_LENGTH_MSECS); - drawtext(startX - 10, bottomY + 15, 0.1, 0, 1, 0, out, 1, 0, 0); + //sprintf(out, "%3.1fms\n", JITTER_BUFFER_LENGTH_MSECS); + //drawtext(startX - 10, bottomY + 15, 0.1, 0, 1, 0, out, 1, 0, 0); } } +// +// Very Simple LowPass filter which works by averaging a bunch of samples with a moving window +// +void Audio::lowPassFilter(int16_t* inputBuffer) { + static int16_t outputBuffer[BUFFER_LENGTH_SAMPLES_PER_CHANNEL]; + for (int i = 2; i < BUFFER_LENGTH_SAMPLES_PER_CHANNEL - 2; i++) { + outputBuffer[i] = (int16_t)(0.125f * (float)inputBuffer[i - 2] + + 0.25f * (float)inputBuffer[i - 1] + + 0.25f * (float)inputBuffer[i] + + 0.25f * (float)inputBuffer[i + 1] + + 0.125f * (float)inputBuffer[i + 2] ); + } + outputBuffer[0] = inputBuffer[0]; + outputBuffer[1] = inputBuffer[1]; + outputBuffer[BUFFER_LENGTH_SAMPLES_PER_CHANNEL - 2] = inputBuffer[BUFFER_LENGTH_SAMPLES_PER_CHANNEL - 2]; + outputBuffer[BUFFER_LENGTH_SAMPLES_PER_CHANNEL - 1] = inputBuffer[BUFFER_LENGTH_SAMPLES_PER_CHANNEL - 1]; + memcpy(inputBuffer, outputBuffer, BUFFER_LENGTH_SAMPLES_PER_CHANNEL * sizeof(int16_t)); +} + #endif diff --git a/interface/src/Audio.h b/interface/src/Audio.h index 197daf0194..a90bfba64f 100644 --- a/interface/src/Audio.h +++ b/interface/src/Audio.h @@ -19,7 +19,7 @@ class Audio { public: // initializes audio I/O - Audio(Oscilloscope* scope); + Audio(Oscilloscope* scope, int16_t initialJitterBufferSamples); ~Audio(); void render(int screenWidth, int screenHeight); @@ -29,10 +29,11 @@ public: void setLastAcceleration(glm::vec3 lastAcceleration) { _lastAcceleration = lastAcceleration; }; void setLastVelocity(glm::vec3 lastVelocity) { _lastVelocity = lastVelocity; }; - void addProceduralSounds(int16_t* inputBuffer, int numSamples); - void analyzeEcho(int16_t* inputBuffer, int16_t* outputBuffer, int numSamples); - + void setJitterBufferSamples(int samples) { _jitterBufferSamples = samples; }; + void addProceduralSounds(int16_t* inputBuffer, int numSamples); + void lowPassFilter(int16_t* inputBuffer); + void analyzeEcho(int16_t* inputBuffer, int16_t* outputBuffer, int numSamples); void addReceivedAudioToBuffer(unsigned char* receivedData, int receivedBytes); void startEchoTest(); @@ -46,8 +47,7 @@ private: timeval _lastReceiveTime; float _averagedLatency; float _measuredJitter; - float _jitterBufferLengthMsecs; - short _jitterBufferSamples; + int16_t _jitterBufferSamples; int _wasStarved; float _lastInputLoudness; glm::vec3 _lastVelocity; diff --git a/interface/src/Head.cpp b/interface/src/Head.cpp index 63fc6d6c25..3ef99fa542 100644 --- a/interface/src/Head.cpp +++ b/interface/src/Head.cpp @@ -485,9 +485,7 @@ void Head::renderEyeBalls() { _irisProgram->bind(); glBindTexture(GL_TEXTURE_2D, _irisTextureID); glEnable(GL_TEXTURE_2D); - - glm::vec3 front = getFrontDirection(); - + // render left iris glPushMatrix(); { glTranslatef(_leftEyePosition.x, _leftEyePosition.y, _leftEyePosition.z); //translate to eyeball position From 2c8c34f9bdc64c69bbc3518afd343206e48adbaf Mon Sep 17 00:00:00 2001 From: Philip Rosedale Date: Thu, 20 Jun 2013 09:08:37 -0700 Subject: [PATCH 2/4] set jitter buffer on application start to 1/2 of a packet --- interface/src/Application.cpp | 6 +- interface/src/Audio.cpp | 82 ++++++++++++++++++--------- interface/src/Audio.h | 7 +++ libraries/audio/src/AudioRingBuffer.h | 2 +- 4 files changed, 67 insertions(+), 30 deletions(-) diff --git a/interface/src/Application.cpp b/interface/src/Application.cpp index ce7312fbef..d53e6a4327 100644 --- a/interface/src/Application.cpp +++ b/interface/src/Application.cpp @@ -72,6 +72,10 @@ const glm::vec3 START_LOCATION(4.f, 0.f, 5.f); // Where one's own agent begin const int IDLE_SIMULATE_MSECS = 16; // How often should call simulate and other stuff // in the idle loop? (60 FPS is default) +const int STARTUP_JITTER_SAMPLES = PACKET_LENGTH_SAMPLES_PER_CHANNEL / 2; + // Startup optimistically with small jitter buffer that + // will start playback on the second received audio packet. + // customized canvas that simply forwards requests/events to the singleton application class GLCanvas : public QGLWidget { protected: @@ -155,7 +159,7 @@ Application::Application(int& argc, char** argv, timeval &startup_time) : _oculusProgram(0), _oculusDistortionScale(1.25), #ifndef _WIN32 - _audio(&_audioScope, 0), + _audio(&_audioScope, STARTUP_JITTER_SAMPLES), #endif _stopNetworkReceiveThread(false), _packetCount(0), diff --git a/interface/src/Audio.cpp b/interface/src/Audio.cpp index 1df4f72a7b..edc0737b60 100644 --- a/interface/src/Audio.cpp +++ b/interface/src/Audio.cpp @@ -28,12 +28,6 @@ // Uncomment the following definition to test audio device latency by copying output to input //#define TEST_AUDIO_LOOPBACK -const int NUM_AUDIO_CHANNELS = 2; - -const int PACKET_LENGTH_BYTES = 1024; -const int PACKET_LENGTH_BYTES_PER_CHANNEL = PACKET_LENGTH_BYTES / 2; -const int PACKET_LENGTH_SAMPLES = PACKET_LENGTH_BYTES / sizeof(int16_t); -const int PACKET_LENGTH_SAMPLES_PER_CHANNEL = PACKET_LENGTH_SAMPLES / 2; const int PHASE_DELAY_AT_90 = 20; const float AMPLITUDE_RATIO_AT_90 = 0.5; @@ -172,31 +166,39 @@ int audioCallback (const void* inputBuffer, // just play some silence if (ringBuffer->getEndOfLastWrite()) { - if (!ringBuffer->isStarted() && ringBuffer->diffLastWriteNextOutput() < (PACKET_LENGTH_SAMPLES + parentAudio->_jitterBufferSamples)) { -// printLog("Held back, buffer has %d of %d samples required.\n", -// ringBuffer->diffLastWriteNextOutput(), -// PACKET_LENGTH_SAMPLES + parentAudio->_jitterBufferSamples); + // + // If not enough audio has arrived to start playback, keep waiting + // + //printLog("Held back, buffer has %d of %d samples required.\n", + // ringBuffer->diffLastWriteNextOutput(), + // PACKET_LENGTH_SAMPLES + parentAudio->_jitterBufferSamples); } else if (ringBuffer->diffLastWriteNextOutput() < PACKET_LENGTH_SAMPLES) { + // + // If we have run out of audio to send to the audio device, we have starved, + // so reset the ring buffer and packet counters. + // ringBuffer->setStarted(false); ::numStarves++; parentAudio->_packetsReceivedThisPlayback = 0; - printLog("Starved, remaining buffer msecs = %.0f\n", - ringBuffer->diffLastWriteNextOutput() / PACKET_LENGTH_SAMPLES * AUDIO_CALLBACK_MSECS); + //printLog("Starved, remaining buffer msecs = %.0f\n", + // ringBuffer->diffLastWriteNextOutput() / PACKET_LENGTH_SAMPLES * AUDIO_CALLBACK_MSECS); - parentAudio->_wasStarved = 10; // Frames to render the indication that the system was starved. + parentAudio->_wasStarved = 10; // display frame countdown to show red 'starved' bar. } else { + // + // We are either already playing back, or we have enough audio to start playing back. + // if (!ringBuffer->isStarted()) { ringBuffer->setStarted(true); - printLog("starting playback %0.1f msecs delayed, jitter = %d, pkts recvd: %d \n", - (usecTimestampNow() - usecTimestamp(&parentAudio->_firstPacketReceivedTime))/1000.0, - parentAudio->_jitterBufferSamples, - parentAudio->_packetsReceivedThisPlayback); - } else { - // printLog("pushing buffer\n"); + //printLog("starting playback %0.1f msecs delayed, jitter = %d, pkts recvd: %d \n", + // (usecTimestampNow() - usecTimestamp(&parentAudio->_firstPacketReceivedTime))/1000.0, + // parentAudio->_jitterBufferSamples, + // parentAudio->_packetsReceivedThisPlayback); } + // // play whatever we have in the audio buffer - + // // if we haven't fired off the flange effect, check if we should // TODO: lastMeasuredHeadYaw is now relative to body - check if this still works. @@ -322,14 +324,34 @@ Audio::Audio(Oscilloscope* scope, int16_t initialJitterBufferSamples) : // unsigned long FRAMES_PER_BUFFER = BUFFER_LENGTH_SAMPLES_PER_CHANNEL; + #define PA_SAMPLE_TYPE paInt16 + PaStreamParameters inputParameters, outputParameters; + + inputParameters.device = Pa_GetDefaultInputDevice(); + inputParameters.channelCount = 2; // Stereo input + inputParameters.sampleFormat = (paInt16 | paNonInterleaved); + //inputParameters.suggestedLatency = Pa_GetDeviceInfo( inputParameters.device )->defaultLowInputLatency; + inputParameters.suggestedLatency = 0.0116; + inputParameters.hostApiSpecificStreamInfo = NULL; + + outputParameters.device = Pa_GetDefaultOutputDevice(); + outputParameters.channelCount = 2; // Stereo output + outputParameters.sampleFormat = (paInt16 | paNonInterleaved); + //outputParameters.suggestedLatency = Pa_GetDeviceInfo( outputParameters.device )->defaultLowOutputLatency; + outputParameters.suggestedLatency = 0.0116; + outputParameters.hostApiSpecificStreamInfo = NULL; + + + outputPortAudioError(Pa_OpenStream(&_stream, + &inputParameters, + &outputParameters, + SAMPLE_RATE, + FRAMES_PER_BUFFER, + paNoFlag, + audioCallback, + (void*) this)); + /* - const PaStreamParameters* inputParameters; - const PaStreamParameters* outputParameters; - inputParameters->channelCount = - - outputPortAudioError(Pa_OpenStream(&_stream, <#const PaStreamParameters *inputParameters#>, <#const PaStreamParameters *outputParameters#>, SAMPLE_RATE, FRAMES_PER_BUFFER, <#PaStreamFlags streamFlags#>, audioCallback, (void*) this)); - */ - outputPortAudioError(Pa_OpenDefaultStream(&_stream, 2, 2, @@ -338,9 +360,13 @@ Audio::Audio(Oscilloscope* scope, int16_t initialJitterBufferSamples) : FRAMES_PER_BUFFER, audioCallback, (void*) this)); + */ // start the stream now that sources are good to go outputPortAudioError(Pa_StartStream(_stream)); + printLog("Default low input, output latency (secs): %0.4f, %0.4f\n", + Pa_GetDeviceInfo(Pa_GetDefaultInputDevice())->defaultLowInputLatency, + Pa_GetDeviceInfo(Pa_GetDefaultOutputDevice())->defaultLowOutputLatency); const PaStreamInfo* streamInfo = Pa_GetStreamInfo(_stream); printLog("Audio started, msecs latency In: %.0f, Out: %.0f\n", streamInfo->inputLatency * 1000.f, @@ -494,7 +520,7 @@ void Audio::render(int screenWidth, int screenHeight) { glVertex2f(currentX, topY); glVertex2f(currentX, bottomY); - for (int i = 0; i < RING_BUFFER_LENGTH_FRAMES; i++) { + for (int i = 0; i < RING_BUFFER_LENGTH_FRAMES / 2; i++) { glVertex2f(currentX, halfY); glVertex2f(currentX + frameWidth, halfY); currentX += frameWidth; diff --git a/interface/src/Audio.h b/interface/src/Audio.h index 5c5c73b2be..c3b36a1fd1 100644 --- a/interface/src/Audio.h +++ b/interface/src/Audio.h @@ -16,6 +16,13 @@ #include "Oscilloscope.h" #include "Avatar.h" +const int NUM_AUDIO_CHANNELS = 2; + +const int PACKET_LENGTH_BYTES = 1024; +const int PACKET_LENGTH_BYTES_PER_CHANNEL = PACKET_LENGTH_BYTES / 2; +const int PACKET_LENGTH_SAMPLES = PACKET_LENGTH_BYTES / sizeof(int16_t); +const int PACKET_LENGTH_SAMPLES_PER_CHANNEL = PACKET_LENGTH_SAMPLES / 2; + class Audio { public: // initializes audio I/O diff --git a/libraries/audio/src/AudioRingBuffer.h b/libraries/audio/src/AudioRingBuffer.h index 4ef8c4ead5..a3c6e59594 100644 --- a/libraries/audio/src/AudioRingBuffer.h +++ b/libraries/audio/src/AudioRingBuffer.h @@ -22,7 +22,7 @@ const int BUFFER_LENGTH_BYTES_STEREO = 1024; const int BUFFER_LENGTH_BYTES_PER_CHANNEL = 512; const int BUFFER_LENGTH_SAMPLES_PER_CHANNEL = BUFFER_LENGTH_BYTES_PER_CHANNEL / sizeof(int16_t); -const short RING_BUFFER_LENGTH_FRAMES = 10; +const short RING_BUFFER_LENGTH_FRAMES = 20; const short RING_BUFFER_LENGTH_SAMPLES = RING_BUFFER_LENGTH_FRAMES * BUFFER_LENGTH_SAMPLES_PER_CHANNEL; class AudioRingBuffer : public AgentData { From 56b8a5880d62ebf6c89b04b2b796fa3e58a5f2df Mon Sep 17 00:00:00 2001 From: Philip Rosedale Date: Thu, 20 Jun 2013 10:59:50 -0700 Subject: [PATCH 3/4] cleaned up stdout audio reporting --- interface/src/Audio.cpp | 27 +++++++-------------------- 1 file changed, 7 insertions(+), 20 deletions(-) diff --git a/interface/src/Audio.cpp b/interface/src/Audio.cpp index fb5d5ca1d5..f8c0b65038 100644 --- a/interface/src/Audio.cpp +++ b/interface/src/Audio.cpp @@ -357,17 +357,14 @@ Audio::Audio(Oscilloscope* scope, int16_t initialJitterBufferSamples) : inputParameters.channelCount = 2; // Stereo input inputParameters.sampleFormat = (paInt16 | paNonInterleaved); inputParameters.suggestedLatency = Pa_GetDeviceInfo( inputParameters.device )->defaultLowInputLatency; - //inputParameters.suggestedLatency = 0.0116; inputParameters.hostApiSpecificStreamInfo = NULL; outputParameters.device = Pa_GetDefaultOutputDevice(); outputParameters.channelCount = 2; // Stereo output outputParameters.sampleFormat = (paInt16 | paNonInterleaved); outputParameters.suggestedLatency = Pa_GetDeviceInfo( outputParameters.device )->defaultLowOutputLatency; - //outputParameters.suggestedLatency = 0.0116; outputParameters.hostApiSpecificStreamInfo = NULL; - outputPortAudioError(Pa_OpenStream(&_stream, &inputParameters, &outputParameters, @@ -376,18 +373,7 @@ Audio::Audio(Oscilloscope* scope, int16_t initialJitterBufferSamples) : paNoFlag, audioCallback, (void*) this)); - - /* - outputPortAudioError(Pa_OpenDefaultStream(&_stream, - 2, - 2, - (paInt16 | paNonInterleaved), - SAMPLE_RATE, - FRAMES_PER_BUFFER, - audioCallback, - (void*) this)); - */ - + if (! _stream) { return; } @@ -428,15 +414,16 @@ Audio::Audio(Oscilloscope* scope, int16_t initialJitterBufferSamples) : // start the stream now that sources are good to go outputPortAudioError(Pa_StartStream(_stream)); - printLog("Default low input, output latency (secs): %0.4f, %0.4f\n", - Pa_GetDeviceInfo(Pa_GetDefaultInputDevice())->defaultLowInputLatency, - Pa_GetDeviceInfo(Pa_GetDefaultOutputDevice())->defaultLowOutputLatency); + + // Uncomment these lines to see the system-reported latency + //printLog("Default low input, output latency (secs): %0.4f, %0.4f\n", + // Pa_GetDeviceInfo(Pa_GetDefaultInputDevice())->defaultLowInputLatency, + // Pa_GetDeviceInfo(Pa_GetDefaultOutputDevice())->defaultLowOutputLatency); const PaStreamInfo* streamInfo = Pa_GetStreamInfo(_stream); - printLog("Audio started, msecs latency In: %.0f, Out: %.0f\n", streamInfo->inputLatency * 1000.f, + printLog("Started audio with reported latency msecs In/Out: %.0f, %.0f\n", streamInfo->inputLatency * 1000.f, streamInfo->outputLatency * 1000.f); - gettimeofday(&_lastReceiveTime, NULL); } From 13b729d9b46111e064f6c1e1ac5bd126065815e6 Mon Sep 17 00:00:00 2001 From: Philip Rosedale Date: Thu, 20 Jun 2013 18:49:05 -0700 Subject: [PATCH 4/4] Fixed problems with ring buffer not handling stereo right, reset audio buffers on spacebar. --- interface/src/Application.cpp | 1 + interface/src/Audio.cpp | 50 ++++++++++++++-------- interface/src/Audio.h | 1 + interface/src/Avatar.cpp | 12 +++++- interface/src/Balls.cpp | 57 +++++++++++++++++-------- interface/src/Balls.h | 13 +++--- libraries/audio/src/AudioRingBuffer.cpp | 6 +++ libraries/audio/src/AudioRingBuffer.h | 6 ++- 8 files changed, 101 insertions(+), 45 deletions(-) diff --git a/interface/src/Application.cpp b/interface/src/Application.cpp index 9daab8e42f..65cde3c7be 100644 --- a/interface/src/Application.cpp +++ b/interface/src/Application.cpp @@ -537,6 +537,7 @@ void Application::keyPressEvent(QKeyEvent* event) { case Qt::Key_Space: resetSensors(); + _audio.reset(); break; case Qt::Key_G: diff --git a/interface/src/Audio.cpp b/interface/src/Audio.cpp index f8c0b65038..fd72826dfb 100644 --- a/interface/src/Audio.cpp +++ b/interface/src/Audio.cpp @@ -28,7 +28,7 @@ // Uncomment the following definition to test audio device latency by copying output to input //#define TEST_AUDIO_LOOPBACK - +//#define SHOW_AUDIO_DEBUG #define VISUALIZE_ECHO_CANCELLATION @@ -152,41 +152,50 @@ inline void Audio::performIO(int16_t* inputLeft, int16_t* outputLeft, int16_t* o AudioRingBuffer* ringBuffer = &_ringBuffer; - // if we've been reset, and there isn't any new packets yet - // just play some silence + // if there is anything in the ring buffer, decide what to do: if (ringBuffer->getEndOfLastWrite()) { - if (!ringBuffer->isStarted() && ringBuffer->diffLastWriteNextOutput() < (PACKET_LENGTH_SAMPLES + _jitterBufferSamples)) { + if (!ringBuffer->isStarted() && ringBuffer->diffLastWriteNextOutput() < (PACKET_LENGTH_SAMPLES + _jitterBufferSamples * (ringBuffer->isStereo() ? 2 : 1))) { // // If not enough audio has arrived to start playback, keep waiting // - //printLog("Held back, buffer has %d of %d samples required.\n", - // ringBuffer->diffLastWriteNextOutput(), - // PACKET_LENGTH_SAMPLES + parentAudio->_jitterBufferSamples); - } else if (ringBuffer->diffLastWriteNextOutput() < PACKET_LENGTH_SAMPLES) { +#ifdef SHOW_AUDIO_DEBUG + printLog("%i,%i,%i,%i\n", + _packetsReceivedThisPlayback, + ringBuffer->diffLastWriteNextOutput(), + PACKET_LENGTH_SAMPLES, + _jitterBufferSamples); +#endif + } else if (ringBuffer->isStarted() && (ringBuffer->diffLastWriteNextOutput() + < PACKET_LENGTH_SAMPLES * (ringBuffer->isStereo() ? 2 : 1))) { // - // If we have run out of audio to send to the audio device, we have starved, - // so reset the ring buffer and packet counters. + // If we have started and now have run out of audio to send to the audio device, + // this means we've starved and should restart. // ringBuffer->setStarted(false); _numStarves++; _packetsReceivedThisPlayback = 0; - - //printLog("Starved, remaining buffer msecs = %.0f\n", - // ringBuffer->diffLastWriteNextOutput() / PACKET_LENGTH_SAMPLES * AUDIO_CALLBACK_MSECS); - _wasStarved = 10; // Frames to render the indication that the system was starved. + _wasStarved = 10; // Frames for which to render the indication that the system was starved. +#ifdef SHOW_AUDIO_DEBUG + printLog("Starved, remaining samples = %.0f\n", + ringBuffer->diffLastWriteNextOutput()); +#endif + } else { // // We are either already playing back, or we have enough audio to start playing back. // if (!ringBuffer->isStarted()) { ringBuffer->setStarted(true); - //printLog("starting playback %0.1f msecs delayed, jitter = %d, pkts recvd: %d \n", - // (usecTimestampNow() - usecTimestamp(&parentAudio->_firstPacketReceivedTime))/1000.0, - // parentAudio->_jitterBufferSamples, - // parentAudio->_packetsReceivedThisPlayback); +#ifdef SHOW_AUDIO_DEBUG + printLog("starting playback %0.1f msecs delayed, jitter = %d, pkts recvd: %d \n", + (usecTimestampNow() - usecTimestamp(&_firstPacketReceivedTime))/1000.0, + _jitterBufferSamples, + _packetsReceivedThisPlayback); +#endif } + // // play whatever we have in the audio buffer // @@ -313,6 +322,11 @@ static void outputPortAudioError(PaError error) { } } +void Audio::reset() { + _packetsReceivedThisPlayback = 0; + _ringBuffer.reset(); +} + Audio::Audio(Oscilloscope* scope, int16_t initialJitterBufferSamples) : _stream(NULL), _ringBuffer(true), diff --git a/interface/src/Audio.h b/interface/src/Audio.h index 09ff961f7d..1e2c69616f 100644 --- a/interface/src/Audio.h +++ b/interface/src/Audio.h @@ -33,6 +33,7 @@ public: Audio(Oscilloscope* scope, int16_t initialJitterBufferSamples); ~Audio(); + void reset(); void render(int screenWidth, int screenHeight); void addReceivedAudioToBuffer(unsigned char* receivedData, int receivedBytes); diff --git a/interface/src/Avatar.cpp b/interface/src/Avatar.cpp index d659a5efdb..5588fef328 100644 --- a/interface/src/Avatar.cpp +++ b/interface/src/Avatar.cpp @@ -443,7 +443,16 @@ void Avatar::simulate(float deltaTime, Transmitter* transmitter) { } // update balls - if (_balls) { _balls->simulate(deltaTime); } + if (_balls) { + _balls->moveOrigin(_position); + glm::vec3 lookAt = _head.getLookAtPosition(); + if (glm::length(lookAt) > EPSILON) { + _balls->moveOrigin(lookAt); + } else { + _balls->moveOrigin(_position); + } + _balls->simulate(deltaTime); + } // update torso rotation based on head lean _skeleton.joint[AVATAR_JOINT_TORSO].rotation = glm::quat(glm::radians(glm::vec3( @@ -978,7 +987,6 @@ void Avatar::render(bool lookingInMirror, bool renderAvatarBalls) { // Render the balls if (_balls) { glPushMatrix(); - glTranslatef(_position.x, _position.y, _position.z); _balls->render(); glPopMatrix(); } diff --git a/interface/src/Balls.cpp b/interface/src/Balls.cpp index 5f88ff512f..49932c84ce 100644 --- a/interface/src/Balls.cpp +++ b/interface/src/Balls.cpp @@ -7,36 +7,55 @@ // A cloud of spring-mass spheres to simulate the avatar body/skin. Each ball // connects to as many as 4 neighbors, and executes motion according to a damped // spring, while responding physically to other avatars. -// +// +#include +#include "Util.h" +#include "sharedUtil.h" +#include "world.h" +#include "InterfaceConfig.h" #include "Balls.h" +const float INITIAL_AREA = 0.2f; +const float BALL_RADIUS = 0.025f; +const glm::vec3 INITIAL_COLOR(0.62f, 0.74f, 0.91f); + Balls::Balls(int numberOfBalls) { _numberOfBalls = numberOfBalls; _balls = new Ball[_numberOfBalls]; for (unsigned int i = 0; i < _numberOfBalls; ++i) { - _balls[i].position = glm::vec3(1.0 + randFloat() * 0.5, - 0.5 + randFloat() * 0.5, - 1.0 + randFloat() * 0.5); - _balls[i].radius = 0.02 + randFloat() * 0.06; + _balls[i].position = randVector() * INITIAL_AREA; + _balls[i].targetPosition = _balls[i].position; + _balls[i].velocity = glm::vec3(0, 0, 0); + _balls[i].radius = BALL_RADIUS; for (unsigned int j = 0; j < NUMBER_SPRINGS; ++j) { - _balls[i].links[j] = rand() % (numberOfBalls + 1); - if (_balls[i].links[j]-1 == i) { _balls[i].links[j] = 0; } - _balls[i].springLength[j] = 0.5; - } + _balls[i].links[j] = NULL; + } + } + _color = INITIAL_COLOR; + _origin = glm::vec3(0, 0, 0); +} + +void Balls::moveOrigin(const glm::vec3& newOrigin) { + glm::vec3 delta = newOrigin - _origin; + if (glm::length(delta) > EPSILON) { + _origin = newOrigin; + for (unsigned int i = 0; i < _numberOfBalls; ++i) { + _balls[i].targetPosition += delta; + } } } -const bool RENDER_SPRINGS = true; +const bool RENDER_SPRINGS = false; void Balls::render() { // Render Balls NOTE: This needs to become something other that GlutSpheres! - glColor3f(0.62,0.74,0.91); + glColor3fv(&_color.x); for (unsigned int i = 0; i < _numberOfBalls; ++i) { glPushMatrix(); glTranslatef(_balls[i].position.x, _balls[i].position.y, _balls[i].position.z); - glutSolidSphere(_balls[i].radius, 15, 15); + glutSolidSphere(_balls[i].radius, 8, 8); glPopMatrix(); } @@ -71,18 +90,22 @@ void Balls::simulate(float deltaTime) { // Move particles _balls[i].position += _balls[i].velocity * deltaTime; + _balls[i].targetPosition += _balls[i].velocity * deltaTime; // Drag: decay velocity _balls[i].velocity *= (1.f - CONSTANT_VELOCITY_DAMPING * deltaTime); // Add noise - _balls[i].velocity += glm::vec3((randFloat() - 0.5) * NOISE_SCALE, - (randFloat() - 0.5) * NOISE_SCALE, - (randFloat() - 0.5) * NOISE_SCALE); + _balls[i].velocity += randVector() * NOISE_SCALE; + // Approach target position + for (unsigned int i = 0; i < _numberOfBalls; ++i) { + _balls[i].position += randFloat() * deltaTime * (_balls[i].targetPosition - _balls[i].position); + } + // Spring Force - + /* for (unsigned int j = 0; j < NUMBER_SPRINGS; ++j) { if(_balls[i].links[j] > 0) { float separation = glm::distance(_balls[i].position, @@ -96,7 +119,7 @@ void Balls::simulate(float deltaTime) { //_balls[i].velocity *= (1.f - SPRING_DAMPING*deltaTime); } - } + } */ diff --git a/interface/src/Balls.h b/interface/src/Balls.h index 653854a0c6..440f340307 100644 --- a/interface/src/Balls.h +++ b/interface/src/Balls.h @@ -9,12 +9,6 @@ #ifndef hifi_Balls_h #define hifi_Balls_h -#include -#include "Util.h" -#include "world.h" -#include "InterfaceConfig.h" - - const int NUMBER_SPRINGS = 4; class Balls { @@ -24,14 +18,19 @@ public: void simulate(float deltaTime); void render(); + void setColor(const glm::vec3& c) { _color = c; }; + void moveOrigin(const glm::vec3& newOrigin); + private: struct Ball { - glm::vec3 position, velocity; + glm::vec3 position, targetPosition, velocity; int links[NUMBER_SPRINGS]; float springLength[NUMBER_SPRINGS]; float radius; } *_balls; int _numberOfBalls; + glm::vec3 _origin; + glm::vec3 _color; }; #endif diff --git a/libraries/audio/src/AudioRingBuffer.cpp b/libraries/audio/src/AudioRingBuffer.cpp index 19af07d600..2d396439aa 100644 --- a/libraries/audio/src/AudioRingBuffer.cpp +++ b/libraries/audio/src/AudioRingBuffer.cpp @@ -27,6 +27,12 @@ AudioRingBuffer::~AudioRingBuffer() { delete[] _buffer; } +void AudioRingBuffer::reset() { + _endOfLastWrite = _buffer; + _nextOutput = _buffer; + _isStarted = false; +} + int AudioRingBuffer::parseData(unsigned char* sourceBuffer, int numBytes) { return parseAudioSamples(sourceBuffer + sizeof(PACKET_HEADER_MIXED_AUDIO), numBytes - sizeof(PACKET_HEADER_MIXED_AUDIO)); } diff --git a/libraries/audio/src/AudioRingBuffer.h b/libraries/audio/src/AudioRingBuffer.h index a3c6e59594..4a86e1242b 100644 --- a/libraries/audio/src/AudioRingBuffer.h +++ b/libraries/audio/src/AudioRingBuffer.h @@ -30,6 +30,7 @@ public: AudioRingBuffer(bool isStereo); ~AudioRingBuffer(); + void reset(); int parseData(unsigned char* sourceBuffer, int numBytes); int parseAudioSamples(unsigned char* sourceBuffer, int numBytes); @@ -44,8 +45,11 @@ public: bool isStarted() const { return _isStarted; } void setStarted(bool isStarted) { _isStarted = isStarted; } - + int diffLastWriteNextOutput() const; + + bool isStereo() const { return _isStereo; } + protected: // disallow copying of AudioRingBuffer objects AudioRingBuffer(const AudioRingBuffer&);