diff --git a/interface/src/Application.cpp b/interface/src/Application.cpp index 1acad36d3b..feffb3a453 100644 --- a/interface/src/Application.cpp +++ b/interface/src/Application.cpp @@ -155,7 +155,7 @@ Application::Application(int& argc, char** argv, timeval &startup_time) : _oculusProgram(0), _oculusDistortionScale(1.25), #ifndef _WIN32 - _audio(&_audioScope), + _audio(&_audioScope, 0), #endif _stopNetworkReceiveThread(false), _packetCount(0), @@ -889,7 +889,13 @@ void Application::editPreferences() { QDoubleSpinBox* leanScale = new QDoubleSpinBox(); leanScale->setValue(_myAvatar.getLeanScale()); form->addRow("Lean Scale:", leanScale); - + + QSpinBox* audioJitterBufferSamples = new QSpinBox(); + audioJitterBufferSamples->setMaximum(10000); + audioJitterBufferSamples->setMinimum(-10000); + audioJitterBufferSamples->setValue(_audioJitterBufferSamples); + form->addRow("Audio Jitter Buffer Samples:", audioJitterBufferSamples); + QDialogButtonBox* buttons = new QDialogButtonBox(QDialogButtonBox::Ok | QDialogButtonBox::Cancel); dialog.connect(buttons, SIGNAL(accepted()), SLOT(accept())); dialog.connect(buttons, SIGNAL(rejected()), SLOT(reject())); @@ -903,7 +909,9 @@ void Application::editPreferences() { sendAvatarVoxelURLMessage(url); _headCameraPitchYawScale = headCameraPitchYawScale->value(); + _audioJitterBufferSamples = audioJitterBufferSamples->value(); _myAvatar.setLeanScale(leanScale->value()); + _audio.setJitterBufferSamples(audioJitterBufferSamples->value()); } void Application::pair() { @@ -1341,6 +1349,8 @@ void Application::initMenu() { debugMenu->addAction("Wants Res-In", this, SLOT(setWantsResIn(bool)))->setCheckable(true); debugMenu->addAction("Wants Monochrome", this, SLOT(setWantsMonochrome(bool)))->setCheckable(true); debugMenu->addAction("Wants View Delta Sending", this, SLOT(setWantsDelta(bool)))->setCheckable(true); + (_shouldLowPassFilter = debugMenu->addAction("Test: LowPass filter"))->setCheckable(true); + QMenu* settingsMenu = menuBar->addMenu("Settings"); (_settingsAutosave = settingsMenu->addAction("Autosave"))->setCheckable(true); @@ -1420,6 +1430,9 @@ void Application::init() { gettimeofday(&_lastTimeIdle, NULL); loadSettings(); + _audio.setJitterBufferSamples(_audioJitterBufferSamples); + printLog("Loaded settings.\n"); + sendAvatarVoxelURLMessage(_myAvatar.getVoxels()->getVoxelURL()); } @@ -2661,7 +2674,7 @@ void Application::loadSettings(QSettings* settings) { } _headCameraPitchYawScale = loadSetting(settings, "headCameraPitchYawScale", 0.0f); - + _audioJitterBufferSamples = loadSetting(settings, "audioJitterBufferSamples", 0); settings->beginGroup("View Frustum Offset Camera"); // in case settings is corrupt or missing loadSetting() will check for NaN _viewFrustumOffsetYaw = loadSetting(settings, "viewFrustumOffsetYaw" , 0.0f); @@ -2682,6 +2695,7 @@ void Application::saveSettings(QSettings* settings) { } settings->setValue("headCameraPitchYawScale", _headCameraPitchYawScale); + settings->setValue("audioJitterBufferSamples", _audioJitterBufferSamples); settings->beginGroup("View Frustum Offset Camera"); settings->setValue("viewFrustumOffsetYaw", _viewFrustumOffsetYaw); settings->setValue("viewFrustumOffsetPitch", _viewFrustumOffsetPitch); diff --git a/interface/src/Application.h b/interface/src/Application.h index 1d50b5065f..c13ecf23eb 100644 --- a/interface/src/Application.h +++ b/interface/src/Application.h @@ -76,6 +76,7 @@ public: QSettings* getSettings() { return _settings; } Environment* getEnvironment() { return &_environment; } bool shouldEchoAudio() { return _echoAudioMode->isChecked(); } + bool shouldLowPassFilter() { return _shouldLowPassFilter->isChecked(); } QNetworkAccessManager* getNetworkAccessManager() { return _networkAccessManager; } @@ -176,6 +177,7 @@ private: QAction* _lookingInMirror; // Are we currently rendering one's own head as if in mirror? QAction* _echoAudioMode; // Are we asking the mixer to echo back our audio? + QAction* _shouldLowPassFilter; // Use test lowpass filter QAction* _gyroLook; // Whether to allow the gyro data from head to move your view QAction* _renderAvatarBalls; // Switch between voxels and joints/balls for avatar render QAction* _mouseLook; // Whether the have the mouse near edge of screen move your view @@ -257,6 +259,8 @@ private: int _headMouseX, _headMouseY; float _headCameraPitchYawScale; + int _audioJitterBufferSamples; // Number of extra samples to wait before starting audio playback + HandControl _handControl; int _mouseX; diff --git a/interface/src/Audio.cpp b/interface/src/Audio.cpp index 1f1f9cc335..ad6ccae23c 100644 --- a/interface/src/Audio.cpp +++ b/interface/src/Audio.cpp @@ -25,6 +25,9 @@ #include "Util.h" #include "Log.h" +// Uncomment the following definition to test audio device latency by copying output to input +//#define TEST_AUDIO_LOOPBACK + const int NUM_AUDIO_CHANNELS = 2; const int PACKET_LENGTH_BYTES = 1024; @@ -41,13 +44,8 @@ const float FLANGE_BASE_RATE = 4; const float MAX_FLANGE_SAMPLE_WEIGHT = 0.50; const float MIN_FLANGE_INTENSITY = 0.25; -const float JITTER_BUFFER_LENGTH_MSECS = 12; -const short JITTER_BUFFER_SAMPLES = JITTER_BUFFER_LENGTH_MSECS * - NUM_AUDIO_CHANNELS * (SAMPLE_RATE / 1000.0); - const float AUDIO_CALLBACK_MSECS = (float)BUFFER_LENGTH_SAMPLES_PER_CHANNEL / (float)SAMPLE_RATE * 1000.0; - const int AGENT_LOOPBACK_MODIFIER = 307; int numStarves = 0; @@ -84,6 +82,13 @@ int audioCallback (const void* inputBuffer, int16_t* outputLeft = ((int16_t**) outputBuffer)[0]; int16_t* outputRight = ((int16_t**) outputBuffer)[1]; + // LowPass filter test + if (Application::getInstance()->shouldLowPassFilter()) { + parentAudio->lowPassFilter(inputLeft); + memcpy(parentAudio->_echoInputSamples, inputLeft, + PACKET_LENGTH_SAMPLES_PER_CHANNEL * sizeof(int16_t)); + } + // Add Procedural effects to input samples parentAudio->addProceduralSounds(inputLeft, BUFFER_LENGTH_SAMPLES_PER_CHANNEL); @@ -168,9 +173,10 @@ int audioCallback (const void* inputBuffer, if (ringBuffer->getEndOfLastWrite()) { - if (!ringBuffer->isStarted() && ringBuffer->diffLastWriteNextOutput() < PACKET_LENGTH_SAMPLES + JITTER_BUFFER_SAMPLES) { + if (!ringBuffer->isStarted() && ringBuffer->diffLastWriteNextOutput() < (PACKET_LENGTH_SAMPLES + parentAudio->_jitterBufferSamples)) { // printLog("Held back, buffer has %d of %d samples required.\n", -// ringBuffer->diffLastWriteNextOutput(), PACKET_LENGTH_SAMPLES + JITTER_BUFFER_SAMPLES); +// ringBuffer->diffLastWriteNextOutput(), +// PACKET_LENGTH_SAMPLES + parentAudio->_jitterBufferSamples); } else if (ringBuffer->diffLastWriteNextOutput() < PACKET_LENGTH_SAMPLES) { ringBuffer->setStarted(false); @@ -182,7 +188,9 @@ int audioCallback (const void* inputBuffer, } else { if (!ringBuffer->isStarted()) { ringBuffer->setStarted(true); - // printLog("starting playback %3.1f msecs delayed \n", (usecTimestampNow() - usecTimestamp(&firstPlaybackTimer))/1000.0); + //printLog("starting playback %3.1f msecs delayed, jitter buffer = %d \n", + // (usecTimestampNow() - usecTimestamp(&parentAudio->_firstPlaybackTime))/1000.0, + // parentAudio->_jitterBufferSamples); } else { // printLog("pushing buffer\n"); } @@ -248,9 +256,13 @@ int audioCallback (const void* inputBuffer, } } } - +#ifndef TEST_AUDIO_LOOPBACK outputLeft[s] = leftSample; outputRight[s] = rightSample; +#else + outputLeft[s] = inputLeft[s]; + outputRight[s] = inputLeft[s]; +#endif } ringBuffer->setNextOutput(ringBuffer->getNextOutput() + PACKET_LENGTH_SAMPLES); @@ -282,15 +294,13 @@ void outputPortAudioError(PaError error) { } } -Audio::Audio(Oscilloscope* scope) : +Audio::Audio(Oscilloscope* scope, int16_t initialJitterBufferSamples) : _stream(NULL), _ringBuffer(true), _scope(scope), _averagedLatency(0.0), _measuredJitter(0), - _jitterBufferLengthMsecs(12.0), - _jitterBufferSamples(_jitterBufferLengthMsecs * - NUM_AUDIO_CHANNELS * (SAMPLE_RATE / 1000.0)), + _jitterBufferSamples(initialJitterBufferSamples), _wasStarved(0), _lastInputLoudness(0), _lastVelocity(0), @@ -304,12 +314,20 @@ Audio::Audio(Oscilloscope* scope) : _isGatheringEchoOutputFrames(false) { outputPortAudioError(Pa_Initialize()); + + // NOTE: Portaudio documentation is unclear as to whether it is safe to specify the + // number of frames per buffer explicitly versus setting this value to zero. + // Possible source of latency that we need to investigate further. + // + unsigned long FRAMES_PER_BUFFER = BUFFER_LENGTH_SAMPLES_PER_CHANNEL; + //unsigned long FRAMES_PER_BUFFER = 0; + outputPortAudioError(Pa_OpenDefaultStream(&_stream, 2, 2, (paInt16 | paNonInterleaved), SAMPLE_RATE, - BUFFER_LENGTH_SAMPLES_PER_CHANNEL, + FRAMES_PER_BUFFER, audioCallback, (void*) this)); @@ -533,9 +551,28 @@ void Audio::render(int screenWidth, int screenHeight) { sprintf(out,"%3.1f\n", _measuredJitter); drawtext(startX + jitterPels - 5, topY-10, 0.10, 0, 1, 0, out, 0,1,1); - sprintf(out, "%3.1fms\n", JITTER_BUFFER_LENGTH_MSECS); - drawtext(startX - 10, bottomY + 15, 0.1, 0, 1, 0, out, 1, 0, 0); + //sprintf(out, "%3.1fms\n", JITTER_BUFFER_LENGTH_MSECS); + //drawtext(startX - 10, bottomY + 15, 0.1, 0, 1, 0, out, 1, 0, 0); } } +// +// Very Simple LowPass filter which works by averaging a bunch of samples with a moving window +// +void Audio::lowPassFilter(int16_t* inputBuffer) { + static int16_t outputBuffer[BUFFER_LENGTH_SAMPLES_PER_CHANNEL]; + for (int i = 2; i < BUFFER_LENGTH_SAMPLES_PER_CHANNEL - 2; i++) { + outputBuffer[i] = (int16_t)(0.125f * (float)inputBuffer[i - 2] + + 0.25f * (float)inputBuffer[i - 1] + + 0.25f * (float)inputBuffer[i] + + 0.25f * (float)inputBuffer[i + 1] + + 0.125f * (float)inputBuffer[i + 2] ); + } + outputBuffer[0] = inputBuffer[0]; + outputBuffer[1] = inputBuffer[1]; + outputBuffer[BUFFER_LENGTH_SAMPLES_PER_CHANNEL - 2] = inputBuffer[BUFFER_LENGTH_SAMPLES_PER_CHANNEL - 2]; + outputBuffer[BUFFER_LENGTH_SAMPLES_PER_CHANNEL - 1] = inputBuffer[BUFFER_LENGTH_SAMPLES_PER_CHANNEL - 1]; + memcpy(inputBuffer, outputBuffer, BUFFER_LENGTH_SAMPLES_PER_CHANNEL * sizeof(int16_t)); +} + #endif diff --git a/interface/src/Audio.h b/interface/src/Audio.h index 197daf0194..a90bfba64f 100644 --- a/interface/src/Audio.h +++ b/interface/src/Audio.h @@ -19,7 +19,7 @@ class Audio { public: // initializes audio I/O - Audio(Oscilloscope* scope); + Audio(Oscilloscope* scope, int16_t initialJitterBufferSamples); ~Audio(); void render(int screenWidth, int screenHeight); @@ -29,10 +29,11 @@ public: void setLastAcceleration(glm::vec3 lastAcceleration) { _lastAcceleration = lastAcceleration; }; void setLastVelocity(glm::vec3 lastVelocity) { _lastVelocity = lastVelocity; }; - void addProceduralSounds(int16_t* inputBuffer, int numSamples); - void analyzeEcho(int16_t* inputBuffer, int16_t* outputBuffer, int numSamples); - + void setJitterBufferSamples(int samples) { _jitterBufferSamples = samples; }; + void addProceduralSounds(int16_t* inputBuffer, int numSamples); + void lowPassFilter(int16_t* inputBuffer); + void analyzeEcho(int16_t* inputBuffer, int16_t* outputBuffer, int numSamples); void addReceivedAudioToBuffer(unsigned char* receivedData, int receivedBytes); void startEchoTest(); @@ -46,8 +47,7 @@ private: timeval _lastReceiveTime; float _averagedLatency; float _measuredJitter; - float _jitterBufferLengthMsecs; - short _jitterBufferSamples; + int16_t _jitterBufferSamples; int _wasStarved; float _lastInputLoudness; glm::vec3 _lastVelocity; diff --git a/interface/src/Head.cpp b/interface/src/Head.cpp index 63fc6d6c25..3ef99fa542 100644 --- a/interface/src/Head.cpp +++ b/interface/src/Head.cpp @@ -485,9 +485,7 @@ void Head::renderEyeBalls() { _irisProgram->bind(); glBindTexture(GL_TEXTURE_2D, _irisTextureID); glEnable(GL_TEXTURE_2D); - - glm::vec3 front = getFrontDirection(); - + // render left iris glPushMatrix(); { glTranslatef(_leftEyePosition.x, _leftEyePosition.y, _leftEyePosition.z); //translate to eyeball position