From 7836fd8a8501b4ec0f9de23b2123a1e669b78133 Mon Sep 17 00:00:00 2001 From: Philip Rosedale Date: Thu, 25 Sep 2014 11:58:41 -0700 Subject: [PATCH 1/5] Add world space eye locations (for making eye contact) to the oculus manager --- interface/src/Application.cpp | 8 ++++++-- interface/src/avatar/MyAvatar.cpp | 6 +++++- interface/src/devices/OculusManager.cpp | 14 +++++++++++++- interface/src/devices/OculusManager.h | 7 +++++++ 4 files changed, 31 insertions(+), 4 deletions(-) diff --git a/interface/src/Application.cpp b/interface/src/Application.cpp index 3b46b26468..13aa953365 100644 --- a/interface/src/Application.cpp +++ b/interface/src/Application.cpp @@ -1995,8 +1995,12 @@ void Application::updateMyAvatarLookAtPosition() { glm::vec3 lookAtSpot; if (_myCamera.getMode() == CAMERA_MODE_MIRROR) { // When I am in mirror mode, just look right at the camera (myself) - lookAtSpot = _myCamera.getPosition(); - + if (!OculusManager::isConnected()) { + lookAtSpot = _myCamera.getPosition(); + } else { + lookAtSpot = OculusManager::getLeftEyePosition(); + } + } else { AvatarSharedPointer lookingAt = _myAvatar->getLookAtTargetAvatar().toStrongRef(); if (lookingAt && _myAvatar != lookingAt.data()) { diff --git a/interface/src/avatar/MyAvatar.cpp b/interface/src/avatar/MyAvatar.cpp index 51f582c4f8..012c4560cf 100644 --- a/interface/src/avatar/MyAvatar.cpp +++ b/interface/src/avatar/MyAvatar.cpp @@ -978,7 +978,11 @@ void MyAvatar::updateLookAtTargetAvatar() { howManyLookingAtMe++; // Have that avatar look directly at my camera // Philip TODO: correct to look at left/right eye - avatar->getHead()->setCorrectedLookAtPosition(Application::getInstance()->getViewFrustum()->getPosition()); + if (OculusManager::isConnected()) { + avatar->getHead()->setCorrectedLookAtPosition(OculusManager::getLeftEyePosition()); + } else { + avatar->getHead()->setCorrectedLookAtPosition(Application::getInstance()->getViewFrustum()->getPosition()); + } } else { avatar->getHead()->clearCorrectedLookAtPosition(); } diff --git a/interface/src/devices/OculusManager.cpp b/interface/src/devices/OculusManager.cpp index c47ff2c38b..8fc17ee4d6 100644 --- a/interface/src/devices/OculusManager.cpp +++ b/interface/src/devices/OculusManager.cpp @@ -55,6 +55,9 @@ bool OculusManager::_programInitialized = false; Camera* OculusManager::_camera = NULL; int OculusManager::_activeEyeIndex = -1; +glm::vec3 OculusManager::_leftEyePosition; +glm::vec3 OculusManager::_rightEyePosition; + #endif void OculusManager::connect() { @@ -348,6 +351,15 @@ void OculusManager::display(const glm::quat &bodyOrientation, const glm::vec3 &p _camera->setTargetRotation(bodyOrientation * orientation); _camera->setTargetPosition(position + trackerPosition); + // Store the latest left and right eye render locations for things that need to know + if (eyeIndex == 0) { + //_leftEyePosition = position + trackerPosition; + glm::vec3 leftEye = position + trackerPosition; + qDebug() << "LE: " << leftEye.x << ", " << leftEye.y << ", " << leftEye.z; + } else { + whichCamera.setRightEyePosition(position + trackerPosition); + } + _camera->update(1.0f / Application::getInstance()->getFps()); Matrix4f proj = ovrMatrix4f_Projection(_eyeRenderDesc[eye].Fov, whichCamera.getNearClip(), whichCamera.getFarClip(), true); @@ -362,7 +374,7 @@ void OculusManager::display(const glm::quat &bodyOrientation, const glm::vec3 &p glMatrixMode(GL_MODELVIEW); glLoadIdentity(); glTranslatef(_eyeRenderDesc[eye].ViewAdjust.x, _eyeRenderDesc[eye].ViewAdjust.y, _eyeRenderDesc[eye].ViewAdjust.z); - + Application::getInstance()->displaySide(*_camera); applicationOverlay.displayOverlayTextureOculus(*_camera); diff --git a/interface/src/devices/OculusManager.h b/interface/src/devices/OculusManager.h index d8e55647b4..460ec0e765 100644 --- a/interface/src/devices/OculusManager.h +++ b/interface/src/devices/OculusManager.h @@ -45,6 +45,9 @@ public: static void overrideOffAxisFrustum(float& left, float& right, float& bottom, float& top, float& nearVal, float& farVal, glm::vec4& nearClipPlane, glm::vec4& farClipPlane); + + static glm::vec3 getLeftEyePosition() { return _leftEyePosition; } + static glm::vec3 getRightEyePosition() { return _rightEyePosition; } private: #ifdef HAVE_LIBOVR @@ -96,6 +99,10 @@ private: static bool _programInitialized; static Camera* _camera; static int _activeEyeIndex; + + static glm::vec3 _leftEyePosition; + static glm::vec3 _rightEyePosition; + #endif }; From 1d4b86d85b93bf78c5cb5deb2c1f34aa41a52708 Mon Sep 17 00:00:00 2001 From: Philip Rosedale Date: Thu, 25 Sep 2014 16:00:56 -0700 Subject: [PATCH 2/5] Blink first with one eye and at slightly different speeds, look back and forth exactly at eyes when receiving gaze in Oculus --- interface/src/Application.cpp | 14 +++++++++++--- interface/src/avatar/Head.cpp | 15 +++++++++++---- interface/src/avatar/MyAvatar.cpp | 11 ++++++++++- interface/src/avatar/MyAvatar.h | 4 ++++ interface/src/devices/OculusManager.cpp | 10 ++++++---- 5 files changed, 42 insertions(+), 12 deletions(-) diff --git a/interface/src/Application.cpp b/interface/src/Application.cpp index 81a1ba3e14..1d586f8868 100644 --- a/interface/src/Application.cpp +++ b/interface/src/Application.cpp @@ -2001,7 +2001,11 @@ void Application::updateMyAvatarLookAtPosition() { if (!OculusManager::isConnected()) { lookAtSpot = _myCamera.getPosition(); } else { - lookAtSpot = OculusManager::getLeftEyePosition(); + if (_myAvatar->isLookingAtLeftEye()) { + lookAtSpot = OculusManager::getLeftEyePosition(); + } else { + lookAtSpot = OculusManager::getRightEyePosition(); + } } } else { @@ -2011,7 +2015,7 @@ void Application::updateMyAvatarLookAtPosition() { isLookingAtSomeone = true; // If I am looking at someone else, look directly at one of their eyes if (tracker) { - // If tracker active, look at the eye for the side my gaze is biased toward + // If a face tracker is active, look at the eye for the side my gaze is biased toward if (tracker->getEstimatedEyeYaw() > _myAvatar->getHead()->getFinalYaw()) { // Look at their right eye lookAtSpot = static_cast(lookingAt.data())->getHead()->getRightEyePosition(); @@ -2021,7 +2025,11 @@ void Application::updateMyAvatarLookAtPosition() { } } else { // Need to add randomly looking back and forth between left and right eye for case with no tracker - lookAtSpot = static_cast(lookingAt.data())->getHead()->getEyePosition(); + if (_myAvatar->isLookingAtLeftEye()) { + lookAtSpot = static_cast(lookingAt.data())->getHead()->getLeftEyePosition(); + } else { + lookAtSpot = static_cast(lookingAt.data())->getHead()->getRightEyePosition(); + } } } else { // I am not looking at anyone else, so just look forward diff --git a/interface/src/avatar/Head.cpp b/interface/src/avatar/Head.cpp index 9c41cf8084..85e771d3dd 100644 --- a/interface/src/avatar/Head.cpp +++ b/interface/src/avatar/Head.cpp @@ -110,7 +110,7 @@ void Head::simulate(float deltaTime, bool isMine, bool billboard) { bool forceBlink = false; const float TALKING_LOUDNESS = 100.0f; const float BLINK_AFTER_TALKING = 0.25f; - if (_averageLoudness > TALKING_LOUDNESS) { + if ((_averageLoudness - _longTermAverageLoudness) > TALKING_LOUDNESS) { _timeWithoutTalking = 0.0f; } else if (_timeWithoutTalking < BLINK_AFTER_TALKING && (_timeWithoutTalking += deltaTime) >= BLINK_AFTER_TALKING) { @@ -128,16 +128,23 @@ void Head::simulate(float deltaTime, bool isMine, bool billboard) { _browAudioLift = glm::clamp(_browAudioLift *= 0.7f, 0.0f, 1.0f); const float BLINK_SPEED = 10.0f; + const float BLINK_SPEED_VARIABILITY = 1.0f; + const float BLINK_START_VARIABILITY = 0.25f; const float FULLY_OPEN = 0.0f; const float FULLY_CLOSED = 1.0f; if (_leftEyeBlinkVelocity == 0.0f && _rightEyeBlinkVelocity == 0.0f) { // no blinking when brows are raised; blink less with increasing loudness const float BASE_BLINK_RATE = 15.0f / 60.0f; const float ROOT_LOUDNESS_TO_BLINK_INTERVAL = 0.25f; - if (forceBlink || (_browAudioLift < EPSILON && shouldDo(glm::max(1.0f, sqrt(_averageLoudness) * + if (forceBlink || (_browAudioLift < EPSILON && shouldDo(glm::max(1.0f, sqrt(fabs(_averageLoudness - _longTermAverageLoudness)) * ROOT_LOUDNESS_TO_BLINK_INTERVAL) / BASE_BLINK_RATE, deltaTime))) { - _leftEyeBlinkVelocity = BLINK_SPEED; - _rightEyeBlinkVelocity = BLINK_SPEED; + _leftEyeBlinkVelocity = BLINK_SPEED + randFloat() * BLINK_SPEED_VARIABILITY; + _rightEyeBlinkVelocity = BLINK_SPEED + randFloat() * BLINK_SPEED_VARIABILITY; + if (randFloat() < 0.5f) { + _leftEyeBlink = BLINK_START_VARIABILITY; + } else { + _rightEyeBlink = BLINK_START_VARIABILITY; + } } } else { _leftEyeBlink = glm::clamp(_leftEyeBlink + _leftEyeBlinkVelocity * deltaTime, FULLY_OPEN, FULLY_CLOSED); diff --git a/interface/src/avatar/MyAvatar.cpp b/interface/src/avatar/MyAvatar.cpp index 012c4560cf..b4fbadeaa9 100644 --- a/interface/src/avatar/MyAvatar.cpp +++ b/interface/src/avatar/MyAvatar.cpp @@ -86,7 +86,8 @@ MyAvatar::MyAvatar() : _shouldRender(true), _billboardValid(false), _physicsSimulation(), - _voxelShapeManager() + _voxelShapeManager(), + _isLookingAtLeftEye(true) { ShapeCollider::initDispatchTable(); for (int i = 0; i < MAX_DRIVE_KEYS; i++) { @@ -997,6 +998,14 @@ void MyAvatar::clearLookAtTargetAvatar() { _lookAtTargetAvatar.clear(); } +bool MyAvatar::isLookingAtLeftEye() { + float const CHANCE_OF_CHANGING_EYE = 0.01f; + if (randFloat() < CHANCE_OF_CHANGING_EYE) { + _isLookingAtLeftEye = !_isLookingAtLeftEye; + } + return _isLookingAtLeftEye; +} + glm::vec3 MyAvatar::getUprightHeadPosition() const { return _position + getWorldAlignedOrientation() * glm::vec3(0.0f, getPelvisToHeadLength(), 0.0f); } diff --git a/interface/src/avatar/MyAvatar.h b/interface/src/avatar/MyAvatar.h index e38f3f4b2b..14b3ea9924 100644 --- a/interface/src/avatar/MyAvatar.h +++ b/interface/src/avatar/MyAvatar.h @@ -105,6 +105,8 @@ public: void jump() { _shouldJump = true; }; bool isMyAvatar() { return true; } + + bool isLookingAtLeftEye(); virtual int parseDataAtOffset(const QByteArray& packet, int offset); @@ -229,6 +231,8 @@ private: QList _animationHandles; PhysicsSimulation _physicsSimulation; VoxelShapeManager _voxelShapeManager; + + bool _isLookingAtLeftEye; RecorderPointer _recorder; diff --git a/interface/src/devices/OculusManager.cpp b/interface/src/devices/OculusManager.cpp index 8fc17ee4d6..5f5a5650c9 100644 --- a/interface/src/devices/OculusManager.cpp +++ b/interface/src/devices/OculusManager.cpp @@ -352,12 +352,14 @@ void OculusManager::display(const glm::quat &bodyOrientation, const glm::vec3 &p _camera->setTargetPosition(position + trackerPosition); // Store the latest left and right eye render locations for things that need to know + glm::vec3 thisEyePosition = position + trackerPosition + + (bodyOrientation * glm::quat(orientation.x, orientation.y, orientation.z, orientation.w) * + glm::vec3(_eyeRenderDesc[eye].ViewAdjust.x, _eyeRenderDesc[eye].ViewAdjust.y, _eyeRenderDesc[eye].ViewAdjust.z)); + if (eyeIndex == 0) { - //_leftEyePosition = position + trackerPosition; - glm::vec3 leftEye = position + trackerPosition; - qDebug() << "LE: " << leftEye.x << ", " << leftEye.y << ", " << leftEye.z; + _leftEyePosition = thisEyePosition; } else { - whichCamera.setRightEyePosition(position + trackerPosition); + _rightEyePosition = thisEyePosition; } _camera->update(1.0f / Application::getInstance()->getFps()); From 3fd1ab40e43714dd2e8c7e0a77b5b3082a8f3a3b Mon Sep 17 00:00:00 2001 From: Philip Rosedale Date: Thu, 25 Sep 2014 16:17:05 -0700 Subject: [PATCH 3/5] remove magic number --- interface/src/avatar/Head.cpp | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/interface/src/avatar/Head.cpp b/interface/src/avatar/Head.cpp index 85e771d3dd..f67a131221 100644 --- a/interface/src/avatar/Head.cpp +++ b/interface/src/avatar/Head.cpp @@ -118,7 +118,8 @@ void Head::simulate(float deltaTime, bool isMine, bool billboard) { } // Update audio attack data for facial animation (eyebrows and mouth) - _audioAttack = 0.9f * _audioAttack + 0.1f * fabs((_audioLoudness - _longTermAverageLoudness) - _lastLoudness); + const float AUDIO_ATTACK_AVERAGING_RATE = 0.9f; + _audioAttack = AUDIO_ATTACK_AVERAGING_RATE * _audioAttack + (1.0f - AUDIO_ATTACK_AVERAGING_RATE) * fabs((_audioLoudness - _longTermAverageLoudness) - _lastLoudness); _lastLoudness = (_audioLoudness - _longTermAverageLoudness); const float BROW_LIFT_THRESHOLD = 100.0f; From 597871442e7c74064b59a4b6983050f298520a90 Mon Sep 17 00:00:00 2001 From: Philip Rosedale Date: Thu, 25 Sep 2014 17:39:53 -0700 Subject: [PATCH 4/5] fix linux build fail --- interface/src/devices/OculusManager.h | 13 ++++++++++--- 1 file changed, 10 insertions(+), 3 deletions(-) diff --git a/interface/src/devices/OculusManager.h b/interface/src/devices/OculusManager.h index 460ec0e765..09880243e9 100644 --- a/interface/src/devices/OculusManager.h +++ b/interface/src/devices/OculusManager.h @@ -46,9 +46,15 @@ public: static void overrideOffAxisFrustum(float& left, float& right, float& bottom, float& top, float& nearVal, float& farVal, glm::vec4& nearClipPlane, glm::vec4& farClipPlane); +#ifdef HAVE_LIBOVR static glm::vec3 getLeftEyePosition() { return _leftEyePosition; } static glm::vec3 getRightEyePosition() { return _rightEyePosition; } - +#else + static glm::vec3 getLeftEyePosition() { return glm::vec3(); } + static glm::vec3 getRightEyePosition() { return glm::vec3(); } +#endif + + private: #ifdef HAVE_LIBOVR static void generateDistortionMesh(); @@ -99,11 +105,12 @@ private: static bool _programInitialized; static Camera* _camera; static int _activeEyeIndex; - static glm::vec3 _leftEyePosition; static glm::vec3 _rightEyePosition; - + #endif + + }; #endif // hifi_OculusManager_h From f66f565f534bad68560bebea85d7b6fbef592e52 Mon Sep 17 00:00:00 2001 From: Philip Rosedale Date: Thu, 25 Sep 2014 18:14:02 -0700 Subject: [PATCH 5/5] update stDev (standard deviation) class to coding standard --- libraries/shared/src/StdDev.cpp | 38 ++++++++++++--------------------- libraries/shared/src/StdDev.h | 20 ++++++++--------- 2 files changed, 24 insertions(+), 34 deletions(-) diff --git a/libraries/shared/src/StdDev.cpp b/libraries/shared/src/StdDev.cpp index c5d78345e4..387b67cf4f 100644 --- a/libraries/shared/src/StdDev.cpp +++ b/libraries/shared/src/StdDev.cpp @@ -13,50 +13,40 @@ #include #include "StdDev.h" -const int MAX_STDEV_SAMPLES = 1000; +const int NUM_SAMPLES = 1000; StDev::StDev() { - data = new float[MAX_STDEV_SAMPLES]; - sampleCount = 0; + _data = new float[NUM_SAMPLES]; + _sampleCount = 0; } void StDev::reset() { - sampleCount = 0; + _sampleCount = 0; } void StDev::addValue(float v) { - data[sampleCount++] = v; - if (sampleCount == MAX_STDEV_SAMPLES) sampleCount = 0; + _data[_sampleCount++] = v; + if (_sampleCount == NUM_SAMPLES) _sampleCount = 0; } float StDev::getAverage() const { float average = 0; - for (int i = 0; i < sampleCount; i++) { - average += data[i]; + for (int i = 0; i < _sampleCount; i++) { + average += _data[i]; } - if (sampleCount > 0) - return average/(float)sampleCount; + if (_sampleCount > 0) + return average / (float)_sampleCount; else return 0; } -/* -float StDev::getMax() { - float average = -FLT_MAX; - for (int i = 0; i < sampleCount; i++) { - average += data[i]; - } - if (sampleCount > 0) - return average/(float)sampleCount; - else return 0; -}*/ float StDev::getStDev() const { float average = getAverage(); float stdev = 0; - for (int i = 0; i < sampleCount; i++) { - stdev += powf(data[i] - average, 2); + for (int i = 0; i < _sampleCount; i++) { + stdev += powf(_data[i] - average, 2); } - if (sampleCount > 1) - return sqrt(stdev/(float)(sampleCount - 1.0)); + if (_sampleCount > 1) + return sqrt(stdev / (float)(_sampleCount - 1.0f)); else return 0; } diff --git a/libraries/shared/src/StdDev.h b/libraries/shared/src/StdDev.h index a05cc32992..40148007a9 100644 --- a/libraries/shared/src/StdDev.h +++ b/libraries/shared/src/StdDev.h @@ -13,16 +13,16 @@ #define hifi_StdDev_h class StDev { - public: - StDev(); - void reset(); - void addValue(float v); - float getAverage() const; - float getStDev() const; - int getSamples() const { return sampleCount; } - private: - float * data; - int sampleCount; +public: + StDev(); + void reset(); + void addValue(float v); + float getAverage() const; + float getStDev() const; + int getSamples() const { return _sampleCount; } +private: + float* _data; + int _sampleCount; }; #endif // hifi_StdDev_h