From 37aa90932237b981e9e00730afd73daf1d82e4a4 Mon Sep 17 00:00:00 2001 From: Andrzej Kapolka Date: Mon, 9 Sep 2013 11:22:46 -0700 Subject: [PATCH 1/4] Try using the eye coefficients rather than the eye directions reported by Faceshift. --- interface/src/devices/Faceshift.cpp | 45 ++++++++++++++++++++++++++++- interface/src/devices/Faceshift.h | 10 +++++++ 2 files changed, 54 insertions(+), 1 deletion(-) diff --git a/interface/src/devices/Faceshift.cpp b/interface/src/devices/Faceshift.cpp index 4418d19223..5e35d23415 100644 --- a/interface/src/devices/Faceshift.cpp +++ b/interface/src/devices/Faceshift.cpp @@ -26,7 +26,11 @@ Faceshift::Faceshift() : _browHeight(0.0f), _browUpCenterIndex(-1), _mouthSize(0.0f), - _jawOpenIndex(-1) + _jawOpenIndex(-1), + _leftEyeUpIndex(-1), + _leftEyeInIndex(-1), + _rightEyeUpIndex(-1), + _rightEyeInIndex(-1) { connect(&_socket, SIGNAL(connected()), SLOT(noteConnected())); connect(&_socket, SIGNAL(error(QAbstractSocket::SocketError)), SLOT(noteError(QAbstractSocket::SocketError))); @@ -108,6 +112,21 @@ void Faceshift::readFromSocket() { if (_jawOpenIndex != -1) { _mouthSize = data.m_coeffs[_jawOpenIndex]; } + const float PITCH_SCALE = 45.0f; + if (_leftEyeUpIndex != -1) { + _eyeGazeLeftPitch = PITCH_SCALE * (data.m_coeffs[_leftEyeUpIndex] - data.m_coeffs[_leftEyeDownIndex]); + } + const float YAW_SCALE = 45.0f; + if (_leftEyeInIndex != -1) { + _eyeGazeLeftYaw = YAW_SCALE * (data.m_coeffs[_leftEyeOutIndex] - data.m_coeffs[_leftEyeInIndex]); + } + if (_rightEyeUpIndex != -1) { + _eyeGazeRightPitch = PITCH_SCALE * (data.m_coeffs[_rightEyeUpIndex] - + data.m_coeffs[_rightEyeDownIndex]); + } + if (_rightEyeInIndex != -1) { + _eyeGazeRightYaw = YAW_SCALE * (data.m_coeffs[_rightEyeInIndex] - data.m_coeffs[_rightEyeOutIndex]); + } } break; } @@ -125,6 +144,30 @@ void Faceshift::readFromSocket() { } else if (names[i] == "JawOpen") { _jawOpenIndex = i; + + } else if (names[i] == "EyeUp_L") { + _leftEyeUpIndex = i; + + } else if (names[i] == "EyeDown_L") { + _leftEyeDownIndex = i; + + } else if (names[i] == "EyeIn_L") { + _leftEyeInIndex = i; + + } else if (names[i] == "EyeOut_L") { + _leftEyeOutIndex = i; + + } else if (names[i] == "EyeUp_R") { + _rightEyeUpIndex = i; + + } else if (names[i] == "EyeDown_R") { + _rightEyeDownIndex = i; + + } else if (names[i] == "EyeIn_R") { + _rightEyeInIndex = i; + + } else if (names[i] == "EyeOut_R") { + _rightEyeOutIndex = i; } } break; diff --git a/interface/src/devices/Faceshift.h b/interface/src/devices/Faceshift.h index 286a7a56a1..745ce6f099 100644 --- a/interface/src/devices/Faceshift.h +++ b/interface/src/devices/Faceshift.h @@ -85,6 +85,16 @@ private: float _mouthSize; int _jawOpenIndex; + + int _leftEyeUpIndex; + int _leftEyeDownIndex; + int _leftEyeInIndex; + int _leftEyeOutIndex; + + int _rightEyeUpIndex; + int _rightEyeDownIndex; + int _rightEyeInIndex; + int _rightEyeOutIndex; }; #endif /* defined(__interface__Faceshift__) */ From 597c57a11740b529d80bded8a692ad9f05aa4d40 Mon Sep 17 00:00:00 2001 From: Andrzej Kapolka Date: Mon, 9 Sep 2013 12:17:12 -0700 Subject: [PATCH 2/4] Revert "Try using the eye coefficients rather than the eye directions reported by" This reverts commit 37aa90932237b981e9e00730afd73daf1d82e4a4. --- interface/src/devices/Faceshift.cpp | 45 +---------------------------- interface/src/devices/Faceshift.h | 10 ------- 2 files changed, 1 insertion(+), 54 deletions(-) diff --git a/interface/src/devices/Faceshift.cpp b/interface/src/devices/Faceshift.cpp index 5e35d23415..4418d19223 100644 --- a/interface/src/devices/Faceshift.cpp +++ b/interface/src/devices/Faceshift.cpp @@ -26,11 +26,7 @@ Faceshift::Faceshift() : _browHeight(0.0f), _browUpCenterIndex(-1), _mouthSize(0.0f), - _jawOpenIndex(-1), - _leftEyeUpIndex(-1), - _leftEyeInIndex(-1), - _rightEyeUpIndex(-1), - _rightEyeInIndex(-1) + _jawOpenIndex(-1) { connect(&_socket, SIGNAL(connected()), SLOT(noteConnected())); connect(&_socket, SIGNAL(error(QAbstractSocket::SocketError)), SLOT(noteError(QAbstractSocket::SocketError))); @@ -112,21 +108,6 @@ void Faceshift::readFromSocket() { if (_jawOpenIndex != -1) { _mouthSize = data.m_coeffs[_jawOpenIndex]; } - const float PITCH_SCALE = 45.0f; - if (_leftEyeUpIndex != -1) { - _eyeGazeLeftPitch = PITCH_SCALE * (data.m_coeffs[_leftEyeUpIndex] - data.m_coeffs[_leftEyeDownIndex]); - } - const float YAW_SCALE = 45.0f; - if (_leftEyeInIndex != -1) { - _eyeGazeLeftYaw = YAW_SCALE * (data.m_coeffs[_leftEyeOutIndex] - data.m_coeffs[_leftEyeInIndex]); - } - if (_rightEyeUpIndex != -1) { - _eyeGazeRightPitch = PITCH_SCALE * (data.m_coeffs[_rightEyeUpIndex] - - data.m_coeffs[_rightEyeDownIndex]); - } - if (_rightEyeInIndex != -1) { - _eyeGazeRightYaw = YAW_SCALE * (data.m_coeffs[_rightEyeInIndex] - data.m_coeffs[_rightEyeOutIndex]); - } } break; } @@ -144,30 +125,6 @@ void Faceshift::readFromSocket() { } else if (names[i] == "JawOpen") { _jawOpenIndex = i; - - } else if (names[i] == "EyeUp_L") { - _leftEyeUpIndex = i; - - } else if (names[i] == "EyeDown_L") { - _leftEyeDownIndex = i; - - } else if (names[i] == "EyeIn_L") { - _leftEyeInIndex = i; - - } else if (names[i] == "EyeOut_L") { - _leftEyeOutIndex = i; - - } else if (names[i] == "EyeUp_R") { - _rightEyeUpIndex = i; - - } else if (names[i] == "EyeDown_R") { - _rightEyeDownIndex = i; - - } else if (names[i] == "EyeIn_R") { - _rightEyeInIndex = i; - - } else if (names[i] == "EyeOut_R") { - _rightEyeOutIndex = i; } } break; diff --git a/interface/src/devices/Faceshift.h b/interface/src/devices/Faceshift.h index 745ce6f099..286a7a56a1 100644 --- a/interface/src/devices/Faceshift.h +++ b/interface/src/devices/Faceshift.h @@ -85,16 +85,6 @@ private: float _mouthSize; int _jawOpenIndex; - - int _leftEyeUpIndex; - int _leftEyeDownIndex; - int _leftEyeInIndex; - int _leftEyeOutIndex; - - int _rightEyeUpIndex; - int _rightEyeDownIndex; - int _rightEyeInIndex; - int _rightEyeOutIndex; }; #endif /* defined(__interface__Faceshift__) */ From 8fec78d82afb2be6ad9b0b93bcc935c77b94d6f2 Mon Sep 17 00:00:00 2001 From: Andrzej Kapolka Date: Mon, 9 Sep 2013 13:48:46 -0700 Subject: [PATCH 3/4] Disable random saccades for own eyes. --- interface/src/avatar/Head.cpp | 26 +++++++++++++------------- 1 file changed, 13 insertions(+), 13 deletions(-) diff --git a/interface/src/avatar/Head.cpp b/interface/src/avatar/Head.cpp index 08efb32062..abf16248c2 100644 --- a/interface/src/avatar/Head.cpp +++ b/interface/src/avatar/Head.cpp @@ -146,19 +146,6 @@ void Head::resetHairPhysics() { void Head::simulate(float deltaTime, bool isMine, float gyroCameraSensitivity) { - // Update eye saccades - const float AVERAGE_MICROSACCADE_INTERVAL = 0.50f; - const float AVERAGE_SACCADE_INTERVAL = 4.0f; - const float MICROSACCADE_MAGNITUDE = 0.002f; - const float SACCADE_MAGNITUDE = 0.04; - - if (randFloat() < deltaTime / AVERAGE_MICROSACCADE_INTERVAL) { - _saccadeTarget = MICROSACCADE_MAGNITUDE * randVector(); - } else if (randFloat() < deltaTime / AVERAGE_SACCADE_INTERVAL) { - _saccadeTarget = SACCADE_MAGNITUDE * randVector(); - } - _saccade += (_saccadeTarget - _saccade) * 0.50f; - // Update audio trailing average for rendering facial animations Faceshift* faceshift = Application::getInstance()->getFaceshift(); if (isMine && faceshift->isActive()) { @@ -173,6 +160,19 @@ void Head::simulate(float deltaTime, bool isMine, float gyroCameraSensitivity) { _browAudioLift = faceshift->getBrowHeight() * BROW_HEIGHT_SCALE; } else { + // Update eye saccades + const float AVERAGE_MICROSACCADE_INTERVAL = 0.50f; + const float AVERAGE_SACCADE_INTERVAL = 4.0f; + const float MICROSACCADE_MAGNITUDE = 0.002f; + const float SACCADE_MAGNITUDE = 0.04f; + + if (randFloat() < deltaTime / AVERAGE_MICROSACCADE_INTERVAL) { + _saccadeTarget = MICROSACCADE_MAGNITUDE * randVector(); + } else if (randFloat() < deltaTime / AVERAGE_SACCADE_INTERVAL) { + _saccadeTarget = SACCADE_MAGNITUDE * randVector(); + } + _saccade += (_saccadeTarget - _saccade) * 0.50f; + const float AUDIO_AVERAGING_SECS = 0.05; _averageLoudness = (1.f - deltaTime / AUDIO_AVERAGING_SECS) * _averageLoudness + (deltaTime / AUDIO_AVERAGING_SECS) * _audioLoudness; From f8aee88a5b03d41e5f171fffdb42b01405b3ea7b Mon Sep 17 00:00:00 2001 From: Andrzej Kapolka Date: Mon, 9 Sep 2013 14:30:26 -0700 Subject: [PATCH 4/4] Let's try subtracting the long-term average from the eye directions. --- interface/src/Application.cpp | 11 +++++------ interface/src/devices/Faceshift.cpp | 19 ++++++++++++++++++- interface/src/devices/Faceshift.h | 10 ++++++++++ 3 files changed, 33 insertions(+), 7 deletions(-) diff --git a/interface/src/Application.cpp b/interface/src/Application.cpp index ea61179057..36e6147d1d 100644 --- a/interface/src/Application.cpp +++ b/interface/src/Application.cpp @@ -1531,14 +1531,15 @@ void Application::update(float deltaTime) { // Set where I am looking based on my mouse ray (so that other people can see) glm::vec3 lookAtSpot; + // Update faceshift + _faceshift.update(); + // if we have faceshift, use that to compute the lookat direction glm::vec3 lookAtRayOrigin = mouseRayOrigin, lookAtRayDirection = mouseRayDirection; if (_faceshift.isActive()) { lookAtRayOrigin = _myAvatar.getHead().calculateAverageEyePosition(); - float averagePitch = (_faceshift.getEyeGazeLeftPitch() + _faceshift.getEyeGazeRightPitch()) / 2.0f; - float averageYaw = (_faceshift.getEyeGazeLeftYaw() + _faceshift.getEyeGazeRightYaw()) / 2.0f; - lookAtRayDirection = _myAvatar.getHead().getOrientation() * - glm::quat(glm::radians(glm::vec3(averagePitch, averageYaw, 0.0f))) * glm::vec3(0.0f, 0.0f, -1.0f); + lookAtRayDirection = _myAvatar.getHead().getOrientation() * glm::quat(glm::radians(glm::vec3( + _faceshift.getEstimatedEyePitch(), _faceshift.getEstimatedEyeYaw(), 0.0f))) * glm::vec3(0.0f, 0.0f, -1.0f); } _isLookingAtOtherAvatar = isLookingAtOtherAvatar(lookAtRayOrigin, lookAtRayDirection, lookAtSpot); @@ -1707,8 +1708,6 @@ void Application::update(float deltaTime) { _serialHeadSensor.readData(deltaTime); } - // Update transmitter - // Sample hardware, update view frustum if needed, and send avatar data to mixer/nodes updateAvatar(deltaTime); diff --git a/interface/src/devices/Faceshift.cpp b/interface/src/devices/Faceshift.cpp index 4418d19223..399369cdd3 100644 --- a/interface/src/devices/Faceshift.cpp +++ b/interface/src/devices/Faceshift.cpp @@ -26,13 +26,30 @@ Faceshift::Faceshift() : _browHeight(0.0f), _browUpCenterIndex(-1), _mouthSize(0.0f), - _jawOpenIndex(-1) + _jawOpenIndex(-1), + _longTermAverageEyePitch(0.0f), + _longTermAverageEyeYaw(0.0f), + _estimatedEyePitch(0.0f), + _estimatedEyeYaw(0.0f) { connect(&_socket, SIGNAL(connected()), SLOT(noteConnected())); connect(&_socket, SIGNAL(error(QAbstractSocket::SocketError)), SLOT(noteError(QAbstractSocket::SocketError))); connect(&_socket, SIGNAL(readyRead()), SLOT(readFromSocket())); } +void Faceshift::update() { + if (!isActive()) { + return; + } + float averageEyePitch = (_eyeGazeLeftPitch + _eyeGazeRightPitch) / 2.0f; + float averageEyeYaw = (_eyeGazeLeftYaw + _eyeGazeRightYaw) / 2.0f; + const float LONG_TERM_AVERAGE_SMOOTHING = 0.999f; + _longTermAverageEyePitch = glm::mix(averageEyePitch, _longTermAverageEyePitch, LONG_TERM_AVERAGE_SMOOTHING); + _longTermAverageEyeYaw = glm::mix(averageEyeYaw, _longTermAverageEyeYaw, LONG_TERM_AVERAGE_SMOOTHING); + _estimatedEyePitch = averageEyePitch - _longTermAverageEyePitch; + _estimatedEyeYaw = averageEyeYaw - _longTermAverageEyeYaw; +} + void Faceshift::reset() { if (isActive()) { string message; diff --git a/interface/src/devices/Faceshift.h b/interface/src/devices/Faceshift.h index 286a7a56a1..408513e5d7 100644 --- a/interface/src/devices/Faceshift.h +++ b/interface/src/devices/Faceshift.h @@ -35,6 +35,9 @@ public: float getEyeGazeRightPitch() const { return _eyeGazeRightPitch; } float getEyeGazeRightYaw() const { return _eyeGazeRightYaw; } + float getEstimatedEyePitch() const { return _estimatedEyePitch; } + float getEstimatedEyeYaw() const { return _estimatedEyeYaw; } + float getLeftBlink() const { return _leftBlink; } float getRightBlink() const { return _rightBlink; } @@ -42,6 +45,7 @@ public: float getMouthSize() const { return _mouthSize; } + void update(); void reset(); public slots: @@ -85,6 +89,12 @@ private: float _mouthSize; int _jawOpenIndex; + + float _longTermAverageEyePitch; + float _longTermAverageEyeYaw; + + float _estimatedEyePitch; + float _estimatedEyeYaw; }; #endif /* defined(__interface__Faceshift__) */