Merge pull request #5348 from ctrlaltdavid/20602

DO NOT MERGE - QA for #20602 - Fix and improve eye saccades

interface/src/Application.cpp

@@ -2318,23 +2318,21 @@ void Application::updateMyAvatarLookAtPosition() {
             lookAtSpot = _myAvatar->getHead()->getEyePosition() +
                 (_myAvatar->getHead()->getFinalOrientationInWorldFrame() * glm::vec3(0.0f, 0.0f, -TREE_SCALE));
         }
-    }
 
-    // Deflect the eyes a bit to match the detected gaze from Faceshift if active.
+        // Deflect the eyes a bit to match the detected gaze from the face tracker if active.
-    // DDE doesn't track eyes.
+        if (tracker && !tracker->isMuted()) {
-    if (tracker && typeid(*tracker) == typeid(Faceshift) && !tracker->isMuted()) {
+            float eyePitch = tracker->getEstimatedEyePitch();
-        float eyePitch = tracker->getEstimatedEyePitch();
+            float eyeYaw = tracker->getEstimatedEyeYaw();
-        float eyeYaw = tracker->getEstimatedEyeYaw();
+            const float GAZE_DEFLECTION_REDUCTION_DURING_EYE_CONTACT = 0.1f;
-        const float GAZE_DEFLECTION_REDUCTION_DURING_EYE_CONTACT = 0.1f;
+            glm::vec3 origin = _myAvatar->getHead()->getEyePosition();
-        glm::vec3 origin = _myAvatar->getHead()->getEyePosition();
+            float deflection = tracker->getEyeDeflection();
-        float pitchSign = (_myCamera.getMode() == CAMERA_MODE_MIRROR) ? -1.0f : 1.0f;
+            if (isLookingAtSomeone) {
-        float deflection = DependencyManager::get<Faceshift>()->getEyeDeflection();
+                deflection *= GAZE_DEFLECTION_REDUCTION_DURING_EYE_CONTACT;
-        if (isLookingAtSomeone) {
+            }
-            deflection *= GAZE_DEFLECTION_REDUCTION_DURING_EYE_CONTACT;
+            lookAtSpot = origin + _myCamera.getRotation() * glm::quat(glm::radians(glm::vec3(
-        }
+                eyePitch * deflection, eyeYaw * deflection, 0.0f))) *
-        lookAtSpot = origin + _myCamera.getRotation() * glm::quat(glm::radians(glm::vec3(
-            eyePitch * pitchSign * deflection, eyeYaw * deflection, 0.0f))) *
                 glm::inverse(_myCamera.getRotation()) * (lookAtSpot - origin);
+        }
     }
 
     _myAvatar->getHead()->setLookAtPosition(lookAtSpot);

interface/src/avatar/FaceModel.cpp

@@ -72,8 +72,8 @@ void FaceModel::maybeUpdateEyeRotation(Model* model, const JointState& parentSta
             glm::translate(state.getDefaultTranslationInConstrainedFrame()) *
             joint.preTransform * glm::mat4_cast(joint.preRotation * joint.rotation));
     glm::vec3 front = glm::vec3(inverse * glm::vec4(_owningHead->getFinalOrientationInWorldFrame() * IDENTITY_FRONT, 0.0f));
-    glm::vec3 lookAt = glm::vec3(inverse * glm::vec4(_owningHead->getCorrectedLookAtPosition() +
+    glm::vec3 lookAtDelta = _owningHead->getCorrectedLookAtPosition() - model->getTranslation();
-        _owningHead->getSaccade() - model->getTranslation(), 1.0f));
+    glm::vec3 lookAt = glm::vec3(inverse * glm::vec4(lookAtDelta + glm::length(lookAtDelta) * _owningHead->getSaccade(), 1.0f));
     glm::quat between = rotationBetween(front, lookAt);
     const float MAX_ANGLE = 30.0f * RADIANS_PER_DEGREE;
     state.setRotationInConstrainedFrame(glm::angleAxis(glm::clamp(glm::angle(between), -MAX_ANGLE, MAX_ANGLE), glm::axis(between)) *

interface/src/avatar/Head.cpp

@@ -129,13 +129,14 @@ void Head::simulate(float deltaTime, bool isMine, bool billboard) {
         const float AVERAGE_SACCADE_INTERVAL = 6.0f;
         const float MICROSACCADE_MAGNITUDE = 0.002f;
         const float SACCADE_MAGNITUDE = 0.04f;
+        const float NOMINAL_FRAME_RATE = 60.0f;
 
         if (randFloat() < deltaTime / AVERAGE_MICROSACCADE_INTERVAL) {
             _saccadeTarget = MICROSACCADE_MAGNITUDE * randVector();
         } else if (randFloat() < deltaTime / AVERAGE_SACCADE_INTERVAL) {
             _saccadeTarget = SACCADE_MAGNITUDE * randVector();
         }
-        _saccade += (_saccadeTarget - _saccade) * 0.5f;
+        _saccade += (_saccadeTarget - _saccade) * pow(0.5f, NOMINAL_FRAME_RATE * deltaTime);
 
         //  Detect transition from talking to not; force blink after that and a delay
         bool forceBlink = false;
@@ -342,7 +343,8 @@ glm::quat Head::getCameraOrientation() const {
 
 glm::quat Head::getEyeRotation(const glm::vec3& eyePosition) const {
     glm::quat orientation = getOrientation();
-    return rotationBetween(orientation * IDENTITY_FRONT, _lookAtPosition + _saccade - eyePosition) * orientation;
+    glm::vec3 lookAtDelta = _lookAtPosition - eyePosition;
+    return rotationBetween(orientation * IDENTITY_FRONT, lookAtDelta + glm::length(lookAtDelta) * _saccade) * orientation;
 }
 
 glm::vec3 Head::getScalePivot() const {

interface/src/devices/DdeFaceTracker.cpp

@@ -157,6 +157,10 @@ DdeFaceTracker::DdeFaceTracker(const QHostAddress& host, quint16 serverPort, qui
     _reset(false),
     _leftBlinkIndex(0), // see http://support.faceshift.com/support/articles/35129-export-of-blendshapes
     _rightBlinkIndex(1),
+    _leftEyeDownIndex(4),
+    _rightEyeDownIndex(5),
+    _leftEyeInIndex(6),
+    _rightEyeInIndex(7),
     _leftEyeOpenIndex(8),
     _rightEyeOpenIndex(9),
     _browDownLeftIndex(14),
@@ -173,6 +177,14 @@ DdeFaceTracker::DdeFaceTracker(const QHostAddress& host, quint16 serverPort, qui
     _filteredHeadTranslation(glm::vec3(0.0f)),
     _lastBrowUp(0.0f),
     _filteredBrowUp(0.0f),
+    _eyePitch(0.0f),
+    _eyeYaw(0.0f),
+    _lastEyePitch(0.0f),
+    _lastEyeYaw(0.0f),
+    _filteredEyePitch(0.0f),
+    _filteredEyeYaw(0.0f),
+    _longTermAverageEyePitch(0.0f),
+    _longTermAverageEyeYaw(0.0f),
     _lastEyeBlinks(),
     _filteredEyeBlinks(),
     _lastEyeCoefficients(),
@@ -282,6 +294,17 @@ void DdeFaceTracker::reset() {
     }
 }
 
+void DdeFaceTracker::update(float deltaTime) {
+    if (!isActive()) {
+        return;
+    }
+    FaceTracker::update(deltaTime);
+
+    glm::vec3 headEulers = glm::degrees(glm::eulerAngles(_headRotation));
+    _estimatedEyePitch = _eyePitch - headEulers.x;
+    _estimatedEyeYaw = _eyeYaw - headEulers.y;
+}
+
 bool DdeFaceTracker::isActive() const {
     return (_ddeProcess != NULL);
 }
@@ -436,6 +459,28 @@ void DdeFaceTracker::decodePacket(const QByteArray& buffer) {
         _coefficients[_mouthSmileLeftIndex] = _coefficients[_mouthSmileLeftIndex] - SMILE_THRESHOLD;
         _coefficients[_mouthSmileRightIndex] = _coefficients[_mouthSmileRightIndex] - SMILE_THRESHOLD;
 
+        // Eye pitch and yaw
+        // EyeDown coefficients work better over both +ve and -ve values than EyeUp values.
+        // EyeIn coefficients work better over both +ve and -ve values than EyeOut values.
+        // Pitch and yaw values are relative to the screen.
+        const float EYE_PITCH_SCALE = -1500.0f;  // Sign, scale, and average to be similar to Faceshift values.
+        _eyePitch = EYE_PITCH_SCALE * (_coefficients[_leftEyeDownIndex] + _coefficients[_rightEyeDownIndex]);
+        const float EYE_YAW_SCALE = 2000.0f;  // Scale and average to be similar to Faceshift values.
+        _eyeYaw = EYE_YAW_SCALE * (_coefficients[_leftEyeInIndex] + _coefficients[_rightEyeInIndex]);
+        if (isFiltering) {
+            const float EYE_VELOCITY_FILTER_STRENGTH = 0.005f;
+            float pitchVelocity = fabsf(_eyePitch - _lastEyePitch) / _averageMessageTime;
+            float pitchVelocityFilter = glm::clamp(pitchVelocity * EYE_VELOCITY_FILTER_STRENGTH, 0.0f, 1.0f);
+            _filteredEyePitch = pitchVelocityFilter * _eyePitch + (1.0f - pitchVelocityFilter) * _filteredEyePitch;
+            _lastEyePitch = _eyePitch;
+            _eyePitch = _filteredEyePitch;
+            float yawVelocity = fabsf(_eyeYaw - _lastEyeYaw) / _averageMessageTime;
+            float yawVelocityFilter = glm::clamp(yawVelocity * EYE_VELOCITY_FILTER_STRENGTH, 0.0f, 1.0f);
+            _filteredEyeYaw = yawVelocityFilter * _eyeYaw + (1.0f - yawVelocityFilter) * _filteredEyeYaw;
+            _lastEyeYaw = _eyeYaw;
+            _eyeYaw = _filteredEyeYaw;
+        }
+
         // Velocity filter EyeBlink values
         const float DDE_EYEBLINK_SCALE = 3.0f;
         float eyeBlinks[] = { DDE_EYEBLINK_SCALE * _coefficients[_leftBlinkIndex],

interface/src/devices/DdeFaceTracker.h

@@ -31,6 +31,7 @@ class DdeFaceTracker : public FaceTracker, public Dependency {
 public:
     virtual void init();
     virtual void reset();
+    virtual void update(float deltaTime);
 
     virtual bool isActive() const;
     virtual bool isTracking() const;
@@ -93,6 +94,11 @@ private:
     int _leftEyeOpenIndex;
     int _rightEyeOpenIndex;
 
+    int _leftEyeDownIndex;
+    int _rightEyeDownIndex;
+    int _leftEyeInIndex;
+    int _rightEyeInIndex;
+
     int _browDownLeftIndex;
     int _browDownRightIndex;
     int _browUpCenterIndex;
@@ -115,6 +121,16 @@ private:
     float _lastBrowUp;
     float _filteredBrowUp;
 
+    float _eyePitch;  // Degrees, relative to screen
+    float _eyeYaw;
+    float _lastEyePitch;
+    float _lastEyeYaw;
+    float _filteredEyePitch;
+    float _filteredEyeYaw;
+    float _longTermAverageEyePitch = 0.0f;
+    float _longTermAverageEyeYaw = 0.0f;
+    bool _longTermAverageInitialized = false;
+
     enum EyeState {
         EYE_UNCONTROLLED,
         EYE_OPEN,

interface/src/devices/FaceTracker.cpp

@@ -20,6 +20,9 @@
 const int FPS_TIMER_DELAY = 2000;  // ms
 const int FPS_TIMER_DURATION = 2000;  // ms
 
+const float DEFAULT_EYE_DEFLECTION = 0.25f;
+Setting::Handle<float> FaceTracker::_eyeDeflection("faceshiftEyeDeflection", DEFAULT_EYE_DEFLECTION);
+
 void FaceTracker::init() {
     _isMuted = Menu::getInstance()->isOptionChecked(MenuOption::MuteFaceTracking);
     _isInitialized = true;  // FaceTracker can be used now
@@ -106,3 +109,7 @@ void FaceTracker::toggleMute() {
     _isMuted = !_isMuted;
     emit muteToggled();
 }
+
+void FaceTracker::setEyeDeflection(float eyeDeflection) {
+    _eyeDeflection.set(eyeDeflection);
+}

interface/src/devices/FaceTracker.h

@@ -18,6 +18,8 @@
 #include <glm/glm.hpp>
 #include <glm/gtc/quaternion.hpp>
 
+#include <SettingHandle.h>
+
 /// Base class for face trackers (Faceshift, DDE).
 class FaceTracker : public QObject {
     Q_OBJECT
@@ -47,6 +49,9 @@ public:
     void setIsMuted(bool isMuted) { _isMuted = isMuted; }
     void toggleMute();
 
+    static float getEyeDeflection() { return _eyeDeflection.get(); }
+    static void setEyeDeflection(float eyeDeflection);
+
 signals:
     void muteToggled();
 
@@ -77,6 +82,8 @@ private slots:
 private:
     bool _isCalculatingFPS = false;
     int _frameCount = 0;
+
+    static Setting::Handle<float> _eyeDeflection;
 };
 
 #endif // hifi_FaceTracker_h

interface/src/devices/Faceshift.cpp

@@ -28,10 +28,8 @@ using namespace std;
 
 const QString DEFAULT_FACESHIFT_HOSTNAME = "localhost";
 const quint16 FACESHIFT_PORT = 33433;
-const float DEFAULT_FACESHIFT_EYE_DEFLECTION = 0.25f;
 
 Faceshift::Faceshift() :
-    _eyeDeflection("faceshiftEyeDeflection", DEFAULT_FACESHIFT_EYE_DEFLECTION),
     _hostname("faceshiftHostname", DEFAULT_FACESHIFT_HOSTNAME)
 {
 #ifdef HAVE_FACESHIFT
@@ -306,10 +304,6 @@ void Faceshift::receive(const QByteArray& buffer) {
     FaceTracker::countFrame();
 }
 
-void Faceshift::setEyeDeflection(float faceshiftEyeDeflection) {
-    _eyeDeflection.set(faceshiftEyeDeflection);
-}
-
 void Faceshift::setHostname(const QString& hostname) {
     _hostname.set(hostname);
 }

interface/src/devices/Faceshift.h

@@ -68,9 +68,6 @@ public:
     float getMouthSmileLeft() const { return getBlendshapeCoefficient(_mouthSmileLeftIndex); }
     float getMouthSmileRight() const { return getBlendshapeCoefficient(_mouthSmileRightIndex); }
     
-    float getEyeDeflection() { return _eyeDeflection.get(); }
-    void setEyeDeflection(float faceshiftEyeDeflection);
-    
     QString getHostname() { return _hostname.get(); }
     void setHostname(const QString& hostname);
     
@@ -134,7 +131,6 @@ private:
     float _longTermAverageEyeYaw = 0.0f;
     bool _longTermAverageInitialized = false;
     
-    Setting::Handle<float> _eyeDeflection;
     Setting::Handle<QString> _hostname;
     
     // see http://support.faceshift.com/support/articles/35129-export-of-blendshapes

interface/src/ui/PreferencesDialog.cpp

@@ -142,10 +142,10 @@ void PreferencesDialog::loadPreferences() {
     ui.ddeEyeClosingThresholdSlider->setValue(dde->getEyeClosingThreshold() * 
                                               ui.ddeEyeClosingThresholdSlider->maximum());
 
-    auto faceshift = DependencyManager::get<Faceshift>();
+    ui.faceTrackerEyeDeflectionSider->setValue(FaceTracker::getEyeDeflection() *
-    ui.faceshiftEyeDeflectionSider->setValue(faceshift->getEyeDeflection() *
+                                               ui.faceTrackerEyeDeflectionSider->maximum());
-                                             ui.faceshiftEyeDeflectionSider->maximum());
     
+    auto faceshift = DependencyManager::get<Faceshift>();
     ui.faceshiftHostnameEdit->setText(faceshift->getHostname());
 
     auto audio = DependencyManager::get<AudioClient>();
@@ -233,10 +233,10 @@ void PreferencesDialog::savePreferences() {
     dde->setEyeClosingThreshold(ui.ddeEyeClosingThresholdSlider->value() / 
                                 (float)ui.ddeEyeClosingThresholdSlider->maximum());
 
-    auto faceshift = DependencyManager::get<Faceshift>();
+    FaceTracker::setEyeDeflection(ui.faceTrackerEyeDeflectionSider->value() /
-    faceshift->setEyeDeflection(ui.faceshiftEyeDeflectionSider->value() /
+                                (float)ui.faceTrackerEyeDeflectionSider->maximum());
-                                (float)ui.faceshiftEyeDeflectionSider->maximum());
     
+    auto faceshift = DependencyManager::get<Faceshift>();
     faceshift->setHostname(ui.faceshiftHostnameEdit->text());
     
     qApp->setMaxOctreePacketsPerSecond(ui.maxOctreePPSSpin->value());

interface/ui/preferencesDialog.ui

@@ -1468,13 +1468,13 @@
             </font>
            </property>
            <property name="text">
-            <string>Faceshift eye deflection</string>
+            <string>Face tracker eye deflection</string>
            </property>
            <property name="indent">
             <number>0</number>
            </property>
            <property name="buddy">
-            <cstring>faceshiftEyeDeflectionSider</cstring>
+            <cstring>faceTrackerEyeDeflectionSider</cstring>
            </property>
           </widget>
          </item>
@@ -1497,7 +1497,7 @@
           </spacer>
          </item>
          <item>
-          <widget class="QSlider" name="faceshiftEyeDeflectionSider">
+          <widget class="QSlider" name="faceTrackerEyeDeflectionSider">
            <property name="sizePolicy">
             <sizepolicy hsizetype="Fixed" vsizetype="Fixed">
              <horstretch>0</horstretch>