Calculate and look at the point the eye tracker user is looking at

interface/src/Application.cpp

@@ -2282,7 +2282,8 @@ void Application::updateMyAvatarLookAtPosition() {
     PerformanceWarning warn(showWarnings, "Application::updateMyAvatarLookAtPosition()");
 
     _myAvatar->updateLookAtTargetAvatar();
-    FaceTracker* tracker = getActiveFaceTracker();
+    FaceTracker* faceTracker = getActiveFaceTracker();
+    auto eyeTracker = DependencyManager::get<EyeTracker>();
 
     bool isLookingAtSomeone = false;
     glm::vec3 lookAtSpot;
@@ -2294,6 +2295,10 @@ void Application::updateMyAvatarLookAtPosition() {
         } else {
             lookAtSpot = _myCamera.getPosition() + OculusManager::getMidEyePosition();
         }
+    } else if (eyeTracker->isTracking()) {
+        //  Look at the point that the user is looking at.
+        lookAtSpot = _myAvatar->getHead()->getEyePosition() +
+            (_myAvatar->getHead()->getFinalOrientationInWorldFrame() * eyeTracker->getLookAtPosition());
     } else {
         AvatarSharedPointer lookingAt = _myAvatar->getLookAtTargetAvatar().lock();
         if (lookingAt && _myAvatar != lookingAt.get()) {
@@ -2331,12 +2336,12 @@ void Application::updateMyAvatarLookAtPosition() {
         }
 
         // Deflect the eyes a bit to match the detected gaze from the face tracker if active.
-        if (tracker && !tracker->isMuted()) {
-            float eyePitch = tracker->getEstimatedEyePitch();
-            float eyeYaw = tracker->getEstimatedEyeYaw();
+        if (faceTracker && !faceTracker->isMuted()) {
+            float eyePitch = faceTracker->getEstimatedEyePitch();
+            float eyeYaw = faceTracker->getEstimatedEyeYaw();
             const float GAZE_DEFLECTION_REDUCTION_DURING_EYE_CONTACT = 0.1f;
             glm::vec3 origin = _myAvatar->getHead()->getEyePosition();
-            float deflection = tracker->getEyeDeflection();
+            float deflection = faceTracker->getEyeDeflection();
             if (isLookingAtSomeone) {
                 deflection *= GAZE_DEFLECTION_REDUCTION_DURING_EYE_CONTACT;
             }

interface/src/devices/EyeTracker.cpp

@@ -14,6 +14,7 @@
 #include <QMessageBox>
 
 #include "InterfaceLogging.h"
+#include "OctreeConstants.h"
 
 #ifdef HAVE_IVIEWHMD
 static void CALLBACK eyeTrackerCallback(smi_CallbackDataStruct* data) {
@@ -40,7 +41,55 @@ void EyeTracker::processData(smi_CallbackDataStruct* data) {
 
 #ifdef HAVE_IVIEWHMD
     if (data->type == SMI_SIMPLE_GAZE_SAMPLE) {
+        // Calculate the intersections of the left and right eye look-at vectors with a vertical plane along the monocular
+        // gaze direction. Average these positions to give the look-at point.
+        // If the eyes are parallel or diverged, gaze at a distant look-at point calculated the same as for non eye tracking.
+        // Line-plane intersection: https://en.wikipedia.org/wiki/Line%E2%80%93plane_intersection
+
         smi_SampleHMDStruct* sample = (smi_SampleHMDStruct*)data->result;
+        // The iViewHMD coordinate system has x and z axes reversed compared to Interface, i.e., wearing the HMD:
+        // - x is left
+        // - y is up
+        // - z is forwards
+
+        // Plane
+        smi_Vec3d point = sample->gazeBasePoint;  // mm
+        smi_Vec3d direction = sample->gazeDirection;
+        glm::vec3 planePoint = glm::vec3(-point.x, point.y, -point.z) / 1000.0f;
+        glm::vec3 planeNormal = glm::vec3(-direction.z, 0.0f, direction.x);
+        glm::vec3 monocularDirection = glm::vec3(-direction.x, direction.y, -direction.z);
+
+        // Left eye
+        point = sample->left.gazeBasePoint;  // mm
+        direction = sample->left.gazeDirection;
+        glm::vec3 leftLinePoint = glm::vec3(-point.x, point.y, -point.z) / 1000.0f;
+        glm::vec3 leftLineDirection = glm::vec3(-direction.x, direction.y, -direction.z);
+
+        // Right eye
+        point = sample->right.gazeBasePoint;  // mm
+        direction = sample->right.gazeDirection;
+        glm::vec3 rightLinePoint = glm::vec3(-point.x, point.y, -point.z) / 1000.0f;
+        glm::vec3 rightLineDirection = glm::vec3(-direction.x, direction.y, -direction.z);
+
+        // Plane - line dot products
+        float leftLinePlaneDotProduct = glm::dot(leftLineDirection, planeNormal);
+        float rightLinePlaneDotProduct = glm::dot(rightLineDirection, planeNormal);
+
+        // Gaze into distance if eyes are parallel or diverged; otherwise the look-at is the average of look-at points
+        if (abs(leftLinePlaneDotProduct) <= FLT_EPSILON || abs(rightLinePlaneDotProduct) <= FLT_EPSILON) {
+            _lookAtPosition = monocularDirection * (float)TREE_SCALE;
+        } else {
+            float leftDistance = glm::dot(planePoint - leftLinePoint, planeNormal) / leftLinePlaneDotProduct;
+            float rightDistance = glm::dot(planePoint - rightLinePoint, planeNormal) / rightLinePlaneDotProduct;
+            if (leftDistance <= 0.0f || rightDistance <= 0.0f 
+                || leftDistance > (float)TREE_SCALE || rightDistance > (float)TREE_SCALE) {
+                _lookAtPosition = monocularDirection * (float)TREE_SCALE;
+            } else {
+                glm::vec3 leftIntersectionPoint = leftLinePoint + leftDistance * leftLineDirection;
+                glm::vec3 rightIntersectionPoint = rightLinePoint + rightDistance * rightLineDirection;
+                _lookAtPosition = (leftIntersectionPoint + rightIntersectionPoint) / 2.0f;
+            }
+        }
     }
 #endif
 }

interface/src/devices/EyeTracker.h

@@ -14,6 +14,8 @@
 
 #include <QObject>
 
+#include <glm/glm.hpp>
+
 #include <DependencyManager.h>
 #include <iViewHMDAPI.h>
 
@@ -24,6 +26,10 @@ class EyeTracker : public QObject, public Dependency {
 
 public:
     ~EyeTracker();
+
+    bool isTracking() { return _isEnabled; }
+
+    glm::vec3 getLookAtPosition() { return _lookAtPosition; }  // From mid eye point in head frame.
     
     void processData(smi_CallbackDataStruct* data);
 
@@ -36,8 +42,9 @@ private:
     QString smiReturnValueToString(int value);
     
     bool _isInitialized = false;
-    bool _isStreaming = false;
     bool _isEnabled = false;
+
+    glm::vec3 _lookAtPosition;
 };
 
 #endif // hifi_EyeTracker_h