From 1ae920368ddcd5e717b6f1f5472d6bebd7c08993 Mon Sep 17 00:00:00 2001 From: David Rowe Date: Wed, 29 Jul 2015 08:38:20 -0700 Subject: [PATCH] Calculate and look at the point the eye tracker user is looking at --- interface/src/Application.cpp | 15 ++++++--- interface/src/devices/EyeTracker.cpp | 49 ++++++++++++++++++++++++++++ interface/src/devices/EyeTracker.h | 9 ++++- 3 files changed, 67 insertions(+), 6 deletions(-) diff --git a/interface/src/Application.cpp b/interface/src/Application.cpp index fcbe64cb06..5861adc704 100644 --- a/interface/src/Application.cpp +++ b/interface/src/Application.cpp @@ -2282,7 +2282,8 @@ void Application::updateMyAvatarLookAtPosition() { PerformanceWarning warn(showWarnings, "Application::updateMyAvatarLookAtPosition()"); _myAvatar->updateLookAtTargetAvatar(); - FaceTracker* tracker = getActiveFaceTracker(); + FaceTracker* faceTracker = getActiveFaceTracker(); + auto eyeTracker = DependencyManager::get(); bool isLookingAtSomeone = false; glm::vec3 lookAtSpot; @@ -2294,6 +2295,10 @@ void Application::updateMyAvatarLookAtPosition() { } else { lookAtSpot = _myCamera.getPosition() + OculusManager::getMidEyePosition(); } + } else if (eyeTracker->isTracking()) { + // Look at the point that the user is looking at. + lookAtSpot = _myAvatar->getHead()->getEyePosition() + + (_myAvatar->getHead()->getFinalOrientationInWorldFrame() * eyeTracker->getLookAtPosition()); } else { AvatarSharedPointer lookingAt = _myAvatar->getLookAtTargetAvatar().lock(); if (lookingAt && _myAvatar != lookingAt.get()) { @@ -2331,12 +2336,12 @@ void Application::updateMyAvatarLookAtPosition() { } // Deflect the eyes a bit to match the detected gaze from the face tracker if active. - if (tracker && !tracker->isMuted()) { - float eyePitch = tracker->getEstimatedEyePitch(); - float eyeYaw = tracker->getEstimatedEyeYaw(); + if (faceTracker && !faceTracker->isMuted()) { + float eyePitch = faceTracker->getEstimatedEyePitch(); + float eyeYaw = faceTracker->getEstimatedEyeYaw(); const float GAZE_DEFLECTION_REDUCTION_DURING_EYE_CONTACT = 0.1f; glm::vec3 origin = _myAvatar->getHead()->getEyePosition(); - float deflection = tracker->getEyeDeflection(); + float deflection = faceTracker->getEyeDeflection(); if (isLookingAtSomeone) { deflection *= GAZE_DEFLECTION_REDUCTION_DURING_EYE_CONTACT; } diff --git a/interface/src/devices/EyeTracker.cpp b/interface/src/devices/EyeTracker.cpp index 04e6364e33..f13dd0f61a 100644 --- a/interface/src/devices/EyeTracker.cpp +++ b/interface/src/devices/EyeTracker.cpp @@ -14,6 +14,7 @@ #include #include "InterfaceLogging.h" +#include "OctreeConstants.h" #ifdef HAVE_IVIEWHMD static void CALLBACK eyeTrackerCallback(smi_CallbackDataStruct* data) { @@ -40,7 +41,55 @@ void EyeTracker::processData(smi_CallbackDataStruct* data) { #ifdef HAVE_IVIEWHMD if (data->type == SMI_SIMPLE_GAZE_SAMPLE) { + // Calculate the intersections of the left and right eye look-at vectors with a vertical plane along the monocular + // gaze direction. Average these positions to give the look-at point. + // If the eyes are parallel or diverged, gaze at a distant look-at point calculated the same as for non eye tracking. + // Line-plane intersection: https://en.wikipedia.org/wiki/Line%E2%80%93plane_intersection + smi_SampleHMDStruct* sample = (smi_SampleHMDStruct*)data->result; + // The iViewHMD coordinate system has x and z axes reversed compared to Interface, i.e., wearing the HMD: + // - x is left + // - y is up + // - z is forwards + + // Plane + smi_Vec3d point = sample->gazeBasePoint; // mm + smi_Vec3d direction = sample->gazeDirection; + glm::vec3 planePoint = glm::vec3(-point.x, point.y, -point.z) / 1000.0f; + glm::vec3 planeNormal = glm::vec3(-direction.z, 0.0f, direction.x); + glm::vec3 monocularDirection = glm::vec3(-direction.x, direction.y, -direction.z); + + // Left eye + point = sample->left.gazeBasePoint; // mm + direction = sample->left.gazeDirection; + glm::vec3 leftLinePoint = glm::vec3(-point.x, point.y, -point.z) / 1000.0f; + glm::vec3 leftLineDirection = glm::vec3(-direction.x, direction.y, -direction.z); + + // Right eye + point = sample->right.gazeBasePoint; // mm + direction = sample->right.gazeDirection; + glm::vec3 rightLinePoint = glm::vec3(-point.x, point.y, -point.z) / 1000.0f; + glm::vec3 rightLineDirection = glm::vec3(-direction.x, direction.y, -direction.z); + + // Plane - line dot products + float leftLinePlaneDotProduct = glm::dot(leftLineDirection, planeNormal); + float rightLinePlaneDotProduct = glm::dot(rightLineDirection, planeNormal); + + // Gaze into distance if eyes are parallel or diverged; otherwise the look-at is the average of look-at points + if (abs(leftLinePlaneDotProduct) <= FLT_EPSILON || abs(rightLinePlaneDotProduct) <= FLT_EPSILON) { + _lookAtPosition = monocularDirection * (float)TREE_SCALE; + } else { + float leftDistance = glm::dot(planePoint - leftLinePoint, planeNormal) / leftLinePlaneDotProduct; + float rightDistance = glm::dot(planePoint - rightLinePoint, planeNormal) / rightLinePlaneDotProduct; + if (leftDistance <= 0.0f || rightDistance <= 0.0f + || leftDistance > (float)TREE_SCALE || rightDistance > (float)TREE_SCALE) { + _lookAtPosition = monocularDirection * (float)TREE_SCALE; + } else { + glm::vec3 leftIntersectionPoint = leftLinePoint + leftDistance * leftLineDirection; + glm::vec3 rightIntersectionPoint = rightLinePoint + rightDistance * rightLineDirection; + _lookAtPosition = (leftIntersectionPoint + rightIntersectionPoint) / 2.0f; + } + } } #endif } diff --git a/interface/src/devices/EyeTracker.h b/interface/src/devices/EyeTracker.h index 31c6648712..cb970d5be5 100644 --- a/interface/src/devices/EyeTracker.h +++ b/interface/src/devices/EyeTracker.h @@ -14,6 +14,8 @@ #include +#include + #include #include @@ -24,6 +26,10 @@ class EyeTracker : public QObject, public Dependency { public: ~EyeTracker(); + + bool isTracking() { return _isEnabled; } + + glm::vec3 getLookAtPosition() { return _lookAtPosition; } // From mid eye point in head frame. void processData(smi_CallbackDataStruct* data); @@ -36,8 +42,9 @@ private: QString smiReturnValueToString(int value); bool _isInitialized = false; - bool _isStreaming = false; bool _isEnabled = false; + + glm::vec3 _lookAtPosition; }; #endif // hifi_EyeTracker_h