From ae8938cadcd51cbbe0090b399b270d70cdb87892 Mon Sep 17 00:00:00 2001 From: "Anthony J. Thibault" Date: Tue, 27 Oct 2015 18:30:35 -0700 Subject: [PATCH 1/4] Fix for local eye tracking in AnimGraph Also moved Rig::updateAnimations() now occurs after Rig::updateFromHeadParameters() and Rig::updateFromHandParameters(). This should remove a frame of lag for head and hand IK targets. Rig::updateFromEyeParameters() occurs after Rig::updateAnimations(). But now the eye JointStates are re-computed, this is the actual fix for the local eye tracking issue. --- interface/src/avatar/SkeletonModel.cpp | 43 +++++++++++++++++++------- libraries/animation/src/Rig.cpp | 35 +++------------------ libraries/animation/src/Rig.h | 19 +++++++----- 3 files changed, 47 insertions(+), 50 deletions(-) diff --git a/interface/src/avatar/SkeletonModel.cpp b/interface/src/avatar/SkeletonModel.cpp index 28c7941c52..c3716aa77a 100644 --- a/interface/src/avatar/SkeletonModel.cpp +++ b/interface/src/avatar/SkeletonModel.cpp @@ -114,15 +114,12 @@ void SkeletonModel::updateRig(float deltaTime, glm::mat4 parentTransform) { if (_owningAvatar->isMyAvatar()) { _rig->computeMotionAnimationState(deltaTime, _owningAvatar->getPosition(), _owningAvatar->getVelocity(), _owningAvatar->getOrientation()); } - Model::updateRig(deltaTime, parentTransform); Head* head = _owningAvatar->getHead(); if (_owningAvatar->isMyAvatar()) { MyAvatar* myAvatar = static_cast(_owningAvatar); const FBXGeometry& geometry = _geometry->getFBXGeometry(); Rig::HeadParameters headParams; - headParams.modelRotation = getRotation(); - headParams.modelTranslation = getTranslation(); headParams.enableLean = qApp->getAvatarUpdater()->isHMDMode(); headParams.leanSideways = head->getFinalLeanSideways(); headParams.leanForward = head->getFinalLeanForward(); @@ -156,19 +153,13 @@ void SkeletonModel::updateRig(float deltaTime, glm::mat4 parentTransform) { headParams.worldHeadOrientation = head->getFinalOrientationInWorldFrame(); } - headParams.eyeLookAt = head->getLookAtPosition(); - headParams.eyeSaccade = head->getSaccade(); headParams.leanJointIndex = geometry.leanJointIndex; headParams.neckJointIndex = geometry.neckJointIndex; - headParams.leftEyeJointIndex = geometry.leftEyeJointIndex; - headParams.rightEyeJointIndex = geometry.rightEyeJointIndex; - headParams.isTalking = head->getTimeWithoutTalking() <= 1.5f; _rig->updateFromHeadParameters(headParams, deltaTime); Rig::HandParameters handParams; - const PalmData* leftPalm = getPalmWithIndex(myAvatar->getHand(), LEFT_HAND_INDEX); if (leftPalm && leftPalm->isActive()) { handParams.isLeftEnabled = true; @@ -191,7 +182,28 @@ void SkeletonModel::updateRig(float deltaTime, glm::mat4 parentTransform) { _rig->updateFromHandParameters(handParams, deltaTime); + // evaluate AnimGraph animation and update jointStates. + Model::updateRig(deltaTime, parentTransform); + + Rig::EyeParameters eyeParams; + eyeParams.worldHeadOrientation = headParams.worldHeadOrientation; + eyeParams.eyeLookAt = head->getLookAtPosition(); + eyeParams.eyeSaccade = head->getSaccade(); + eyeParams.modelRotation = getRotation(); + eyeParams.modelTranslation = getTranslation(); + eyeParams.leftEyeJointIndex = geometry.leftEyeJointIndex; + eyeParams.rightEyeJointIndex = geometry.rightEyeJointIndex; + + _rig->updateFromEyeParameters(eyeParams, deltaTime); + + // rebuild the jointState transform for the eyes only + _rig->updateJointState(eyeParams.leftEyeJointIndex, parentTransform); + _rig->updateJointState(eyeParams.rightEyeJointIndex, parentTransform); + } else { + + Model::updateRig(deltaTime, parentTransform); + // This is a little more work than we really want. // // Other avatars joint, including their eyes, should already be set just like any other joints @@ -208,9 +220,16 @@ void SkeletonModel::updateRig(float deltaTime, glm::mat4 parentTransform) { head->setBasePitch(glm::degrees(-eulers.x)); head->setBaseYaw(glm::degrees(eulers.y)); head->setBaseRoll(glm::degrees(-eulers.z)); - _rig->updateEyeJoints(geometry.leftEyeJointIndex, geometry.rightEyeJointIndex, - getTranslation(), getRotation(), - head->getFinalOrientationInWorldFrame(), head->getCorrectedLookAtPosition()); + + Rig::EyeParameters eyeParams; + eyeParams.worldHeadOrientation = head->getFinalOrientationInWorldFrame(); + eyeParams.eyeLookAt = head->getCorrectedLookAtPosition(); + eyeParams.eyeSaccade = glm::vec3(0); + eyeParams.modelRotation = getRotation(); + eyeParams.modelTranslation = getTranslation(); + eyeParams.leftEyeJointIndex = geometry.leftEyeJointIndex; + eyeParams.rightEyeJointIndex = geometry.rightEyeJointIndex; + _rig->updateFromEyeParameters(eyeParams, deltaTime); } } diff --git a/libraries/animation/src/Rig.cpp b/libraries/animation/src/Rig.cpp index 6fd6f5cbf9..023fcb9800 100644 --- a/libraries/animation/src/Rig.cpp +++ b/libraries/animation/src/Rig.cpp @@ -22,34 +22,6 @@ #include "AnimSkeleton.h" #include "IKTarget.h" - -void Rig::HeadParameters::dump() const { - qCDebug(animation, "HeadParameters ="); - qCDebug(animation, " leanSideways = %0.5f", (double)leanSideways); - qCDebug(animation, " leanForward = %0.5f", (double)leanForward); - qCDebug(animation, " torsoTwist = %0.5f", (double)torsoTwist); - glm::vec3 axis = glm::axis(localHeadOrientation); - float theta = glm::angle(localHeadOrientation); - qCDebug(animation, " localHeadOrientation axis = (%.5f, %.5f, %.5f), theta = %0.5f", (double)axis.x, (double)axis.y, (double)axis.z, (double)theta); - axis = glm::axis(worldHeadOrientation); - theta = glm::angle(worldHeadOrientation); - qCDebug(animation, " localHead pitch = %.5f, yaw = %.5f, roll = %.5f", (double)localHeadPitch, (double)localHeadYaw, (double)localHeadRoll); - qCDebug(animation, " localHeadPosition = (%.5f, %.5f, %.5f)", (double)localHeadPosition.x, (double)localHeadPosition.y, (double)localHeadPosition.z); - qCDebug(animation, " isInHMD = %s", isInHMD ? "true" : "false"); - qCDebug(animation, " worldHeadOrientation axis = (%.5f, %.5f, %.5f), theta = %0.5f", (double)axis.x, (double)axis.y, (double)axis.z, (double)theta); - axis = glm::axis(modelRotation); - theta = glm::angle(modelRotation); - qCDebug(animation, " modelRotation axis = (%.5f, %.5f, %.5f), theta = %0.5f", (double)axis.x, (double)axis.y, (double)axis.z, (double)theta); - qCDebug(animation, " modelTranslation = (%.5f, %.5f, %.5f)", (double)modelTranslation.x, (double)modelTranslation.y, (double)modelTranslation.z); - qCDebug(animation, " eyeLookAt = (%.5f, %.5f, %.5f)", (double)eyeLookAt.x, (double)eyeLookAt.y, (double)eyeLookAt.z); - qCDebug(animation, " eyeSaccade = (%.5f, %.5f, %.5f)", (double)eyeSaccade.x, (double)eyeSaccade.y, (double)eyeSaccade.z); - qCDebug(animation, " leanJointIndex = %.d", leanJointIndex); - qCDebug(animation, " neckJointIndex = %.d", neckJointIndex); - qCDebug(animation, " leftEyeJointIndex = %.d", leftEyeJointIndex); - qCDebug(animation, " rightEyeJointIndex = %.d", rightEyeJointIndex); - qCDebug(animation, " isTalking = %s", isTalking ? "true" : "false"); -} - void insertSorted(QList& handles, const AnimationHandlePointer& handle) { for (QList::iterator it = handles.begin(); it != handles.end(); it++) { if (handle->getPriority() > (*it)->getPriority()) { @@ -981,8 +953,6 @@ void Rig::updateFromHeadParameters(const HeadParameters& params, float dt) { _animVars.unset("lean"); } updateNeckJoint(params.neckJointIndex, params); - updateEyeJoints(params.leftEyeJointIndex, params.rightEyeJointIndex, params.modelTranslation, params.modelRotation, - params.worldHeadOrientation, params.eyeLookAt, params.eyeSaccade); if (_enableAnimGraph) { _animVars.set("isTalking", params.isTalking); @@ -990,6 +960,11 @@ void Rig::updateFromHeadParameters(const HeadParameters& params, float dt) { } } +void Rig::updateFromEyeParameters(const EyeParameters& params, float dt) { + updateEyeJoints(params.leftEyeJointIndex, params.rightEyeJointIndex, params.modelTranslation, params.modelRotation, + params.worldHeadOrientation, params.eyeLookAt, params.eyeSaccade); +} + static const glm::vec3 X_AXIS(1.0f, 0.0f, 0.0f); static const glm::vec3 Y_AXIS(0.0f, 1.0f, 0.0f); static const glm::vec3 Z_AXIS(0.0f, 0.0f, 1.0f); diff --git a/libraries/animation/src/Rig.h b/libraries/animation/src/Rig.h index c9c42cbbb6..7ae0721625 100644 --- a/libraries/animation/src/Rig.h +++ b/libraries/animation/src/Rig.h @@ -57,24 +57,26 @@ public: float leanForward = 0.0f; // degrees float torsoTwist = 0.0f; // degrees bool enableLean = false; - glm::quat modelRotation = glm::quat(); + glm::quat worldHeadOrientation = glm::quat(); glm::quat localHeadOrientation = glm::quat(); float localHeadPitch = 0.0f; // degrees float localHeadYaw = 0.0f; // degrees float localHeadRoll = 0.0f; // degrees glm::vec3 localHeadPosition = glm::vec3(); bool isInHMD = false; + int leanJointIndex = -1; + int neckJointIndex = -1; + bool isTalking = false; + }; + + struct EyeParameters { glm::quat worldHeadOrientation = glm::quat(); glm::vec3 eyeLookAt = glm::vec3(); // world space glm::vec3 eyeSaccade = glm::vec3(); // world space glm::vec3 modelTranslation = glm::vec3(); - int leanJointIndex = -1; - int neckJointIndex = -1; + glm::quat modelRotation = glm::quat(); int leftEyeJointIndex = -1; int rightEyeJointIndex = -1; - bool isTalking = false; - - void dump() const; }; struct HandParameters { @@ -185,8 +187,7 @@ public: bool getEnableAnimGraph() const { return _enableAnimGraph; } void updateFromHeadParameters(const HeadParameters& params, float dt); - void updateEyeJoints(int leftEyeIndex, int rightEyeIndex, const glm::vec3& modelTranslation, const glm::quat& modelRotation, - const glm::quat& worldHeadOrientation, const glm::vec3& lookAtSpot, const glm::vec3& saccade = glm::vec3(0.0f)); + void updateFromEyeParameters(const EyeParameters& params, float dt); void updateFromHandParameters(const HandParameters& params, float dt); @@ -207,6 +208,8 @@ public: void updateLeanJoint(int index, float leanSideways, float leanForward, float torsoTwist); void updateNeckJoint(int index, const HeadParameters& params); void updateEyeJoint(int index, const glm::vec3& modelTranslation, const glm::quat& modelRotation, const glm::quat& worldHeadOrientation, const glm::vec3& lookAt, const glm::vec3& saccade); + void updateEyeJoints(int leftEyeIndex, int rightEyeIndex, const glm::vec3& modelTranslation, const glm::quat& modelRotation, + const glm::quat& worldHeadOrientation, const glm::vec3& lookAtSpot, const glm::vec3& saccade); QVector _jointStates; int _rootJointIndex = -1; From b3734b5ddb3e44c6d5185abb86a3a0eb1992a456 Mon Sep 17 00:00:00 2001 From: "Anthony J. Thibault" Date: Wed, 28 Oct 2015 10:46:09 -0700 Subject: [PATCH 2/4] Rig: removed redundant method updateEyeJoints() Instead we call updateEyeJoint() twice, once for each eye. --- libraries/animation/src/Rig.cpp | 12 ++++-------- libraries/animation/src/Rig.h | 2 -- 2 files changed, 4 insertions(+), 10 deletions(-) diff --git a/libraries/animation/src/Rig.cpp b/libraries/animation/src/Rig.cpp index 023fcb9800..8c7fa22c22 100644 --- a/libraries/animation/src/Rig.cpp +++ b/libraries/animation/src/Rig.cpp @@ -961,8 +961,10 @@ void Rig::updateFromHeadParameters(const HeadParameters& params, float dt) { } void Rig::updateFromEyeParameters(const EyeParameters& params, float dt) { - updateEyeJoints(params.leftEyeJointIndex, params.rightEyeJointIndex, params.modelTranslation, params.modelRotation, - params.worldHeadOrientation, params.eyeLookAt, params.eyeSaccade); + updateEyeJoint(params.leftEyeJointIndex, params.modelTranslation, params.modelRotation, + params.worldHeadOrientation, params.eyeLookAt, params.eyeSaccade); + updateEyeJoint(params.rightEyeJointIndex, params.modelTranslation, params.modelRotation, + params.worldHeadOrientation, params.eyeLookAt, params.eyeSaccade); } static const glm::vec3 X_AXIS(1.0f, 0.0f, 0.0f); @@ -1120,12 +1122,6 @@ void Rig::updateNeckJoint(int index, const HeadParameters& params) { } } -void Rig::updateEyeJoints(int leftEyeIndex, int rightEyeIndex, const glm::vec3& modelTranslation, const glm::quat& modelRotation, - const glm::quat& worldHeadOrientation, const glm::vec3& lookAtSpot, const glm::vec3& saccade) { - updateEyeJoint(leftEyeIndex, modelTranslation, modelRotation, worldHeadOrientation, lookAtSpot, saccade); - updateEyeJoint(rightEyeIndex, modelTranslation, modelRotation, worldHeadOrientation, lookAtSpot, saccade); -} - void Rig::updateEyeJoint(int index, const glm::vec3& modelTranslation, const glm::quat& modelRotation, const glm::quat& worldHeadOrientation, const glm::vec3& lookAtSpot, const glm::vec3& saccade) { if (index >= 0 && _jointStates[index].getParentIndex() >= 0) { auto& state = _jointStates[index]; diff --git a/libraries/animation/src/Rig.h b/libraries/animation/src/Rig.h index 7ae0721625..25e823d6bc 100644 --- a/libraries/animation/src/Rig.h +++ b/libraries/animation/src/Rig.h @@ -208,8 +208,6 @@ public: void updateLeanJoint(int index, float leanSideways, float leanForward, float torsoTwist); void updateNeckJoint(int index, const HeadParameters& params); void updateEyeJoint(int index, const glm::vec3& modelTranslation, const glm::quat& modelRotation, const glm::quat& worldHeadOrientation, const glm::vec3& lookAt, const glm::vec3& saccade); - void updateEyeJoints(int leftEyeIndex, int rightEyeIndex, const glm::vec3& modelTranslation, const glm::quat& modelRotation, - const glm::quat& worldHeadOrientation, const glm::vec3& lookAtSpot, const glm::vec3& saccade); QVector _jointStates; int _rootJointIndex = -1; From 6ae5c540fdeb914ee6538de64f62056ba202ca74 Mon Sep 17 00:00:00 2001 From: "Anthony J. Thibault" Date: Wed, 28 Oct 2015 11:09:24 -0700 Subject: [PATCH 3/4] SkeletonModel: coding standard fix Replaced glm::vec3(0) with glm::vec3(). --- interface/src/avatar/SkeletonModel.cpp | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/interface/src/avatar/SkeletonModel.cpp b/interface/src/avatar/SkeletonModel.cpp index c3716aa77a..04be8a1f63 100644 --- a/interface/src/avatar/SkeletonModel.cpp +++ b/interface/src/avatar/SkeletonModel.cpp @@ -224,7 +224,7 @@ void SkeletonModel::updateRig(float deltaTime, glm::mat4 parentTransform) { Rig::EyeParameters eyeParams; eyeParams.worldHeadOrientation = head->getFinalOrientationInWorldFrame(); eyeParams.eyeLookAt = head->getCorrectedLookAtPosition(); - eyeParams.eyeSaccade = glm::vec3(0); + eyeParams.eyeSaccade = glm::vec3(); eyeParams.modelRotation = getRotation(); eyeParams.modelTranslation = getTranslation(); eyeParams.leftEyeJointIndex = geometry.leftEyeJointIndex; From 5e73af927237cc285337338e8efd4f5e354ffe52 Mon Sep 17 00:00:00 2001 From: "Anthony J. Thibault" Date: Wed, 28 Oct 2015 18:10:41 -0700 Subject: [PATCH 4/4] Rig: Removed dt from updateFromEyeParameters() --- interface/src/avatar/SkeletonModel.cpp | 4 ++-- libraries/animation/src/Rig.cpp | 2 +- libraries/animation/src/Rig.h | 3 +-- 3 files changed, 4 insertions(+), 5 deletions(-) diff --git a/interface/src/avatar/SkeletonModel.cpp b/interface/src/avatar/SkeletonModel.cpp index 04be8a1f63..71422f6780 100644 --- a/interface/src/avatar/SkeletonModel.cpp +++ b/interface/src/avatar/SkeletonModel.cpp @@ -194,7 +194,7 @@ void SkeletonModel::updateRig(float deltaTime, glm::mat4 parentTransform) { eyeParams.leftEyeJointIndex = geometry.leftEyeJointIndex; eyeParams.rightEyeJointIndex = geometry.rightEyeJointIndex; - _rig->updateFromEyeParameters(eyeParams, deltaTime); + _rig->updateFromEyeParameters(eyeParams); // rebuild the jointState transform for the eyes only _rig->updateJointState(eyeParams.leftEyeJointIndex, parentTransform); @@ -229,7 +229,7 @@ void SkeletonModel::updateRig(float deltaTime, glm::mat4 parentTransform) { eyeParams.modelTranslation = getTranslation(); eyeParams.leftEyeJointIndex = geometry.leftEyeJointIndex; eyeParams.rightEyeJointIndex = geometry.rightEyeJointIndex; - _rig->updateFromEyeParameters(eyeParams, deltaTime); + _rig->updateFromEyeParameters(eyeParams); } } diff --git a/libraries/animation/src/Rig.cpp b/libraries/animation/src/Rig.cpp index 8c7fa22c22..3fe4c2e83e 100644 --- a/libraries/animation/src/Rig.cpp +++ b/libraries/animation/src/Rig.cpp @@ -960,7 +960,7 @@ void Rig::updateFromHeadParameters(const HeadParameters& params, float dt) { } } -void Rig::updateFromEyeParameters(const EyeParameters& params, float dt) { +void Rig::updateFromEyeParameters(const EyeParameters& params) { updateEyeJoint(params.leftEyeJointIndex, params.modelTranslation, params.modelRotation, params.worldHeadOrientation, params.eyeLookAt, params.eyeSaccade); updateEyeJoint(params.rightEyeJointIndex, params.modelTranslation, params.modelRotation, diff --git a/libraries/animation/src/Rig.h b/libraries/animation/src/Rig.h index 25e823d6bc..6e0a88d768 100644 --- a/libraries/animation/src/Rig.h +++ b/libraries/animation/src/Rig.h @@ -187,8 +187,7 @@ public: bool getEnableAnimGraph() const { return _enableAnimGraph; } void updateFromHeadParameters(const HeadParameters& params, float dt); - void updateFromEyeParameters(const EyeParameters& params, float dt); - + void updateFromEyeParameters(const EyeParameters& params); void updateFromHandParameters(const HandParameters& params, float dt); virtual void setHandPosition(int jointIndex, const glm::vec3& position, const glm::quat& rotation,