From 9d6ce441310f0b9a6b60da39b07330537c10d3a6 Mon Sep 17 00:00:00 2001 From: "Anthony J. Thibault" Date: Wed, 20 Nov 2019 15:30:16 -0800 Subject: [PATCH] Head is now computed in proper sensor space Eye tracking now works. --- interface/resources/controllers/osc.json | 105 ++++++++++-------- interface/src/avatar/MyHead.cpp | 8 +- .../src/avatars-renderer/Head.cpp | 3 + plugins/hifiOsc/src/OscPlugin.cpp | 19 ++-- 4 files changed, 76 insertions(+), 59 deletions(-) diff --git a/interface/resources/controllers/osc.json b/interface/resources/controllers/osc.json index 5ed3640df5..e5c639f8b7 100644 --- a/interface/resources/controllers/osc.json +++ b/interface/resources/controllers/osc.json @@ -4,57 +4,68 @@ { "from": "OSC.Head", "to" : "Standard.Head" }, { "from": "OSC.LeftEye", "to" : "Standard.LeftEye" }, { "from": "OSC.RightEye", "to" : "Standard.RightEye" }, - { "from": "OSC.BrowInnerUp", "to": "Standard.BrowInnerUp" }, - { "from": "OSC.BrowDownLeft", "to": "Standard.BrowDownLeft" }, - { "from": "OSC.BrowDownRight", "to": "Standard.BrowDownRight" }, - { "from": "OSC.BrowOuterUpLeft", "to": "Standard.BrowOuterUpLeft" }, - { "from": "OSC.BrowOuterUpRight", "to": "Standard.BrowOuterUpRight" }, - { "from": "OSC.EyeLookUpLeft", "to": "Standard.EyeLookUpLeft" }, - { "from": "OSC.EyeLookUpRight", "to": "Standard.EyeLookUpRight" }, - { "from": "OSC.EyeLookDownLeft", "to": "Standard.EyeLookDownLeft" }, - { "from": "OSC.EyeLookDownRight", "to": "Standard.EyeLookDownRight" }, - { "from": "OSC.EyeLookInLeft", "to": "Standard.EyeLookInLeft" }, - { "from": "OSC.EyeLookInRight", "to": "Standard.EyeLookInRight" }, - { "from": "OSC.EyeLookOutLeft", "to": "Standard.EyeLookOutLeft" }, - { "from": "OSC.EyeLookOutRight", "to": "Standard.EyeLookOutRight" }, - { "from": "OSC.EyeBlinkLeft", "to": "Standard.EyeBlinkLeft" }, - { "from": "OSC.EyeBlinkRight", "to": "Standard.EyeBlinkRight" }, - { "from": "OSC.EyeSquintLeft", "to": "Standard.EyeSquintLeft" }, - { "from": "OSC.EyeSquintRight", "to": "Standard.EyeSquintRight" }, - { "from": "OSC.EyeWideLeft", "to": "Standard.EyeWideLeft" }, - { "from": "OSC.EyeWideRight", "to": "Standard.EyeWideRight" }, - { "from": "OSC.CheekPuff", "to": "Standard.CheekPuff" }, - { "from": "OSC.CheekSquintLeft", "to": "Standard.CheekSquintLeft" }, - { "from": "OSC.CheekSquintRight", "to": "Standard.CheekSquintRight" }, - { "from": "OSC.NoseSneerLeft", "to": "Standard.NoseSneerLeft" }, - { "from": "OSC.NoseSneerRight", "to": "Standard.NoseSneerRight" }, - { "from": "OSC.JawOpen", "to": "Standard.JawOpen" }, - { "from": "OSC.JawForward", "to": "Standard.JawForward" }, + + { "from": "OSC.EyeBlink_L", "to": "Standard.EyeBlink_L" }, + { "from": "OSC.EyeBlink_R", "to": "Standard.EyeBlink_R" }, + { "from": "OSC.EyeSquint_L", "to": "Standard.EyeSquint_L" }, + { "from": "OSC.EyeSquint_R", "to": "Standard.EyeSquint_R" }, + { "from": "OSC.EyeDown_L", "to": "Standard.EyeDown_L" }, + { "from": "OSC.EyeDown_R", "to": "Standard.EyeDown_R" }, + { "from": "OSC.EyeIn_L", "to": "Standard.EyeIn_L" }, + { "from": "OSC.EyeIn_R", "to": "Standard.EyeIn_R" }, + { "from": "OSC.EyeOpen_L", "to": "Standard.EyeOpen_L" }, + { "from": "OSC.EyeOpen_R", "to": "Standard.EyeOpen_R" }, + { "from": "OSC.EyeOut_L", "to": "Standard.EyeOut_L" }, + { "from": "OSC.EyeOut_R", "to": "Standard.EyeOut_R" }, + { "from": "OSC.EyeUp_L", "to": "Standard.EyeUp_L" }, + { "from": "OSC.EyeUp_R", "to": "Standard.EyeUp_R" }, + { "from": "OSC.BrowsD_L", "to": "Standard.BrowsD_L" }, + { "from": "OSC.BrowsD_R", "to": "Standard.BrowsD_R" }, + { "from": "OSC.BrowsU_C", "to": "Standard.BrowsU_C" }, + { "from": "OSC.BrowsU_L", "to": "Standard.BrowsU_L" }, + { "from": "OSC.BrowsU_R", "to": "Standard.BrowsU_R" }, + { "from": "OSC.JawFwd", "to": "Standard.JawFwd" }, { "from": "OSC.JawLeft", "to": "Standard.JawLeft" }, + { "from": "OSC.JawOpen", "to": "Standard.JawOpen" }, { "from": "OSC.JawRight", "to": "Standard.JawRight" }, - { "from": "OSC.MouthFunnel", "to": "Standard.MouthFunnel" }, - { "from": "OSC.MouthPucker", "to": "Standard.MouthPucker" }, { "from": "OSC.MouthLeft", "to": "Standard.MouthLeft" }, { "from": "OSC.MouthRight", "to": "Standard.MouthRight" }, - { "from": "OSC.MouthRollUpper", "to": "Standard.MouthRollUpper" }, - { "from": "OSC.MouthRollLower", "to": "Standard.MouthRollLower" }, - { "from": "OSC.MouthShrugUpper", "to": "Standard.MouthShrugUpper" }, - { "from": "OSC.MouthShrugLower", "to": "Standard.MouthShrugLower" }, + { "from": "OSC.MouthFrown_L", "to": "Standard.MouthFrown_L" }, + { "from": "OSC.MouthFrown_R", "to": "Standard.MouthFrown_R" }, + { "from": "OSC.MouthSmile_L", "to": "Standard.MouthSmile_L" }, + { "from": "OSC.MouthSmile_R", "to": "Standard.MouthSmile_R" }, + { "from": "OSC.MouthDimple_L", "to": "Standard.MouthDimple_L" }, + { "from": "OSC.MouthDimple_R", "to": "Standard.MouthDimple_R" }, + { "from": "OSC.LipsStretch_L", "to": "Standard.LipsStretch_L" }, + { "from": "OSC.LipsStretch_R", "to": "Standard.LipsStretch_R" }, + { "from": "OSC.LipsUpperClose", "to": "Standard.LipsUpperClose" }, + { "from": "OSC.LipsLowerClose", "to": "Standard.LipsLowerClose" }, + { "from": "OSC.LipsFunnel", "to": "Standard.LipsFunnel" }, + { "from": "OSC.LipsPucker", "to": "Standard.LipsPucker" }, + { "from": "OSC.Puff", "to": "Standard.Puff" }, + { "from": "OSC.CheekSquint_L", "to": "Standard.CheekSquint_L" }, + { "from": "OSC.CheekSquint_R", "to": "Standard.CheekSquint_R" }, { "from": "OSC.MouthClose", "to": "Standard.MouthClose" }, - { "from": "OSC.MouthSmileLeft", "to": "Standard.MouthSmileLeft" }, - { "from": "OSC.MouthSmileRight", "to": "Standard.MouthSmileRight" }, - { "from": "OSC.MouthFrownLeft", "to": "Standard.MouthFrownLeft" }, - { "from": "OSC.MouthFrownRight", "to": "Standard.MouthFrownRight" }, - { "from": "OSC.MouthDimpleLeft", "to": "Standard.MouthDimpleLeft" }, - { "from": "OSC.MouthDimpleRight", "to": "Standard.MouthDimpleRight" }, - { "from": "OSC.MouthUpperUpLeft", "to": "Standard.MouthUpperUpLeft" }, - { "from": "OSC.MouthUpperUpRight", "to": "Standard.MouthUpperUpRight" }, - { "from": "OSC.MouthLowerDownLeft", "to": "Standard.MouthLowerDownLeft" }, - { "from": "OSC.MouthLowerDownRight", "to": "Standard.MouthLowerDownRight" }, - { "from": "OSC.MouthPressLeft", "to": "Standard.MouthPressLeft" }, - { "from": "OSC.MouthPressRight", "to": "Standard.MouthPressRight" }, - { "from": "OSC.MouthStretchLeft", "to": "Standard.MouthStretchLeft" }, - { "from": "OSC.MouthStretchRight", "to": "Standard.MouthStretchRight" }, - { "from": "OSC.TongueOut", "to": "Standard.TongueOut" } + { "from": "OSC.MouthUpperUp_L", "to": "Standard.MouthUpperUp_L" }, + { "from": "OSC.MouthUpperUp_R", "to": "Standard.MouthUpperUp_R" }, + { "from": "OSC.MouthLowerDown_L", "to": "Standard.MouthLowerDown_L" }, + { "from": "OSC.MouthLowerDown_R", "to": "Standard.MouthLowerDown_R" }, + { "from": "OSC.MouthPress_L", "to": "Standard.MouthPress_L" }, + { "from": "OSC.MouthPress_R", "to": "Standard.MouthPress_R" }, + { "from": "OSC.MouthShrugLower", "to": "Standard.MouthShrugLower" }, + { "from": "OSC.MouthShrugUpper", "to": "Standard.MouthShrugUpper" }, + { "from": "OSC.NoseSneer_L", "to": "Standard.NoseSneer_L" }, + { "from": "OSC.NoseSneer_R", "to": "Standard.NoseSneer_R" }, + { "from": "OSC.TongueOut", "to": "Standard.TongueOut" }, + { "from": "OSC.UserBlendshape0", "to": "Standard.UserBlendshape0" }, + { "from": "OSC.UserBlendshape1", "to": "Standard.UserBlendshape1" }, + { "from": "OSC.UserBlendshape2", "to": "Standard.UserBlendshape2" }, + { "from": "OSC.UserBlendshape3", "to": "Standard.UserBlendshape3" }, + { "from": "OSC.UserBlendshape4", "to": "Standard.UserBlendshape4" }, + { "from": "OSC.UserBlendshape5", "to": "Standard.UserBlendshape5" }, + { "from": "OSC.UserBlendshape6", "to": "Standard.UserBlendshape6" }, + { "from": "OSC.UserBlendshape7", "to": "Standard.UserBlendshape7" }, + { "from": "OSC.UserBlendshape8", "to": "Standard.UserBlendshape8" }, + { "from": "OSC.UserBlendshape9", "to": "Standard.UserBlendshape9" } ] } diff --git a/interface/src/avatar/MyHead.cpp b/interface/src/avatar/MyHead.cpp index 1b88a518c8..4b6f85de1c 100644 --- a/interface/src/avatar/MyHead.cpp +++ b/interface/src/avatar/MyHead.cpp @@ -134,11 +134,11 @@ void MyHead::simulate(float deltaTime) { userInputMapper->getActionStateValid(controller::Action::MOUTHSMILE_L) || userInputMapper->getActionStateValid(controller::Action::MOUTHSMILE_R); - bool eyesTracked = - userInputMapper->getPoseState(controller::Action::LEFT_EYE).valid && - userInputMapper->getPoseState(controller::Action::RIGHT_EYE).valid; - MyAvatar* myAvatar = static_cast(_owningAvatar); + bool eyesTracked = + myAvatar->getControllerPoseInSensorFrame(controller::Action::LEFT_EYE).valid && + myAvatar->getControllerPoseInSensorFrame(controller::Action::RIGHT_EYE).valid; + int leftEyeJointIndex = myAvatar->getJointIndex("LeftEye"); int rightEyeJointIndex = myAvatar->getJointIndex("RightEye"); bool eyeJointsOverridden = myAvatar->getIsJointOverridden(leftEyeJointIndex) || myAvatar->getIsJointOverridden(rightEyeJointIndex); diff --git a/libraries/avatars-renderer/src/avatars-renderer/Head.cpp b/libraries/avatars-renderer/src/avatars-renderer/Head.cpp index b8bc7a03e8..761f73ae05 100644 --- a/libraries/avatars-renderer/src/avatars-renderer/Head.cpp +++ b/libraries/avatars-renderer/src/avatars-renderer/Head.cpp @@ -98,6 +98,7 @@ void Head::simulate(float deltaTime) { const float FULLY_CLOSED = 1.0f; if (getProceduralAnimationFlag(HeadData::BlinkProceduralBlendshapeAnimation) && !getSuppressProceduralAnimationFlag(HeadData::BlinkProceduralBlendshapeAnimation)) { + // handle automatic blinks // Detect transition from talking to not; force blink after that and a delay bool forceBlink = false; @@ -155,6 +156,7 @@ void Head::simulate(float deltaTime) { // use data to update fake Faceshift blendshape coefficients if (getProceduralAnimationFlag(HeadData::AudioProceduralBlendshapeAnimation) && !getSuppressProceduralAnimationFlag(HeadData::AudioProceduralBlendshapeAnimation)) { + // Update audio attack data for facial animation (eyebrows and mouth) float audioAttackAveragingRate = (10.0f - deltaTime * NORMAL_HZ) / 10.0f; // --> 0.9 at 60 Hz _audioAttack = audioAttackAveragingRate * _audioAttack + @@ -188,6 +190,7 @@ void Head::simulate(float deltaTime) { if (getProceduralAnimationFlag(HeadData::LidAdjustmentProceduralBlendshapeAnimation) && !getSuppressProceduralAnimationFlag(HeadData::LidAdjustmentProceduralBlendshapeAnimation)) { + // This controls two things, the eye brow and the upper eye lid, it is driven by the vertical up/down angle of the // eyes relative to the head. This is to try to help prevent sleepy eyes/crazy eyes. applyEyelidOffset(getOrientation()); diff --git a/plugins/hifiOsc/src/OscPlugin.cpp b/plugins/hifiOsc/src/OscPlugin.cpp index 0303d61ae2..475691069f 100644 --- a/plugins/hifiOsc/src/OscPlugin.cpp +++ b/plugins/hifiOsc/src/OscPlugin.cpp @@ -373,16 +373,16 @@ static int genericHandlerFunc(const char* path, const char* types, lo_arg** argv // map /ELR to left eye rot if (path[0] == '/' && path[1] == 'E' && path[2] == 'L' && path[3] == 'R' && - types[0] == 'f' && types[1] == 'f' && types[2] == 'f') { - glm::vec3 euler(-argv[0]->f, -argv[1]->f, argv[2]->f); + types[0] == 'f' && types[1] == 'f') { + glm::vec3 euler(-argv[0]->f, -argv[1]->f, 0.0f); container->_eyeLeftRot = glm::quat(glm::radians(euler)) * Quaternions::Y_180; container->_eyeLeftRotValid = true; } // map /ERR to right eye rot if (path[0] == '/' && path[1] == 'E' && path[2] == 'R' && path[3] == 'R' && - types[0] == 'f' && types[1] == 'f' && types[2] == 'f') { - glm::vec3 euler(-argv[0]->f, -argv[1]->f, argv[2]->f); + types[0] == 'f' && types[1] == 'f') { + glm::vec3 euler((float)-argv[0]->f, (float)-argv[1]->f, 0.0f); container->_eyeRightRot = glm::quat(glm::radians(euler)) * Quaternions::Y_180; container->_eyeRightRotValid = true; } @@ -597,6 +597,7 @@ QString OscPlugin::InputDevice::getDefaultMappingConfig() const { } void OscPlugin::InputDevice::update(float deltaTime, const controller::InputCalibrationData& inputCalibrationData) { + glm::mat4 sensorToAvatarMat = glm::inverse(inputCalibrationData.avatarMat) * inputCalibrationData.sensorToWorldMat; std::lock_guard guard(_container->_dataMutex); for (int i = 0; i < (int)FaceCap::BlendshapeCount; i++) { if (_container->_blendshapeValidFlags[i]) { @@ -604,20 +605,22 @@ void OscPlugin::InputDevice::update(float deltaTime, const controller::InputCali } } if (_container->_headRotValid && _container->_headTransValid) { - const float SMOOTH_TIMESCALE = 2.0f; float tau = deltaTime / SMOOTH_TIMESCALE; _container->_headTransSmoothed = lerp(_container->_headTransSmoothed, _container->_headTransTarget, tau); glm::vec3 delta = _container->_headTransSmoothed - _container->_headTransTarget; glm::vec3 trans = extractTranslation(inputCalibrationData.defaultHeadMat) + delta; - _poseStateMap[controller::HEAD] = controller::Pose(trans, _container->_headRot); + controller::Pose sensorSpacePose(trans, _container->_headRot); + _poseStateMap[controller::HEAD] = sensorSpacePose.transform(sensorToAvatarMat); } if (_container->_eyeLeftRotValid) { - _poseStateMap[controller::LEFT_EYE] = controller::Pose(vec3(0.0f), _container->_eyeLeftRot); + controller::Pose sensorSpacePose(vec3(0.0f), _container->_eyeLeftRot); + _poseStateMap[controller::LEFT_EYE] = sensorSpacePose.transform(sensorToAvatarMat); } if (_container->_eyeRightRotValid) { - _poseStateMap[controller::RIGHT_EYE] = controller::Pose(vec3(0.0f), _container->_eyeRightRot); + controller::Pose sensorSpacePose(vec3(0.0f), _container->_eyeRightRot); + _poseStateMap[controller::RIGHT_EYE] = sensorSpacePose.transform(sensorToAvatarMat); } }