Mirrors/overte-HifiExperiments
3
0
Fork 0
mirror of https://github.com/HifiExperiments/overte.git synced 2025-04-07 10:02:24 +02:00
Code Issues Projects Releases Packages Wiki Activity Actions

Head is now computed in proper sensor space

Browse source 
Eye tracking now works.
This commit is contained in:
Anthony J. Thibault 2019-11-20 15:30:16 -08:00
parent befbdcef48
commit 9d6ce44131
4 changed files with 76 additions and 59 deletions
Show stats Download patch file Download diff file Expand all files Collapse all files

105
interface/resources/controllers/osc.json
View file

@ -4,57 +4,68 @@
{ "from": "OSC.Head", "to" : "Standard.Head" },
{ "from": "OSC.LeftEye", "to" : "Standard.LeftEye" },
{ "from": "OSC.RightEye", "to" : "Standard.RightEye" },
{ "from": "OSC.BrowInnerUp", "to": "Standard.BrowInnerUp" },
{ "from": "OSC.BrowDownLeft", "to": "Standard.BrowDownLeft" },
{ "from": "OSC.BrowDownRight", "to": "Standard.BrowDownRight" },
{ "from": "OSC.BrowOuterUpLeft", "to": "Standard.BrowOuterUpLeft" },
{ "from": "OSC.BrowOuterUpRight", "to": "Standard.BrowOuterUpRight" },
{ "from": "OSC.EyeLookUpLeft", "to": "Standard.EyeLookUpLeft" },
{ "from": "OSC.EyeLookUpRight", "to": "Standard.EyeLookUpRight" },
{ "from": "OSC.EyeLookDownLeft", "to": "Standard.EyeLookDownLeft" },
{ "from": "OSC.EyeLookDownRight", "to": "Standard.EyeLookDownRight" },
{ "from": "OSC.EyeLookInLeft", "to": "Standard.EyeLookInLeft" },
{ "from": "OSC.EyeLookInRight", "to": "Standard.EyeLookInRight" },
{ "from": "OSC.EyeLookOutLeft", "to": "Standard.EyeLookOutLeft" },
{ "from": "OSC.EyeLookOutRight", "to": "Standard.EyeLookOutRight" },
{ "from": "OSC.EyeBlinkLeft", "to": "Standard.EyeBlinkLeft" },
{ "from": "OSC.EyeBlinkRight", "to": "Standard.EyeBlinkRight" },
{ "from": "OSC.EyeSquintLeft", "to": "Standard.EyeSquintLeft" },
{ "from": "OSC.EyeSquintRight", "to": "Standard.EyeSquintRight" },
{ "from": "OSC.EyeWideLeft", "to": "Standard.EyeWideLeft" },
{ "from": "OSC.EyeWideRight", "to": "Standard.EyeWideRight" },
{ "from": "OSC.CheekPuff", "to": "Standard.CheekPuff" },
{ "from": "OSC.CheekSquintLeft", "to": "Standard.CheekSquintLeft" },
{ "from": "OSC.CheekSquintRight", "to": "Standard.CheekSquintRight" },
{ "from": "OSC.NoseSneerLeft", "to": "Standard.NoseSneerLeft" },
{ "from": "OSC.NoseSneerRight", "to": "Standard.NoseSneerRight" },
{ "from": "OSC.JawOpen", "to": "Standard.JawOpen" },
{ "from": "OSC.JawForward", "to": "Standard.JawForward" },
{ "from": "OSC.EyeBlink_L", "to": "Standard.EyeBlink_L" },
{ "from": "OSC.EyeBlink_R", "to": "Standard.EyeBlink_R" },
{ "from": "OSC.EyeSquint_L", "to": "Standard.EyeSquint_L" },
{ "from": "OSC.EyeSquint_R", "to": "Standard.EyeSquint_R" },
{ "from": "OSC.EyeDown_L", "to": "Standard.EyeDown_L" },
{ "from": "OSC.EyeDown_R", "to": "Standard.EyeDown_R" },
{ "from": "OSC.EyeIn_L", "to": "Standard.EyeIn_L" },
{ "from": "OSC.EyeIn_R", "to": "Standard.EyeIn_R" },
{ "from": "OSC.EyeOpen_L", "to": "Standard.EyeOpen_L" },
{ "from": "OSC.EyeOpen_R", "to": "Standard.EyeOpen_R" },
{ "from": "OSC.EyeOut_L", "to": "Standard.EyeOut_L" },
{ "from": "OSC.EyeOut_R", "to": "Standard.EyeOut_R" },
{ "from": "OSC.EyeUp_L", "to": "Standard.EyeUp_L" },
{ "from": "OSC.EyeUp_R", "to": "Standard.EyeUp_R" },
{ "from": "OSC.BrowsD_L", "to": "Standard.BrowsD_L" },
{ "from": "OSC.BrowsD_R", "to": "Standard.BrowsD_R" },
{ "from": "OSC.BrowsU_C", "to": "Standard.BrowsU_C" },
{ "from": "OSC.BrowsU_L", "to": "Standard.BrowsU_L" },
{ "from": "OSC.BrowsU_R", "to": "Standard.BrowsU_R" },
{ "from": "OSC.JawFwd", "to": "Standard.JawFwd" },
{ "from": "OSC.JawLeft", "to": "Standard.JawLeft" },
{ "from": "OSC.JawOpen", "to": "Standard.JawOpen" },
{ "from": "OSC.JawRight", "to": "Standard.JawRight" },
{ "from": "OSC.MouthFunnel", "to": "Standard.MouthFunnel" },
{ "from": "OSC.MouthPucker", "to": "Standard.MouthPucker" },
{ "from": "OSC.MouthLeft", "to": "Standard.MouthLeft" },
{ "from": "OSC.MouthRight", "to": "Standard.MouthRight" },
{ "from": "OSC.MouthRollUpper", "to": "Standard.MouthRollUpper" },
{ "from": "OSC.MouthRollLower", "to": "Standard.MouthRollLower" },
{ "from": "OSC.MouthShrugUpper", "to": "Standard.MouthShrugUpper" },
{ "from": "OSC.MouthShrugLower", "to": "Standard.MouthShrugLower" },
{ "from": "OSC.MouthFrown_L", "to": "Standard.MouthFrown_L" },
{ "from": "OSC.MouthFrown_R", "to": "Standard.MouthFrown_R" },
{ "from": "OSC.MouthSmile_L", "to": "Standard.MouthSmile_L" },
{ "from": "OSC.MouthSmile_R", "to": "Standard.MouthSmile_R" },
{ "from": "OSC.MouthDimple_L", "to": "Standard.MouthDimple_L" },
{ "from": "OSC.MouthDimple_R", "to": "Standard.MouthDimple_R" },
{ "from": "OSC.LipsStretch_L", "to": "Standard.LipsStretch_L" },
{ "from": "OSC.LipsStretch_R", "to": "Standard.LipsStretch_R" },
{ "from": "OSC.LipsUpperClose", "to": "Standard.LipsUpperClose" },
{ "from": "OSC.LipsLowerClose", "to": "Standard.LipsLowerClose" },
{ "from": "OSC.LipsFunnel", "to": "Standard.LipsFunnel" },
{ "from": "OSC.LipsPucker", "to": "Standard.LipsPucker" },
{ "from": "OSC.Puff", "to": "Standard.Puff" },
{ "from": "OSC.CheekSquint_L", "to": "Standard.CheekSquint_L" },
{ "from": "OSC.CheekSquint_R", "to": "Standard.CheekSquint_R" },
{ "from": "OSC.MouthClose", "to": "Standard.MouthClose" },
{ "from": "OSC.MouthSmileLeft", "to": "Standard.MouthSmileLeft" },
{ "from": "OSC.MouthSmileRight", "to": "Standard.MouthSmileRight" },
{ "from": "OSC.MouthFrownLeft", "to": "Standard.MouthFrownLeft" },
{ "from": "OSC.MouthFrownRight", "to": "Standard.MouthFrownRight" },
{ "from": "OSC.MouthDimpleLeft", "to": "Standard.MouthDimpleLeft" },
{ "from": "OSC.MouthDimpleRight", "to": "Standard.MouthDimpleRight" },
{ "from": "OSC.MouthUpperUpLeft", "to": "Standard.MouthUpperUpLeft" },
{ "from": "OSC.MouthUpperUpRight", "to": "Standard.MouthUpperUpRight" },
{ "from": "OSC.MouthLowerDownLeft", "to": "Standard.MouthLowerDownLeft" },
{ "from": "OSC.MouthLowerDownRight", "to": "Standard.MouthLowerDownRight" },
{ "from": "OSC.MouthPressLeft", "to": "Standard.MouthPressLeft" },
{ "from": "OSC.MouthPressRight", "to": "Standard.MouthPressRight" },
{ "from": "OSC.MouthStretchLeft", "to": "Standard.MouthStretchLeft" },
{ "from": "OSC.MouthStretchRight", "to": "Standard.MouthStretchRight" },
{ "from": "OSC.TongueOut", "to": "Standard.TongueOut" }
{ "from": "OSC.MouthUpperUp_L", "to": "Standard.MouthUpperUp_L" },
{ "from": "OSC.MouthUpperUp_R", "to": "Standard.MouthUpperUp_R" },
{ "from": "OSC.MouthLowerDown_L", "to": "Standard.MouthLowerDown_L" },
{ "from": "OSC.MouthLowerDown_R", "to": "Standard.MouthLowerDown_R" },
{ "from": "OSC.MouthPress_L", "to": "Standard.MouthPress_L" },
{ "from": "OSC.MouthPress_R", "to": "Standard.MouthPress_R" },
{ "from": "OSC.MouthShrugLower", "to": "Standard.MouthShrugLower" },
{ "from": "OSC.MouthShrugUpper", "to": "Standard.MouthShrugUpper" },
{ "from": "OSC.NoseSneer_L", "to": "Standard.NoseSneer_L" },
{ "from": "OSC.NoseSneer_R", "to": "Standard.NoseSneer_R" },
{ "from": "OSC.TongueOut", "to": "Standard.TongueOut" },
{ "from": "OSC.UserBlendshape0", "to": "Standard.UserBlendshape0" },
{ "from": "OSC.UserBlendshape1", "to": "Standard.UserBlendshape1" },
{ "from": "OSC.UserBlendshape2", "to": "Standard.UserBlendshape2" },
{ "from": "OSC.UserBlendshape3", "to": "Standard.UserBlendshape3" },
{ "from": "OSC.UserBlendshape4", "to": "Standard.UserBlendshape4" },
{ "from": "OSC.UserBlendshape5", "to": "Standard.UserBlendshape5" },
{ "from": "OSC.UserBlendshape6", "to": "Standard.UserBlendshape6" },
{ "from": "OSC.UserBlendshape7", "to": "Standard.UserBlendshape7" },
{ "from": "OSC.UserBlendshape8", "to": "Standard.UserBlendshape8" },
{ "from": "OSC.UserBlendshape9", "to": "Standard.UserBlendshape9" }
]
}

8
interface/src/avatar/MyHead.cpp
View file

@ -134,11 +134,11 @@ void MyHead::simulate(float deltaTime) {
userInputMapper->getActionStateValid(controller::Action::MOUTHSMILE_L) ||
userInputMapper->getActionStateValid(controller::Action::MOUTHSMILE_R);
bool eyesTracked =
userInputMapper->getPoseState(controller::Action::LEFT_EYE).valid &&
userInputMapper->getPoseState(controller::Action::RIGHT_EYE).valid;
MyAvatar* myAvatar = static_cast<MyAvatar*>(_owningAvatar);
bool eyesTracked =
myAvatar->getControllerPoseInSensorFrame(controller::Action::LEFT_EYE).valid &&
myAvatar->getControllerPoseInSensorFrame(controller::Action::RIGHT_EYE).valid;
int leftEyeJointIndex = myAvatar->getJointIndex("LeftEye");
int rightEyeJointIndex = myAvatar->getJointIndex("RightEye");
bool eyeJointsOverridden = myAvatar->getIsJointOverridden(leftEyeJointIndex) || myAvatar->getIsJointOverridden(rightEyeJointIndex);

3
libraries/avatars-renderer/src/avatars-renderer/Head.cpp
View file

@ -98,6 +98,7 @@ void Head::simulate(float deltaTime) {
const float FULLY_CLOSED = 1.0f;
if (getProceduralAnimationFlag(HeadData::BlinkProceduralBlendshapeAnimation) &&
!getSuppressProceduralAnimationFlag(HeadData::BlinkProceduralBlendshapeAnimation)) {
// handle automatic blinks
// Detect transition from talking to not; force blink after that and a delay
bool forceBlink = false;
@ -155,6 +156,7 @@ void Head::simulate(float deltaTime) {
// use data to update fake Faceshift blendshape coefficients
if (getProceduralAnimationFlag(HeadData::AudioProceduralBlendshapeAnimation) &&
!getSuppressProceduralAnimationFlag(HeadData::AudioProceduralBlendshapeAnimation)) {
// Update audio attack data for facial animation (eyebrows and mouth)
float audioAttackAveragingRate = (10.0f - deltaTime * NORMAL_HZ) / 10.0f; // --> 0.9 at 60 Hz
_audioAttack = audioAttackAveragingRate * _audioAttack +
@ -188,6 +190,7 @@ void Head::simulate(float deltaTime) {
if (getProceduralAnimationFlag(HeadData::LidAdjustmentProceduralBlendshapeAnimation) &&
!getSuppressProceduralAnimationFlag(HeadData::LidAdjustmentProceduralBlendshapeAnimation)) {
// This controls two things, the eye brow and the upper eye lid, it is driven by the vertical up/down angle of the
// eyes relative to the head. This is to try to help prevent sleepy eyes/crazy eyes.
applyEyelidOffset(getOrientation());

19
plugins/hifiOsc/src/OscPlugin.cpp
View file

@ -373,16 +373,16 @@ static int genericHandlerFunc(const char* path, const char* types, lo_arg** argv
// map /ELR to left eye rot
if (path[0] == '/' && path[1] == 'E' && path[2] == 'L' && path[3] == 'R' &&
types[0] == 'f' && types[1] == 'f' && types[2] == 'f') {
glm::vec3 euler(-argv[0]->f, -argv[1]->f, argv[2]->f);
types[0] == 'f' && types[1] == 'f') {
glm::vec3 euler(-argv[0]->f, -argv[1]->f, 0.0f);
container->_eyeLeftRot = glm::quat(glm::radians(euler)) * Quaternions::Y_180;
container->_eyeLeftRotValid = true;
}
// map /ERR to right eye rot
if (path[0] == '/' && path[1] == 'E' && path[2] == 'R' && path[3] == 'R' &&
types[0] == 'f' && types[1] == 'f' && types[2] == 'f') {
glm::vec3 euler(-argv[0]->f, -argv[1]->f, argv[2]->f);
types[0] == 'f' && types[1] == 'f') {
glm::vec3 euler((float)-argv[0]->f, (float)-argv[1]->f, 0.0f);
container->_eyeRightRot = glm::quat(glm::radians(euler)) * Quaternions::Y_180;
container->_eyeRightRotValid = true;
}
@ -597,6 +597,7 @@ QString OscPlugin::InputDevice::getDefaultMappingConfig() const {
}
void OscPlugin::InputDevice::update(float deltaTime, const controller::InputCalibrationData& inputCalibrationData) {
glm::mat4 sensorToAvatarMat = glm::inverse(inputCalibrationData.avatarMat) * inputCalibrationData.sensorToWorldMat;
std::lock_guard<std::mutex> guard(_container->_dataMutex);
for (int i = 0; i < (int)FaceCap::BlendshapeCount; i++) {
if (_container->_blendshapeValidFlags[i]) {
@ -604,20 +605,22 @@ void OscPlugin::InputDevice::update(float deltaTime, const controller::InputCali
}
}
if (_container->_headRotValid && _container->_headTransValid) {
const float SMOOTH_TIMESCALE = 2.0f;
float tau = deltaTime / SMOOTH_TIMESCALE;
_container->_headTransSmoothed = lerp(_container->_headTransSmoothed, _container->_headTransTarget, tau);
glm::vec3 delta = _container->_headTransSmoothed - _container->_headTransTarget;
glm::vec3 trans = extractTranslation(inputCalibrationData.defaultHeadMat) + delta;
_poseStateMap[controller::HEAD] = controller::Pose(trans, _container->_headRot);
controller::Pose sensorSpacePose(trans, _container->_headRot);
_poseStateMap[controller::HEAD] = sensorSpacePose.transform(sensorToAvatarMat);
}
if (_container->_eyeLeftRotValid) {
_poseStateMap[controller::LEFT_EYE] = controller::Pose(vec3(0.0f), _container->_eyeLeftRot);
controller::Pose sensorSpacePose(vec3(0.0f), _container->_eyeLeftRot);
_poseStateMap[controller::LEFT_EYE] = sensorSpacePose.transform(sensorToAvatarMat);
}
if (_container->_eyeRightRotValid) {
_poseStateMap[controller::RIGHT_EYE] = controller::Pose(vec3(0.0f), _container->_eyeRightRot);
controller::Pose sensorSpacePose(vec3(0.0f), _container->_eyeRightRot);
_poseStateMap[controller::RIGHT_EYE] = sensorSpacePose.transform(sensorToAvatarMat);
}
}