From 389f5a1d335ad4c88f358fb8ff7356a74741cac6 Mon Sep 17 00:00:00 2001 From: "Anthony J. Thibault" Date: Tue, 15 Oct 2019 11:37:10 -0700 Subject: [PATCH] ARKit Blendshape hookup work --- interface/resources/controllers/standard.json | 68 ++++++++- .../controllers/standard_nomovement.json | 67 ++++++++- interface/resources/controllers/vive.json | 5 +- interface/src/avatar/MyAvatar.cpp | 12 -- interface/src/avatar/MyHead.cpp | 142 ++++++++++++++---- interface/src/avatar/MySkeletonModel.cpp | 5 + .../src/avatars-renderer/Head.cpp | 12 +- .../src/avatars-renderer/SkeletonModel.cpp | 4 + libraries/avatars/src/AvatarData.cpp | 79 ++++++---- libraries/avatars/src/AvatarData.h | 11 +- libraries/avatars/src/HeadData.cpp | 60 ++++---- libraries/avatars/src/HeadData.h | 43 ++++-- .../controllers/src/controllers/Actions.h | 128 ++++++++-------- .../src/controllers/StandardControls.h | 66 +++++++- .../networking/src/udt/PacketHeaders.cpp | 4 +- libraries/networking/src/udt/PacketHeaders.h | 3 +- libraries/shared/src/BlendshapeConstants.h | 2 +- 17 files changed, 507 insertions(+), 204 deletions(-) diff --git a/interface/resources/controllers/standard.json b/interface/resources/controllers/standard.json index 7f24924f7b..8c84039b86 100644 --- a/interface/resources/controllers/standard.json +++ b/interface/resources/controllers/standard.json @@ -166,10 +166,70 @@ { "from": "Standard.LeftEye", "to": "Actions.LeftEye" }, { "from": "Standard.RightEye", "to": "Actions.RightEye" }, - // AJT: blendshapes - { "from": "Standard.LeftEyeBlink", "to": "Actions.LeftEyeBlink" }, - { "from": "Standard.RightEyeBlink", "to": "Actions.RightEyeBlink" }, - + { "from": "Standard.EyeBlink_L", "to": "Actions.EyeBlink_L" }, + { "from": "Standard.EyeBlink_R", "to": "Actions.EyeBlink_R" }, + { "from": "Standard.EyeSquint_L", "to": "Actions.EyeSquint_L" }, + { "from": "Standard.EyeSquint_R", "to": "Actions.EyeSquint_R" }, + { "from": "Standard.EyeDown_L", "to": "Actions.EyeDown_L" }, + { "from": "Standard.EyeDown_R", "to": "Actions.EyeDown_R" }, + { "from": "Standard.EyeIn_L", "to": "Actions.EyeIn_L" }, + { "from": "Standard.EyeIn_R", "to": "Actions.EyeIn_R" }, + { "from": "Standard.EyeOpen_L", "to": "Actions.EyeOpen_L" }, + { "from": "Standard.EyeOpen_R", "to": "Actions.EyeOpen_R" }, + { "from": "Standard.EyeOut_L", "to": "Actions.EyeOut_L" }, + { "from": "Standard.EyeOut_R", "to": "Actions.EyeOut_R" }, + { "from": "Standard.EyeUp_L", "to": "Actions.EyeUp_L" }, + { "from": "Standard.EyeUp_R", "to": "Actions.EyeUp_R" }, + { "from": "Standard.BrowsD_L", "to": "Actions.BrowsD_L" }, + { "from": "Standard.BrowsD_R", "to": "Actions.BrowsD_R" }, + { "from": "Standard.BrowsU_C", "to": "Actions.BrowsU_C" }, + { "from": "Standard.BrowsU_L", "to": "Actions.BrowsU_L" }, + { "from": "Standard.BrowsU_R", "to": "Actions.BrowsU_R" }, + { "from": "Standard.JawFwd", "to": "Actions.JawFwd" }, + { "from": "Standard.JawLeft", "to": "Actions.JawLeft" }, + { "from": "Standard.JawOpen", "to": "Actions.JawOpen" }, + { "from": "Standard.JawRight", "to": "Actions.JawRight" }, + { "from": "Standard.MouthLeft", "to": "Actions.MouthLeft" }, + { "from": "Standard.MouthRight", "to": "Actions.MouthRight" }, + { "from": "Standard.MouthFrown_L", "to": "Actions.MouthFrown_L" }, + { "from": "Standard.MouthFrown_R", "to": "Actions.MouthFrown_R" }, + { "from": "Standard.MouthSmile_L", "to": "Actions.MouthSmile_L" }, + { "from": "Standard.MouthSmile_R", "to": "Actions.MouthSmile_R" }, + { "from": "Standard.MouthDimple_L", "to": "Actions.MouthDimple_L" }, + { "from": "Standard.MouthDimple_R", "to": "Actions.MouthDimple_R" }, + { "from": "Standard.LipsStretch_L", "to": "Actions.LipsStretch_L" }, + { "from": "Standard.LipsStretch_R", "to": "Actions.LipsStretch_R" }, + { "from": "Standard.LipsUpperClose", "to": "Actions.LipsUpperClose" }, + { "from": "Standard.LipsLowerClose", "to": "Actions.LipsLowerClose" }, + { "from": "Standard.LipsUpperOpen", "to": "Actions.LipsUpperOpen" }, + { "from": "Standard.LipsLowerOpen", "to": "Actions.LipsLowerOpen" }, + { "from": "Standard.LipsFunnel", "to": "Actions.LipsFunnel" }, + { "from": "Standard.LipsPucker", "to": "Actions.LipsPucker" }, + { "from": "Standard.Puff", "to": "Actions.Puff" }, + { "from": "Standard.CheekSquint_L", "to": "Actions.CheekSquint_L" }, + { "from": "Standard.CheekSquint_R", "to": "Actions.CheekSquint_R" }, + { "from": "Standard.LipsTogether", "to": "Actions.LipsTogether" }, + { "from": "Standard.MouthUpperUp_L", "to": "Actions.MouthUpperUp_L" }, + { "from": "Standard.MouthUpperUp_R", "to": "Actions.MouthUpperUp_R" }, + { "from": "Standard.MouthLowerDown_L", "to": "Actions.MouthLowerDown_L" }, + { "from": "Standard.MouthLowerDown_R", "to": "Actions.MouthLowerDown_R" }, + { "from": "Standard.MouthPress_L", "to": "Actions.MouthPress_L" }, + { "from": "Standard.MouthPress_R", "to": "Actions.MouthPress_R" }, + { "from": "Standard.MouthShrugLower", "to": "Actions.MouthShrugLower" }, + { "from": "Standard.MouthShrugUpper", "to": "Actions.MouthShrugUpper" }, + { "from": "Standard.NoseSneer_L", "to": "Actions.NoseSneer_L" }, + { "from": "Standard.NoseSneer_R", "to": "Actions.NoseSneer_R" }, + { "from": "Standard.TongueOut", "to": "Actions.TongueOut" }, + { "from": "Standard.UserBlendshape0", "to": "Actions.UserBlendshape0" }, + { "from": "Standard.UserBlendshape1", "to": "Actions.UserBlendshape1" }, + { "from": "Standard.UserBlendshape2", "to": "Actions.UserBlendshape2" }, + { "from": "Standard.UserBlendshape3", "to": "Actions.UserBlendshape3" }, + { "from": "Standard.UserBlendshape4", "to": "Actions.UserBlendshape4" }, + { "from": "Standard.UserBlendshape5", "to": "Actions.UserBlendshape5" }, + { "from": "Standard.UserBlendshape6", "to": "Actions.UserBlendshape6" }, + { "from": "Standard.UserBlendshape7", "to": "Actions.UserBlendshape7" }, + { "from": "Standard.UserBlendshape8", "to": "Actions.UserBlendshape8" }, + { "from": "Standard.UserBlendshape9", "to": "Actions.UserBlendshape9" }, { "from": "Standard.TrackedObject00", "to" : "Actions.TrackedObject00" }, { "from": "Standard.TrackedObject01", "to" : "Actions.TrackedObject01" }, diff --git a/interface/resources/controllers/standard_nomovement.json b/interface/resources/controllers/standard_nomovement.json index 67eddfad31..e311a5fa62 100644 --- a/interface/resources/controllers/standard_nomovement.json +++ b/interface/resources/controllers/standard_nomovement.json @@ -61,9 +61,70 @@ { "from": "Standard.LeftEye", "to": "Actions.LeftEye" }, { "from": "Standard.RightEye", "to": "Actions.RightEye" }, - // AJT: blendshapes - { "from": "Standard.LeftEyeBlink", "to": "Actions.LeftEyeBlink" }, - { "from": "Standard.RightEyeBlink", "to": "Actions.RightEyeBlink" }, + { "from": "Standard.EyeBlink_L", "to": "Actions.EyeBlink_L" }, + { "from": "Standard.EyeBlink_R", "to": "Actions.EyeBlink_R" }, + { "from": "Standard.EyeSquint_L", "to": "Actions.EyeSquint_L" }, + { "from": "Standard.EyeSquint_R", "to": "Actions.EyeSquint_R" }, + { "from": "Standard.EyeDown_L", "to": "Actions.EyeDown_L" }, + { "from": "Standard.EyeDown_R", "to": "Actions.EyeDown_R" }, + { "from": "Standard.EyeIn_L", "to": "Actions.EyeIn_L" }, + { "from": "Standard.EyeIn_R", "to": "Actions.EyeIn_R" }, + { "from": "Standard.EyeOpen_L", "to": "Actions.EyeOpen_L" }, + { "from": "Standard.EyeOpen_R", "to": "Actions.EyeOpen_R" }, + { "from": "Standard.EyeOut_L", "to": "Actions.EyeOut_L" }, + { "from": "Standard.EyeOut_R", "to": "Actions.EyeOut_R" }, + { "from": "Standard.EyeUp_L", "to": "Actions.EyeUp_L" }, + { "from": "Standard.EyeUp_R", "to": "Actions.EyeUp_R" }, + { "from": "Standard.BrowsD_L", "to": "Actions.BrowsD_L" }, + { "from": "Standard.BrowsD_R", "to": "Actions.BrowsD_R" }, + { "from": "Standard.BrowsU_C", "to": "Actions.BrowsU_C" }, + { "from": "Standard.BrowsU_L", "to": "Actions.BrowsU_L" }, + { "from": "Standard.BrowsU_R", "to": "Actions.BrowsU_R" }, + { "from": "Standard.JawFwd", "to": "Actions.JawFwd" }, + { "from": "Standard.JawLeft", "to": "Actions.JawLeft" }, + { "from": "Standard.JawOpen", "to": "Actions.JawOpen" }, + { "from": "Standard.JawRight", "to": "Actions.JawRight" }, + { "from": "Standard.MouthLeft", "to": "Actions.MouthLeft" }, + { "from": "Standard.MouthRight", "to": "Actions.MouthRight" }, + { "from": "Standard.MouthFrown_L", "to": "Actions.MouthFrown_L" }, + { "from": "Standard.MouthFrown_R", "to": "Actions.MouthFrown_R" }, + { "from": "Standard.MouthSmile_L", "to": "Actions.MouthSmile_L" }, + { "from": "Standard.MouthSmile_R", "to": "Actions.MouthSmile_R" }, + { "from": "Standard.MouthDimple_L", "to": "Actions.MouthDimple_L" }, + { "from": "Standard.MouthDimple_R", "to": "Actions.MouthDimple_R" }, + { "from": "Standard.LipsStretch_L", "to": "Actions.LipsStretch_L" }, + { "from": "Standard.LipsStretch_R", "to": "Actions.LipsStretch_R" }, + { "from": "Standard.LipsUpperClose", "to": "Actions.LipsUpperClose" }, + { "from": "Standard.LipsLowerClose", "to": "Actions.LipsLowerClose" }, + { "from": "Standard.LipsUpperOpen", "to": "Actions.LipsUpperOpen" }, + { "from": "Standard.LipsLowerOpen", "to": "Actions.LipsLowerOpen" }, + { "from": "Standard.LipsFunnel", "to": "Actions.LipsFunnel" }, + { "from": "Standard.LipsPucker", "to": "Actions.LipsPucker" }, + { "from": "Standard.Puff", "to": "Actions.Puff" }, + { "from": "Standard.CheekSquint_L", "to": "Actions.CheekSquint_L" }, + { "from": "Standard.CheekSquint_R", "to": "Actions.CheekSquint_R" }, + { "from": "Standard.LipsTogether", "to": "Actions.LipsTogether" }, + { "from": "Standard.MouthUpperUp_L", "to": "Actions.MouthUpperUp_L" }, + { "from": "Standard.MouthUpperUp_R", "to": "Actions.MouthUpperUp_R" }, + { "from": "Standard.MouthLowerDown_L", "to": "Actions.MouthLowerDown_L" }, + { "from": "Standard.MouthLowerDown_R", "to": "Actions.MouthLowerDown_R" }, + { "from": "Standard.MouthPress_L", "to": "Actions.MouthPress_L" }, + { "from": "Standard.MouthPress_R", "to": "Actions.MouthPress_R" }, + { "from": "Standard.MouthShrugLower", "to": "Actions.MouthShrugLower" }, + { "from": "Standard.MouthShrugUpper", "to": "Actions.MouthShrugUpper" }, + { "from": "Standard.NoseSneer_L", "to": "Actions.NoseSneer_L" }, + { "from": "Standard.NoseSneer_R", "to": "Actions.NoseSneer_R" }, + { "from": "Standard.TongueOut", "to": "Actions.TongueOut" }, + { "from": "Standard.UserBlendshape0", "to": "Actions.UserBlendshape0" }, + { "from": "Standard.UserBlendshape1", "to": "Actions.UserBlendshape1" }, + { "from": "Standard.UserBlendshape2", "to": "Actions.UserBlendshape2" }, + { "from": "Standard.UserBlendshape3", "to": "Actions.UserBlendshape3" }, + { "from": "Standard.UserBlendshape4", "to": "Actions.UserBlendshape4" }, + { "from": "Standard.UserBlendshape5", "to": "Actions.UserBlendshape5" }, + { "from": "Standard.UserBlendshape6", "to": "Actions.UserBlendshape6" }, + { "from": "Standard.UserBlendshape7", "to": "Actions.UserBlendshape7" }, + { "from": "Standard.UserBlendshape8", "to": "Actions.UserBlendshape8" }, + { "from": "Standard.UserBlendshape9", "to": "Actions.UserBlendshape9" }, { "from": "Standard.TrackedObject00", "to" : "Actions.TrackedObject00" }, { "from": "Standard.TrackedObject01", "to" : "Actions.TrackedObject01" }, diff --git a/interface/resources/controllers/vive.json b/interface/resources/controllers/vive.json index 045f72739c..4090256418 100644 --- a/interface/resources/controllers/vive.json +++ b/interface/resources/controllers/vive.json @@ -99,9 +99,8 @@ { "from": "Vive.LeftEye", "to" : "Standard.LeftEye" }, { "from": "Vive.RightEye", "to" : "Standard.RightEye" }, - // AJT: blendshapes (only keep blink) - { "from": "Vive.LeftEyeBlink", "to" : "Standard.LeftEyeBlink" }, - { "from": "Vive.RightEyeBlink", "to" : "Standard.RightEyeBlink" }, + { "from": "Vive.EyeBlink_L", "to" : "Standard.EyeBlink_L" }, + { "from": "Vive.EyeBlink_R", "to" : "Standard.EyeBlink_R" }, { "from": "Vive.LeftFoot", "to" : "Standard.LeftFoot", diff --git a/interface/src/avatar/MyAvatar.cpp b/interface/src/avatar/MyAvatar.cpp index fb42f89048..d697e1869a 100644 --- a/interface/src/avatar/MyAvatar.cpp +++ b/interface/src/avatar/MyAvatar.cpp @@ -780,18 +780,6 @@ void MyAvatar::update(float deltaTime) { emit energyChanged(currentEnergy); updateEyeContactTarget(deltaTime); - - // if we're getting eye rotations from a tracker, disable observer-side procedural eye motions - auto userInputMapper = DependencyManager::get(); - bool eyesTracked = - userInputMapper->getPoseState(controller::Action::LEFT_EYE).valid && - userInputMapper->getPoseState(controller::Action::RIGHT_EYE).valid; - - int leftEyeJointIndex = getJointIndex("LeftEye"); - int rightEyeJointIndex = getJointIndex("RightEye"); - bool eyesAreOverridden = getIsJointOverridden(leftEyeJointIndex) || getIsJointOverridden(rightEyeJointIndex); - - _headData->setHasProceduralEyeMovement(!(eyesTracked || eyesAreOverridden)); } void MyAvatar::updateEyeContactTarget(float deltaTime) { diff --git a/interface/src/avatar/MyHead.cpp b/interface/src/avatar/MyHead.cpp index 73259fe073..f92f8b7218 100644 --- a/interface/src/avatar/MyHead.cpp +++ b/interface/src/avatar/MyHead.cpp @@ -21,6 +21,73 @@ using namespace std; +static controller::Action blendshapeActions[] = { + controller::Action::EYEBLINK_L, + controller::Action::EYEBLINK_R, + controller::Action::EYESQUINT_L, + controller::Action::EYESQUINT_R, + controller::Action::EYEDOWN_L, + controller::Action::EYEDOWN_R, + controller::Action::EYEIN_L, + controller::Action::EYEIN_R, + controller::Action::EYEOPEN_L, + controller::Action::EYEOPEN_R, + controller::Action::EYEOUT_L, + controller::Action::EYEOUT_R, + controller::Action::EYEUP_L, + controller::Action::EYEUP_R, + controller::Action::BROWSD_L, + controller::Action::BROWSD_R, + controller::Action::BROWSU_C, + controller::Action::BROWSU_L, + controller::Action::BROWSU_R, + controller::Action::JAWFWD, + controller::Action::JAWLEFT, + controller::Action::JAWOPEN, + controller::Action::JAWRIGHT, + controller::Action::MOUTHLEFT, + controller::Action::MOUTHRIGHT, + controller::Action::MOUTHFROWN_L, + controller::Action::MOUTHFROWN_R, + controller::Action::MOUTHSMILE_L, + controller::Action::MOUTHSMILE_R, + controller::Action::MOUTHDIMPLE_L, + controller::Action::MOUTHDIMPLE_R, + controller::Action::LIPSSTRETCH_L, + controller::Action::LIPSSTRETCH_R, + controller::Action::LIPSUPPERCLOSE, + controller::Action::LIPSLOWERCLOSE, + controller::Action::LIPSUPPEROPEN, + controller::Action::LIPSLOWEROPEN, + controller::Action::LIPSFUNNEL, + controller::Action::LIPSPUCKER, + controller::Action::PUFF, + controller::Action::CHEEKSQUINT_L, + controller::Action::CHEEKSQUINT_R, + controller::Action::LIPSTOGETHER, + controller::Action::MOUTHUPPERUP_L, + controller::Action::MOUTHUPPERUP_R, + controller::Action::MOUTHLOWERDOWN_L, + controller::Action::MOUTHLOWERDOWN_R, + controller::Action::MOUTHPRESS_L, + controller::Action::MOUTHPRESS_R, + controller::Action::MOUTHSHRUGLOWER, + controller::Action::MOUTHSHRUGUPPER, + controller::Action::NOSESNEER_L, + controller::Action::NOSESNEER_R, + controller::Action::TONGUEOUT, + controller::Action::USERBLENDSHAPE0, + controller::Action::USERBLENDSHAPE1, + controller::Action::USERBLENDSHAPE2, + controller::Action::USERBLENDSHAPE3, + controller::Action::USERBLENDSHAPE4, + controller::Action::USERBLENDSHAPE5, + controller::Action::USERBLENDSHAPE6, + controller::Action::USERBLENDSHAPE7, + controller::Action::USERBLENDSHAPE8, + controller::Action::USERBLENDSHAPE9 +}; + MyHead::MyHead(MyAvatar* owningAvatar) : Head(owningAvatar) { } @@ -44,40 +111,57 @@ void MyHead::simulate(float deltaTime) { auto player = DependencyManager::get(); // Only use face trackers when not playing back a recording. if (!player->isPlaying()) { - // TODO -- finish removing face-tracker specific code. To do this, add input channels for - // each blendshape-coefficient and update the various json files to relay them in a useful way. - // After that, input plugins can be used to drive the avatar's face, and the various "DDE" files - // can be ported into the plugin and removed. - // - // auto faceTracker = qApp->getActiveFaceTracker(); - // const bool hasActualFaceTrackerConnected = faceTracker && !faceTracker->isMuted(); - // _isFaceTrackerConnected = hasActualFaceTrackerConnected || _owningAvatar->getHasScriptedBlendshapes(); - // if (_isFaceTrackerConnected) { - // if (hasActualFaceTrackerConnected) { - // _blendshapeCoefficients = faceTracker->getBlendshapeCoefficients(); - // } - // } - - // AJT: blendshapes auto userInputMapper = DependencyManager::get(); + + // if input system has control over blink blendshapes bool eyeLidsTracked = - userInputMapper->getActionStateValid(controller::Action::LEFT_EYE_BLINK) && - userInputMapper->getActionStateValid(controller::Action::RIGHT_EYE_BLINK); + userInputMapper->getActionStateValid(controller::Action::EYEBLINK_L) || + userInputMapper->getActionStateValid(controller::Action::EYEBLINK_R); - setHasScriptedBlendshapes(eyeLidsTracked); + // if input system has control over the brows. + bool browsTracked = + userInputMapper->getActionStateValid(controller::Action::BROWSD_L) || + userInputMapper->getActionStateValid(controller::Action::BROWSD_R) || + userInputMapper->getActionStateValid(controller::Action::BROWSU_L) || + userInputMapper->getActionStateValid(controller::Action::BROWSU_R) || + userInputMapper->getActionStateValid(controller::Action::BROWSU_C); - if (eyeLidsTracked) { - float leftEyeBlink = userInputMapper->getActionState(controller::Action::LEFT_EYE_BLINK); - float rightEyeBlink = userInputMapper->getActionState(controller::Action::RIGHT_EYE_BLINK); - _blendshapeCoefficients.resize(std::max(_blendshapeCoefficients.size(), 2)); - _blendshapeCoefficients[(int)Blendshapes::EyeBlink_L] = leftEyeBlink; - _blendshapeCoefficients[(int)Blendshapes::EyeBlink_R] = rightEyeBlink; - } else { - const float FULLY_OPEN = 0.0f; - _blendshapeCoefficients.resize(std::max(_blendshapeCoefficients.size(), 2)); - _blendshapeCoefficients[(int)Blendshapes::EyeBlink_L] = FULLY_OPEN; - _blendshapeCoefficients[(int)Blendshapes::EyeBlink_R] = FULLY_OPEN; + // if input system has control of mouth + bool mouthTracked = + userInputMapper->getActionStateValid(controller::Action::JAWOPEN) || + userInputMapper->getActionStateValid(controller::Action::LIPSUPPERCLOSE) || + userInputMapper->getActionStateValid(controller::Action::LIPSLOWERCLOSE) || + userInputMapper->getActionStateValid(controller::Action::LIPSFUNNEL) || + userInputMapper->getActionStateValid(controller::Action::MOUTHSMILE_L) || + userInputMapper->getActionStateValid(controller::Action::MOUTHSMILE_R); + + bool eyesTracked = + userInputMapper->getPoseState(controller::Action::LEFT_EYE).valid && + userInputMapper->getPoseState(controller::Action::RIGHT_EYE).valid; + + MyAvatar* myAvatar = static_cast(_owningAvatar); + int leftEyeJointIndex = myAvatar->getJointIndex("LeftEye"); + int rightEyeJointIndex = myAvatar->getJointIndex("RightEye"); + bool eyeJointsOverridden = myAvatar->getIsJointOverridden(leftEyeJointIndex) || myAvatar->getIsJointOverridden(rightEyeJointIndex); + + bool anyInputTracked = false; + for (int i = 0; i < (int)Blendshapes::BlendshapeCount; i++) { + anyInputTracked = anyInputTracked || userInputMapper->getActionStateValid(blendshapeActions[i]); + } + + setHasInputDrivenBlendshapes(anyInputTracked); + + // suppress any procedural blendshape animation if they overlap with driven input. + setSuppressProceduralAnimationFlag(HeadData::BlinkProceduralBlendshapeAnimation, eyeLidsTracked); + setSuppressProceduralAnimationFlag(HeadData::LidAdjustmentProceduralBlendshapeAnimation, eyeLidsTracked || browsTracked); + setSuppressProceduralAnimationFlag(HeadData::AudioProceduralBlendshapeAnimation, mouthTracked); + setSuppressProceduralAnimationFlag(HeadData::SaccadeProceduralEyeJointAnimation, eyesTracked || eyeJointsOverridden); + + if (anyInputTracked) { + for (int i = 0; i < (int)Blendshapes::BlendshapeCount; i++) { + _blendshapeCoefficients[i] = userInputMapper->getActionState(blendshapeActions[i]); + } } } Parent::simulate(deltaTime); diff --git a/interface/src/avatar/MySkeletonModel.cpp b/interface/src/avatar/MySkeletonModel.cpp index 8d92767321..0d7bd3f3b3 100755 --- a/interface/src/avatar/MySkeletonModel.cpp +++ b/interface/src/avatar/MySkeletonModel.cpp @@ -114,6 +114,9 @@ void MySkeletonModel::updateRig(float deltaTime, glm::mat4 parentTransform) { Head* head = _owningAvatar->getHead(); + // AJT: blendshapes TODO: RE-enable this and avoid duplication with + // SkeletonModel::updateRig() + /* bool eyePosesValid = !head->getHasProceduralEyeMovement(); glm::vec3 lookAt; if (eyePosesValid) { @@ -121,6 +124,8 @@ void MySkeletonModel::updateRig(float deltaTime, glm::mat4 parentTransform) { } else { lookAt = avoidCrossedEyes(head->getLookAtPosition()); } + */ + glm::vec3 lookAt = avoidCrossedEyes(head->getLookAtPosition()); MyAvatar* myAvatar = static_cast(_owningAvatar); assert(myAvatar); diff --git a/libraries/avatars-renderer/src/avatars-renderer/Head.cpp b/libraries/avatars-renderer/src/avatars-renderer/Head.cpp index c7f807509c..2530fea4e7 100644 --- a/libraries/avatars-renderer/src/avatars-renderer/Head.cpp +++ b/libraries/avatars-renderer/src/avatars-renderer/Head.cpp @@ -72,7 +72,8 @@ void Head::simulate(float deltaTime) { _longTermAverageLoudness = glm::mix(_longTermAverageLoudness, _averageLoudness, glm::min(deltaTime / AUDIO_LONG_TERM_AVERAGING_SECS, 1.0f)); } - if (getHasProceduralEyeMovement()) { + if (getProceduralAnimationFlag(HeadData::SaccadeProceduralEyeJointAnimation) && + !getSuppressProceduralAnimationFlag(HeadData::SaccadeProceduralEyeJointAnimation)) { // Update eye saccades const float AVERAGE_MICROSACCADE_INTERVAL = 1.0f; const float AVERAGE_SACCADE_INTERVAL = 6.0f; @@ -95,7 +96,8 @@ void Head::simulate(float deltaTime) { const float BLINK_START_VARIABILITY = 0.25f; const float FULLY_OPEN = 0.0f; const float FULLY_CLOSED = 1.0f; - if (getHasProceduralBlinkFaceMovement()) { + if (getProceduralAnimationFlag(HeadData::BlinkProceduralBlendshapeAnimation) && + !getSuppressProceduralAnimationFlag(HeadData::BlinkProceduralBlendshapeAnimation)) { // handle automatic blinks // Detect transition from talking to not; force blink after that and a delay bool forceBlink = false; @@ -146,7 +148,8 @@ void Head::simulate(float deltaTime) { } // use data to update fake Faceshift blendshape coefficients - if (getHasAudioEnabledFaceMovement()) { + if (getProceduralAnimationFlag(HeadData::AudioProceduralBlendshapeAnimation) && + !getSuppressProceduralAnimationFlag(HeadData::AudioProceduralBlendshapeAnimation)) { // Update audio attack data for facial animation (eyebrows and mouth) float audioAttackAveragingRate = (10.0f - deltaTime * NORMAL_HZ) / 10.0f; // --> 0.9 at 60 Hz _audioAttack = audioAttackAveragingRate * _audioAttack + @@ -178,7 +181,8 @@ void Head::simulate(float deltaTime) { _mouth4, _transientBlendshapeCoefficients); - if (getHasProceduralEyeFaceMovement()) { + if (getProceduralAnimationFlag(HeadData::LidAdjustmentProceduralBlendshapeAnimation) && + !getSuppressProceduralAnimationFlag(HeadData::LidAdjustmentProceduralBlendshapeAnimation)) { // This controls two things, the eye brow and the upper eye lid, it is driven by the vertical up/down angle of the // eyes relative to the head. This is to try to help prevent sleepy eyes/crazy eyes. applyEyelidOffset(getOrientation()); diff --git a/libraries/avatars-renderer/src/avatars-renderer/SkeletonModel.cpp b/libraries/avatars-renderer/src/avatars-renderer/SkeletonModel.cpp index e0fed08955..b52a68f066 100644 --- a/libraries/avatars-renderer/src/avatars-renderer/SkeletonModel.cpp +++ b/libraries/avatars-renderer/src/avatars-renderer/SkeletonModel.cpp @@ -111,6 +111,8 @@ void SkeletonModel::updateRig(float deltaTime, glm::mat4 parentTransform) { Head* head = _owningAvatar->getHead(); + // AJT: blendshapes TODO: RE-enable this. but move into rig? + /* bool eyePosesValid = !head->getHasProceduralEyeMovement(); glm::vec3 lookAt; if (eyePosesValid) { @@ -118,6 +120,8 @@ void SkeletonModel::updateRig(float deltaTime, glm::mat4 parentTransform) { } else { lookAt = avoidCrossedEyes(head->getCorrectedLookAtPosition()); } + */ + glm::vec3 lookAt = avoidCrossedEyes(head->getCorrectedLookAtPosition()); // no need to call Model::updateRig() because otherAvatars get their joint state // copied directly from AvtarData::_jointData (there are no Rig animations to blend) diff --git a/libraries/avatars/src/AvatarData.cpp b/libraries/avatars/src/AvatarData.cpp index 520241d020..c5dcfba1c0 100755 --- a/libraries/avatars/src/AvatarData.cpp +++ b/libraries/avatars/src/AvatarData.cpp @@ -110,7 +110,6 @@ AvatarData::AvatarData() : _targetScale(1.0f), _handState(0), _keyState(NO_KEY_DOWN), - _hasScriptedBlendshapes(false), _headData(NULL), _errorLogExpiry(0), _owningAvatarMixer(), @@ -156,7 +155,7 @@ float AvatarData::getDomainLimitedScale() const { void AvatarData::setHasScriptedBlendshapes(bool hasScriptedBlendshapes) { - if (hasScriptedBlendshapes == _hasScriptedBlendshapes) { + if (hasScriptedBlendshapes == _headData->getHasScriptedBlendshapes()) { return; } if (!hasScriptedBlendshapes) { @@ -165,19 +164,35 @@ void AvatarData::setHasScriptedBlendshapes(bool hasScriptedBlendshapes) { // before sending the update, or else it won't send the neutal blendshapes to the receiving clients sendAvatarDataPacket(true); } - _hasScriptedBlendshapes = hasScriptedBlendshapes; + _headData->setHasScriptedBlendshapes(hasScriptedBlendshapes); } -void AvatarData::setHasProceduralBlinkFaceMovement(bool hasProceduralBlinkFaceMovement) { - _headData->setHasProceduralBlinkFaceMovement(hasProceduralBlinkFaceMovement); +bool AvatarData::getHasScriptedBlendshapes() const { + return _headData->getHasScriptedBlendshapes(); } -void AvatarData::setHasProceduralEyeFaceMovement(bool hasProceduralEyeFaceMovement) { - _headData->setHasProceduralEyeFaceMovement(hasProceduralEyeFaceMovement); +void AvatarData::setHasProceduralBlinkFaceMovement(bool value) { + _headData->setProceduralAnimationFlag(HeadData::BlinkProceduralBlendshapeAnimation, value); } -void AvatarData::setHasAudioEnabledFaceMovement(bool hasAudioEnabledFaceMovement) { - _headData->setHasAudioEnabledFaceMovement(hasAudioEnabledFaceMovement); +bool AvatarData::getHasProceduralBlinkFaceMovement() const { + return _headData->getProceduralAnimationFlag(HeadData::BlinkProceduralBlendshapeAnimation); +} + +void AvatarData::setHasProceduralEyeFaceMovement(bool value) { + _headData->setProceduralAnimationFlag(HeadData::LidAdjustmentProceduralBlendshapeAnimation, value); +} + +bool AvatarData::getHasProceduralEyeFaceMovement() const { + return _headData->getProceduralAnimationFlag(HeadData::LidAdjustmentProceduralBlendshapeAnimation); +} + +void AvatarData::setHasAudioEnabledFaceMovement(bool value) { + _headData->setProceduralAnimationFlag(HeadData::AudioProceduralBlendshapeAnimation, value); +} + +bool AvatarData::getHasAudioEnabledFaceMovement() const { + return _headData->getProceduralAnimationFlag(HeadData::AudioProceduralBlendshapeAnimation); } void AvatarData::setDomainMinimumHeight(float domainMinimumHeight) { @@ -232,9 +247,6 @@ void AvatarData::lazyInitHeadData() const { if (!_headData) { _headData = new HeadData(const_cast(this)); } - if (_hasScriptedBlendshapes) { - _headData->_hasScriptedBlendshapes = true; - } } @@ -555,27 +567,31 @@ QByteArray AvatarData::toByteArray(AvatarDataDetail dataDetail, quint64 lastSent setAtBit16(flags, HAND_STATE_FINGER_POINTING_BIT); } // face tracker state - if (_headData->_hasScriptedBlendshapes) { + if (_headData->_hasScriptedBlendshapes || _headData->_hasInputDrivenBlendshapes) { setAtBit16(flags, HAS_SCRIPTED_BLENDSHAPES); } // eye tracker state - if (!_headData->_hasProceduralEyeMovement) { - setAtBit16(flags, IS_EYE_TRACKER_CONNECTED); + if (_headData->getProceduralAnimationFlag(HeadData::SaccadeProceduralEyeJointAnimation) && + !_headData->getSuppressProceduralAnimationFlag(HeadData::SaccadeProceduralEyeJointAnimation)) { + setAtBit16(flags, HAS_PROCEDURAL_EYE_MOVEMENT); } // referential state if (!parentID.isNull()) { setAtBit16(flags, HAS_REFERENTIAL); } // audio face movement - if (_headData->getHasAudioEnabledFaceMovement()) { + if (_headData->getProceduralAnimationFlag(HeadData::AudioProceduralBlendshapeAnimation) && + !_headData->getSuppressProceduralAnimationFlag(HeadData::AudioProceduralBlendshapeAnimation)) { setAtBit16(flags, AUDIO_ENABLED_FACE_MOVEMENT); } // procedural eye face movement - if (_headData->getHasProceduralEyeFaceMovement()) { + if (_headData->getProceduralAnimationFlag(HeadData::LidAdjustmentProceduralBlendshapeAnimation) && + !_headData->getSuppressProceduralAnimationFlag(HeadData::LidAdjustmentProceduralBlendshapeAnimation)) { setAtBit16(flags, PROCEDURAL_EYE_FACE_MOVEMENT); } // procedural blink face movement - if (_headData->getHasProceduralBlinkFaceMovement()) { + if (_headData->getProceduralAnimationFlag(HeadData::BlinkProceduralBlendshapeAnimation) && + !_headData->getSuppressProceduralAnimationFlag(HeadData::BlinkProceduralBlendshapeAnimation)) { setAtBit16(flags, PROCEDURAL_BLINK_FACE_MOVEMENT); } // avatar collisions enabled @@ -1177,21 +1193,22 @@ int AvatarData::parseDataFromBuffer(const QByteArray& buffer) { + (oneAtBit16(bitItems, HAND_STATE_FINGER_POINTING_BIT) ? IS_FINGER_POINTING_FLAG : 0); auto newHasScriptedBlendshapes = oneAtBit16(bitItems, HAS_SCRIPTED_BLENDSHAPES); - auto newHasntProceduralEyeMovement = oneAtBit16(bitItems, IS_EYE_TRACKER_CONNECTED); - + auto newHasProceduralEyeMovement = oneAtBit16(bitItems, HAS_PROCEDURAL_EYE_MOVEMENT); auto newHasAudioEnabledFaceMovement = oneAtBit16(bitItems, AUDIO_ENABLED_FACE_MOVEMENT); auto newHasProceduralEyeFaceMovement = oneAtBit16(bitItems, PROCEDURAL_EYE_FACE_MOVEMENT); auto newHasProceduralBlinkFaceMovement = oneAtBit16(bitItems, PROCEDURAL_BLINK_FACE_MOVEMENT); + auto newCollideWithOtherAvatars = oneAtBit16(bitItems, COLLIDE_WITH_OTHER_AVATARS); auto newHasPriority = oneAtBit16(bitItems, HAS_HERO_PRIORITY); bool keyStateChanged = (_keyState != newKeyState); bool handStateChanged = (_handState != newHandState); - bool faceStateChanged = (_headData->_hasScriptedBlendshapes != newHasScriptedBlendshapes); - bool eyeStateChanged = (_headData->_hasProceduralEyeMovement == newHasntProceduralEyeMovement); - bool audioEnableFaceMovementChanged = (_headData->getHasAudioEnabledFaceMovement() != newHasAudioEnabledFaceMovement); - bool proceduralEyeFaceMovementChanged = (_headData->getHasProceduralEyeFaceMovement() != newHasProceduralEyeFaceMovement); - bool proceduralBlinkFaceMovementChanged = (_headData->getHasProceduralBlinkFaceMovement() != newHasProceduralBlinkFaceMovement); + bool faceStateChanged = (_headData->getHasScriptedBlendshapes() != newHasScriptedBlendshapes); + + bool eyeStateChanged = (_headData->getProceduralAnimationFlag(HeadData::SaccadeProceduralEyeJointAnimation) != newHasProceduralEyeMovement); + bool audioEnableFaceMovementChanged = (_headData->getProceduralAnimationFlag(HeadData::AudioProceduralBlendshapeAnimation) != newHasAudioEnabledFaceMovement); + bool proceduralEyeFaceMovementChanged = (_headData->getProceduralAnimationFlag(HeadData::LidAdjustmentProceduralBlendshapeAnimation) != newHasProceduralEyeFaceMovement); + bool proceduralBlinkFaceMovementChanged = (_headData->getProceduralAnimationFlag(HeadData::BlinkProceduralBlendshapeAnimation) != newHasProceduralBlinkFaceMovement); bool collideWithOtherAvatarsChanged = (_collideWithOtherAvatars != newCollideWithOtherAvatars); bool hasPriorityChanged = (getHasPriority() != newHasPriority); bool somethingChanged = keyStateChanged || handStateChanged || faceStateChanged || eyeStateChanged || audioEnableFaceMovementChanged || @@ -1200,11 +1217,11 @@ int AvatarData::parseDataFromBuffer(const QByteArray& buffer) { _keyState = newKeyState; _handState = newHandState; - _headData->_hasScriptedBlendshapes = newHasScriptedBlendshapes; - _headData->setHasProceduralEyeMovement(!newHasntProceduralEyeMovement); - _headData->setHasAudioEnabledFaceMovement(newHasAudioEnabledFaceMovement); - _headData->setHasProceduralEyeFaceMovement(newHasProceduralEyeFaceMovement); - _headData->setHasProceduralBlinkFaceMovement(newHasProceduralBlinkFaceMovement); + _headData->setHasScriptedBlendshapes(newHasScriptedBlendshapes); + _headData->setProceduralAnimationFlag(HeadData::SaccadeProceduralEyeJointAnimation, newHasProceduralEyeMovement); + _headData->setProceduralAnimationFlag(HeadData::AudioProceduralBlendshapeAnimation, newHasAudioEnabledFaceMovement); + _headData->setProceduralAnimationFlag(HeadData::LidAdjustmentProceduralBlendshapeAnimation, newHasProceduralEyeFaceMovement); + _headData->setProceduralAnimationFlag(HeadData::BlinkProceduralBlendshapeAnimation, newHasProceduralBlinkFaceMovement); _collideWithOtherAvatars = newCollideWithOtherAvatars; setHasPriorityWithoutTimestampReset(newHasPriority); @@ -1289,7 +1306,7 @@ int AvatarData::parseDataFromBuffer(const QByteArray& buffer) { sourceBuffer += sizeof(AvatarDataPacket::FaceTrackerInfo); PACKET_READ_CHECK(FaceTrackerCoefficients, coefficientsSize); - _headData->_blendshapeCoefficients.resize(numCoefficients); // make sure there's room for the copy! + _headData->_blendshapeCoefficients.resize(std::min(numCoefficients, (int)Blendshapes::BlendshapeCount)); // make sure there's room for the copy! //only copy the blendshapes to headData, not the procedural face info memcpy(_headData->_blendshapeCoefficients.data(), sourceBuffer, coefficientsSize); sourceBuffer += coefficientsSize; diff --git a/libraries/avatars/src/AvatarData.h b/libraries/avatars/src/AvatarData.h index 6bed5fa9db..53903412d0 100755 --- a/libraries/avatars/src/AvatarData.h +++ b/libraries/avatars/src/AvatarData.h @@ -107,7 +107,7 @@ const quint32 AVATAR_MOTION_SCRIPTABLE_BITS = const int KEY_STATE_START_BIT = 0; // 1st and 2nd bits (UNUSED) const int HAND_STATE_START_BIT = 2; // 3rd and 4th bits (UNUSED) const int HAS_SCRIPTED_BLENDSHAPES = 4; // 5th bit -const int IS_EYE_TRACKER_CONNECTED = 5; // 6th bit (was CHAT_CIRCLING) +const int HAS_PROCEDURAL_EYE_MOVEMENT = 5; // 6th bit const int HAS_REFERENTIAL = 6; // 7th bit const int HAND_STATE_FINGER_POINTING_BIT = 7; // 8th bit (UNUSED) const int AUDIO_ENABLED_FACE_MOVEMENT = 8; // 9th bit @@ -703,13 +703,13 @@ public: float getDomainLimitedScale() const; void setHasScriptedBlendshapes(bool hasScriptedBlendshapes); - bool getHasScriptedBlendshapes() const { return _hasScriptedBlendshapes; } + bool getHasScriptedBlendshapes() const; void setHasProceduralBlinkFaceMovement(bool hasProceduralBlinkFaceMovement); - bool getHasProceduralBlinkFaceMovement() const { return _headData->getHasProceduralBlinkFaceMovement(); } + bool getHasProceduralBlinkFaceMovement() const; void setHasProceduralEyeFaceMovement(bool hasProceduralEyeFaceMovement); - bool getHasProceduralEyeFaceMovement() const { return _headData->getHasProceduralEyeFaceMovement(); } + bool getHasProceduralEyeFaceMovement() const; void setHasAudioEnabledFaceMovement(bool hasAudioEnabledFaceMovement); - bool getHasAudioEnabledFaceMovement() const { return _headData->getHasAudioEnabledFaceMovement(); } + bool getHasAudioEnabledFaceMovement() const; /**jsdoc * Gets the minimum scale allowed for this avatar in the current domain. @@ -1716,7 +1716,6 @@ protected: // key state KeyState _keyState; - bool _hasScriptedBlendshapes; bool _hasNewJointData { true }; // set in AvatarData, cleared in Avatar mutable HeadData* _headData { nullptr }; diff --git a/libraries/avatars/src/HeadData.cpp b/libraries/avatars/src/HeadData.cpp index 561f2c798a..f37b88c135 100644 --- a/libraries/avatars/src/HeadData.cpp +++ b/libraries/avatars/src/HeadData.cpp @@ -27,11 +27,10 @@ HeadData::HeadData(AvatarData* owningAvatar) : _basePitch(0.0f), _baseRoll(0.0f), _lookAtPosition(0.0f, 0.0f, 0.0f), - _blendshapeCoefficients(QVector(0, 0.0f)), - _transientBlendshapeCoefficients(QVector(0, 0.0f)), - _summedBlendshapeCoefficients(QVector(0, 0.0f)), _owningAvatar(owningAvatar) { + _userProceduralAnimationFlags.assign((size_t)ProceduralAnimaitonTypeCount, true); + _suppressProceduralAnimationFlags.assign((size_t)ProceduralAnimaitonTypeCount, false); computeBlendshapesLookupMap(); } @@ -102,7 +101,7 @@ const QVector& HeadData::getSummedBlendshapeCoefficients() { void HeadData::setBlendshape(QString name, float val) { - //Check to see if the named blendshape exists, and then set its value if it does + // Check to see if the named blendshape exists, and then set its value if it does auto it = _blendshapeLookupMap.find(name); if (it != _blendshapeLookupMap.end()) { if (_blendshapeCoefficients.size() <= it.value()) { @@ -112,6 +111,19 @@ void HeadData::setBlendshape(QString name, float val) { _transientBlendshapeCoefficients.resize(it.value() + 1); } _blendshapeCoefficients[it.value()] = val; + } else { + // check to see if this is a legacy blendshape that is present in + // ARKit blendshapes but is split. i.e. has left and right halfs. + if (name == "LipsUpperUp") { + _blendshapeCoefficients[(int)Blendshapes::MouthUpperUp_L] = val; + _blendshapeCoefficients[(int)Blendshapes::MouthUpperUp_R] = val; + } else if (name == "LipsLowerDown") { + _blendshapeCoefficients[(int)Blendshapes::MouthLowerDown_L] = val; + _blendshapeCoefficients[(int)Blendshapes::MouthLowerDown_R] = val; + } else if (name == "Sneer") { + _blendshapeCoefficients[(int)Blendshapes::NoseSneer_L] = val; + _blendshapeCoefficients[(int)Blendshapes::NoseSneer_R] = val; + } } } @@ -197,38 +209,34 @@ void HeadData::fromJson(const QJsonObject& json) { } } -bool HeadData::getHasProceduralEyeFaceMovement() const { - return _hasProceduralEyeFaceMovement; +bool HeadData::getProceduralAnimationFlag(ProceduralAnimationType type) const { + return _userProceduralAnimationFlags[(int)type]; } -void HeadData::setHasProceduralEyeFaceMovement(bool hasProceduralEyeFaceMovement) { - _hasProceduralEyeFaceMovement = hasProceduralEyeFaceMovement; +void HeadData::setProceduralAnimationFlag(ProceduralAnimationType type, bool value) { + _userProceduralAnimationFlags[(int)type] = value; } -bool HeadData::getHasProceduralBlinkFaceMovement() const { - return _hasProceduralBlinkFaceMovement; +bool HeadData::getSuppressProceduralAnimationFlag(ProceduralAnimationType type) const { + return _suppressProceduralAnimationFlags[(int)type]; } -void HeadData::setHasProceduralBlinkFaceMovement(bool hasProceduralBlinkFaceMovement) { - _hasProceduralBlinkFaceMovement = hasProceduralBlinkFaceMovement; +void HeadData::setSuppressProceduralAnimationFlag(ProceduralAnimationType type, bool value) { + _suppressProceduralAnimationFlags[(int)type] = value; } -bool HeadData::getHasAudioEnabledFaceMovement() const { - return _hasAudioEnabledFaceMovement; -} - -void HeadData::setHasAudioEnabledFaceMovement(bool hasAudioEnabledFaceMovement) { - _hasAudioEnabledFaceMovement = hasAudioEnabledFaceMovement; -} - -bool HeadData::getHasProceduralEyeMovement() const { - return _hasProceduralEyeMovement; -} - -void HeadData::setHasProceduralEyeMovement(bool hasProceduralEyeMovement) { - _hasProceduralEyeMovement = hasProceduralEyeMovement; +bool HeadData::getHasScriptedBlendshapes() const { + return _hasScriptedBlendshapes; } void HeadData::setHasScriptedBlendshapes(bool value) { _hasScriptedBlendshapes = value; } + +bool HeadData::getHasInputDrivenBlendshapes() const { + return _hasInputDrivenBlendshapes; +} + +void HeadData::setHasInputDrivenBlendshapes(bool value) { + _hasInputDrivenBlendshapes = value; +} diff --git a/libraries/avatars/src/HeadData.h b/libraries/avatars/src/HeadData.h index c6bcc2599b..9652792512 100644 --- a/libraries/avatars/src/HeadData.h +++ b/libraries/avatars/src/HeadData.h @@ -72,17 +72,29 @@ public: } bool lookAtPositionChangedSince(quint64 time) { return _lookAtPositionChanged >= time; } - bool getHasProceduralEyeFaceMovement() const; - void setHasProceduralEyeFaceMovement(bool hasProceduralEyeFaceMovement); - bool getHasProceduralBlinkFaceMovement() const; - void setHasProceduralBlinkFaceMovement(bool hasProceduralBlinkFaceMovement); - bool getHasAudioEnabledFaceMovement() const; - void setHasAudioEnabledFaceMovement(bool hasAudioEnabledFaceMovement); - bool getHasProceduralEyeMovement() const; - void setHasProceduralEyeMovement(bool hasProceduralEyeMovement); + enum ProceduralAnimationType { + AudioProceduralBlendshapeAnimation = 0, + BlinkProceduralBlendshapeAnimation, + LidAdjustmentProceduralBlendshapeAnimation, + SaccadeProceduralEyeJointAnimation, + ProceduralAnimaitonTypeCount, + }; + // called by scripts to enable or disable procedural blendshape or eye joint animations. + bool getProceduralAnimationFlag(ProceduralAnimationType type) const; + void setProceduralAnimationFlag(ProceduralAnimationType type, bool value); + + // called by c++ to suppress, i.e. temporarily disable a procedural animation. + bool getSuppressProceduralAnimationFlag(ProceduralAnimationType flag) const; + void setSuppressProceduralAnimationFlag(ProceduralAnimationType flag, bool value); + + // called by scripts to enable/disable manual adjustment of blendshapes void setHasScriptedBlendshapes(bool value); - bool getHasScriptedBlendshapes() const { return _hasScriptedBlendshapes; } + bool getHasScriptedBlendshapes() const; + + // called by C++ code to denote the presence of manually driven blendshapes. + void setHasInputDrivenBlendshapes(bool value); + bool getHasInputDrivenBlendshapes() const; friend class AvatarData; @@ -98,21 +110,20 @@ protected: glm::vec3 _lookAtPosition; quint64 _lookAtPositionChanged { 0 }; - bool _hasAudioEnabledFaceMovement { true }; - bool _hasProceduralBlinkFaceMovement { true }; - bool _hasProceduralEyeFaceMovement { true }; - bool _hasProceduralEyeMovement { true }; + std::vector _userProceduralAnimationFlags; + std::vector _suppressProceduralAnimationFlags; bool _hasScriptedBlendshapes { false }; + bool _hasInputDrivenBlendshapes { false }; float _leftEyeBlink { 0.0f }; float _rightEyeBlink { 0.0f }; float _averageLoudness { 0.0f }; float _browAudioLift { 0.0f }; - QVector _blendshapeCoefficients; - QVector _transientBlendshapeCoefficients; - QVector _summedBlendshapeCoefficients; + QVector _blendshapeCoefficients { (int)Blendshapes::BlendshapeCount, 0.0f }; + QVector _transientBlendshapeCoefficients { (int)Blendshapes::BlendshapeCount, 0.0f }; + QVector _summedBlendshapeCoefficients { (int)Blendshapes::BlendshapeCount, 0.0f }; QMap _blendshapeLookupMap; AvatarData* _owningAvatar; diff --git a/libraries/controllers/src/controllers/Actions.h b/libraries/controllers/src/controllers/Actions.h index e1ad17aafa..1a5c73a1d8 100644 --- a/libraries/controllers/src/controllers/Actions.h +++ b/libraries/controllers/src/controllers/Actions.h @@ -185,70 +185,70 @@ enum class Action { RIGHT_EYE, // AJT: blendshapes - EyeBlink_L, - EyeBlink_R, - EyeSquint_L, - EyeSquint_R, - EyeDown_L, - EyeDown_R, - EyeIn_L, - EyeIn_R, - EyeOpen_L, - EyeOpen_R, - EyeOut_L, - EyeOut_R, - EyeUp_L, - EyeUp_R, - BrowsD_L, - BrowsD_R, - BrowsU_C, - BrowsU_L, - BrowsU_R, - JawFwd, - JawLeft, - JawOpen, - JawRight, - MouthLeft, - MouthRight, - MouthFrown_L, - MouthFrown_R, - MouthSmile_L, - MouthSmile_R, - MouthDimple_L, - MouthDimple_R, - LipsStretch_L, - LipsStretch_R, - LipsUpperClose, - LipsLowerClose, - LipsUpperOpen, - LipsLowerOpen, - LipsFunnel, - LipsPucker, - Puff, - CheekSquint_L, - CheekSquint_R, - LipsTogether, - MouthUpperUp_L, - MouthUpperUp_R, - MouthLowerDown_L, - MouthLowerDown_R, - MouthPress_L, - MouthPress_R, - MouthShrugLower, - MouthShrugUpper, - NoseSneer_L, - NoseSneer_R, - TongueOut, - UserBlendshape0, - UserBlendshape1, - UserBlendshape2, - UserBlendshape3, - UserBlendshape4, - UserBlendshape5, - UserBlendshape6, - UserBlendshape7, - UserBlendshape8, - UserBlendshape9, + EYEBLINK_L, + EYEBLINK_R, + EYESQUINT_L, + EYESQUINT_R, + EYEDOWN_L, + EYEDOWN_R, + EYEIN_L, + EYEIN_R, + EYEOPEN_L, + EYEOPEN_R, + EYEOUT_L, + EYEOUT_R, + EYEUP_L, + EYEUP_R, + BROWSD_L, + BROWSD_R, + BROWSU_C, + BROWSU_L, + BROWSU_R, + JAWFWD, + JAWLEFT, + JAWOPEN, + JAWRIGHT, + MOUTHLEFT, + MOUTHRIGHT, + MOUTHFROWN_L, + MOUTHFROWN_R, + MOUTHSMILE_L, + MOUTHSMILE_R, + MOUTHDIMPLE_L, + MOUTHDIMPLE_R, + LIPSSTRETCH_L, + LIPSSTRETCH_R, + LIPSUPPERCLOSE, + LIPSLOWERCLOSE, + LIPSUPPEROPEN, + LIPSLOWEROPEN, + LIPSFUNNEL, + LIPSPUCKER, + PUFF, + CHEEKSQUINT_L, + CHEEKSQUINT_R, + LIPSTOGETHER, + MOUTHUPPERUP_L, + MOUTHUPPERUP_R, + MOUTHLOWERDOWN_L, + MOUTHLOWERDOWN_R, + MOUTHPRESS_L, + MOUTHPRESS_R, + MOUTHSHRUGLOWER, + MOUTHSHRUGUPPER, + NOSESNEER_L, + NOSESNEER_R, + TONGUEOUT, + USERBLENDSHAPE0, + USERBLENDSHAPE1, + USERBLENDSHAPE2, + USERBLENDSHAPE3, + USERBLENDSHAPE4, + USERBLENDSHAPE5, + USERBLENDSHAPE6, + USERBLENDSHAPE7, + USERBLENDSHAPE8, + USERBLENDSHAPE9, NUM_ACTIONS }; diff --git a/libraries/controllers/src/controllers/StandardControls.h b/libraries/controllers/src/controllers/StandardControls.h index 2af822ffc2..55bcf0e36a 100644 --- a/libraries/controllers/src/controllers/StandardControls.h +++ b/libraries/controllers/src/controllers/StandardControls.h @@ -92,8 +92,70 @@ namespace controller { RIGHT_GRIP, // AJT: blendshapes - LEFT_EYE_BLINK, - RIGHT_EYE_BLINK, + EYEBLINK_L, + EYEBLINK_R, + EYESQUINT_L, + EYESQUINT_R, + EYEDOWN_L, + EYEDOWN_R, + EYEIN_L, + EYEIN_R, + EYEOPEN_L, + EYEOPEN_R, + EYEOUT_L, + EYEOUT_R, + EYEUP_L, + EYEUP_R, + BROWSD_L, + BROWSD_R, + BROWSU_C, + BROWSU_L, + BROWSU_R, + JAWFWD, + JAWLEFT, + JAWOPEN, + JAWRIGHT, + MOUTHLEFT, + MOUTHRIGHT, + MOUTHFROWN_L, + MOUTHFROWN_R, + MOUTHSMILE_L, + MOUTHSMILE_R, + MOUTHDIMPLE_L, + MOUTHDIMPLE_R, + LIPSSTRETCH_L, + LIPSSTRETCH_R, + LIPSUPPERCLOSE, + LIPSLOWERCLOSE, + LIPSUPPEROPEN, + LIPSLOWEROPEN, + LIPSFUNNEL, + LIPSPUCKER, + PUFF, + CHEEKSQUINT_L, + CHEEKSQUINT_R, + LIPSTOGETHER, + MOUTHUPPERUP_L, + MOUTHUPPERUP_R, + MOUTHLOWERDOWN_L, + MOUTHLOWERDOWN_R, + MOUTHPRESS_L, + MOUTHPRESS_R, + MOUTHSHRUGLOWER, + MOUTHSHRUGUPPER, + NOSESNEER_L, + NOSESNEER_R, + TONGUEOUT, + USERBLENDSHAPE0, + USERBLENDSHAPE1, + USERBLENDSHAPE2, + USERBLENDSHAPE3, + USERBLENDSHAPE4, + USERBLENDSHAPE5, + USERBLENDSHAPE6, + USERBLENDSHAPE7, + USERBLENDSHAPE8, + USERBLENDSHAPE9, NUM_STANDARD_AXES, LZ = LT, diff --git a/libraries/networking/src/udt/PacketHeaders.cpp b/libraries/networking/src/udt/PacketHeaders.cpp index ed68fe89dc..87bd7941d3 100644 --- a/libraries/networking/src/udt/PacketHeaders.cpp +++ b/libraries/networking/src/udt/PacketHeaders.cpp @@ -38,10 +38,10 @@ PacketVersion versionForPacketType(PacketType packetType) { return static_cast(EntityQueryPacketVersion::ConicalFrustums); case PacketType::AvatarIdentity: case PacketType::AvatarData: - return static_cast(AvatarMixerPacketVersion::SendVerificationFailed); + return static_cast(AvatarMixerPacketVersion::ARKitBlendshapes); case PacketType::BulkAvatarData: case PacketType::KillAvatar: - return static_cast(AvatarMixerPacketVersion::SendVerificationFailed); + return static_cast(AvatarMixerPacketVersion::ARKitBlendshapes); case PacketType::MessagesData: return static_cast(MessageDataVersion::TextOrBinaryData); // ICE packets diff --git a/libraries/networking/src/udt/PacketHeaders.h b/libraries/networking/src/udt/PacketHeaders.h index 52a8c8db16..fbf575065e 100644 --- a/libraries/networking/src/udt/PacketHeaders.h +++ b/libraries/networking/src/udt/PacketHeaders.h @@ -339,7 +339,8 @@ enum class AvatarMixerPacketVersion : PacketVersion { SendMaxTranslationDimension, FBXJointOrderChange, HandControllerSection, - SendVerificationFailed + SendVerificationFailed, + ARKitBlendshapes }; enum class DomainConnectRequestVersion : PacketVersion { diff --git a/libraries/shared/src/BlendshapeConstants.h b/libraries/shared/src/BlendshapeConstants.h index 97b459fa0e..63e99f465f 100644 --- a/libraries/shared/src/BlendshapeConstants.h +++ b/libraries/shared/src/BlendshapeConstants.h @@ -91,7 +91,7 @@ enum class LegacyBlendshpaes : int { ChinUpperRaise, // not in ARKit Sneer, // split in ARKit LegacyBlendshapeCount -} +}; // NEW in ARKit // * LipsTogether