Mirrors/overte-HifiExperiments
3
0
Fork 0
mirror of https://github.com/HifiExperiments/overte.git synced 2025-04-07 10:02:24 +02:00
Code Issues Projects Releases Packages Wiki Activity Actions

WIP adding blendshapes to input system

Browse source
This commit is contained in:
Anthony J. Thibault 2019-10-11 09:37:01 -07:00
parent da0911e01b
commit 6efd74a339
11 changed files with 300 additions and 64 deletions
Show stats Download patch file Download diff file Expand all files Collapse all files

1
interface/resources/controllers/standard.json
View file

@ -166,6 +166,7 @@
{ "from": "Standard.LeftEye", "to": "Actions.LeftEye" },
{ "from": "Standard.RightEye", "to": "Actions.RightEye" },
// AJT: blendshapes
{ "from": "Standard.LeftEyeBlink", "to": "Actions.LeftEyeBlink" },
{ "from": "Standard.RightEyeBlink", "to": "Actions.RightEyeBlink" },

1
interface/resources/controllers/standard_nomovement.json
View file

@ -61,6 +61,7 @@
{ "from": "Standard.LeftEye", "to": "Actions.LeftEye" },
{ "from": "Standard.RightEye", "to": "Actions.RightEye" },
// AJT: blendshapes
{ "from": "Standard.LeftEyeBlink", "to": "Actions.LeftEyeBlink" },
{ "from": "Standard.RightEyeBlink", "to": "Actions.RightEyeBlink" },

2
interface/resources/controllers/vive.json
View file

@ -98,6 +98,8 @@
{ "from": "Vive.Head", "to" : "Standard.Head" },
{ "from": "Vive.LeftEye", "to" : "Standard.LeftEye" },
{ "from": "Vive.RightEye", "to" : "Standard.RightEye" },
// AJT: blendshapes (only keep blink)
{ "from": "Vive.LeftEyeBlink", "to" : "Standard.LeftEyeBlink" },
{ "from": "Vive.RightEyeBlink", "to" : "Standard.RightEyeBlink" },

10
interface/src/avatar/MyHead.cpp
View file

@ -58,6 +58,8 @@ void MyHead::simulate(float deltaTime) {
// }
// }
// AJT: blendshapes
auto userInputMapper = DependencyManager::get<UserInputMapper>();
bool eyeLidsTracked =
userInputMapper->getActionStateValid(controller::Action::LEFT_EYE_BLINK) &&
@ -69,13 +71,13 @@ void MyHead::simulate(float deltaTime) {
float leftEyeBlink = userInputMapper->getActionState(controller::Action::LEFT_EYE_BLINK);
float rightEyeBlink = userInputMapper->getActionState(controller::Action::RIGHT_EYE_BLINK);
_blendshapeCoefficients.resize(std::max(_blendshapeCoefficients.size(), 2));
_blendshapeCoefficients[EYE_BLINK_INDICES[0]] = leftEyeBlink;
_blendshapeCoefficients[EYE_BLINK_INDICES[1]] = rightEyeBlink;
_blendshapeCoefficients[(int)Blendshapes::EyeBlink_L] = leftEyeBlink;
_blendshapeCoefficients[(int)Blendshapes::EyeBlink_R] = rightEyeBlink;
} else {
const float FULLY_OPEN = 0.0f;
_blendshapeCoefficients.resize(std::max(_blendshapeCoefficients.size(), 2));
_blendshapeCoefficients[EYE_BLINK_INDICES[0]] = FULLY_OPEN;
_blendshapeCoefficients[EYE_BLINK_INDICES[1]] = FULLY_OPEN;
_blendshapeCoefficients[(int)Blendshapes::EyeBlink_L] = FULLY_OPEN;
_blendshapeCoefficients[(int)Blendshapes::EyeBlink_R] = FULLY_OPEN;
}
}
Parent::simulate(deltaTime);

18
libraries/avatars-renderer/src/avatars-renderer/Head.cpp
View file

@ -262,26 +262,26 @@ void Head::applyEyelidOffset(glm::quat headOrientation) {
float blinkUpCoefficient = -eyelidOffset;
float blinkDownCoefficient = BLINK_DOWN_MULTIPLIER * eyelidOffset;
float openUpCoefficient = eyelidOffset;
float openDownCoefficient = OPEN_DOWN_MULTIPLIER * eyelidOffset;
float browsUpCoefficient = BROW_UP_MULTIPLIER * eyelidOffset;
float browsDownCoefficient = 0.0f;
bool isLookingUp = (eyePitch > 0);
if (isLookingUp) {
for (int i = 0; i < 2; i++) {
_transientBlendshapeCoefficients[EYE_BLINK_INDICES[i]] = blinkUpCoefficient;
_transientBlendshapeCoefficients[EYE_OPEN_INDICES[i]] = openUpCoefficient;
_transientBlendshapeCoefficients[BROWS_U_INDICES[i]] = browsUpCoefficient;
_transientBlendshapeCoefficients[(int)Blendshapes::EyeBlink_L + i] = blinkUpCoefficient;
_transientBlendshapeCoefficients[(int)Blendshapes::EyeOpen_L + i] = openUpCoefficient;
_transientBlendshapeCoefficients[(int)Blendshapes::BrowsU_L + i] = browsUpCoefficient;
}
} else {
for (int i = 0; i < 2; i++) {
_transientBlendshapeCoefficients[EYE_BLINK_INDICES[i]] = blinkDownCoefficient;
_transientBlendshapeCoefficients[EYE_OPEN_INDICES[i]] = openDownCoefficient;
_transientBlendshapeCoefficients[BROWS_U_INDICES[i]] = browsDownCoefficient;
_transientBlendshapeCoefficients[(int)Blendshapes::EyeBlink_L + i] = blinkDownCoefficient;
_transientBlendshapeCoefficients[(int)Blendshapes::EyeOpen_L + i] = openDownCoefficient;
_transientBlendshapeCoefficients[(int)Blendshapes::BrowsU_L + i] = browsDownCoefficient;
}
}
}

68
libraries/controllers/src/controllers/Actions.cpp
View file

@ -349,8 +349,72 @@ namespace controller {
makePosePair(Action::HEAD, "Head"),
makePosePair(Action::LEFT_EYE, "LeftEye"),
makePosePair(Action::RIGHT_EYE, "RightEye"),
makeAxisPair(Action::LEFT_EYE_BLINK, "LeftEyeBlink"),
makeAxisPair(Action::RIGHT_EYE_BLINK, "RightEyeBlink"),
// AJT: blendshapes
makeAxisPair(Action::EYEBLINK_L, "EyeBlink_L"),
makeAxisPair(Action::EYEBLINK_R, "EyeBlink_R"),
makeAxisPair(Action::EYESQUINT_L, "EyeSquint_L"),
makeAxisPair(Action::EYESQUINT_R, "EyeSquint_R"),
makeAxisPair(Action::EYEDOWN_L, "EyeDown_L"),
makeAxisPair(Action::EYEDOWN_R, "EyeDown_R"),
makeAxisPair(Action::EYEIN_L, "EyeIn_L"),
makeAxisPair(Action::EYEIN_R, "EyeIn_R"),
makeAxisPair(Action::EYEOPEN_L, "EyeOpen_L"),
makeAxisPair(Action::EYEOPEN_R, "EyeOpen_R"),
makeAxisPair(Action::EYEOUT_L, "EyeOut_L"),
makeAxisPair(Action::EYEOUT_R, "EyeOut_R"),
makeAxisPair(Action::EYEUP_L, "EyeUp_L"),
makeAxisPair(Action::EYEUP_R, "EyeUp_R"),
makeAxisPair(Action::BROWSD_L, "BrowsD_L"),
makeAxisPair(Action::BROWSD_R, "BrowsD_R"),
makeAxisPair(Action::BROWSU_C, "BrowsU_C"),
makeAxisPair(Action::BROWSU_L, "BrowsU_L"),
makeAxisPair(Action::BROWSU_R, "BrowsU_R"),
makeAxisPair(Action::JAWFWD, "JawFwd"),
makeAxisPair(Action::JAWLEFT, "JawLeft"),
makeAxisPair(Action::JAWOPEN, "JawOpen"),
makeAxisPair(Action::JAWRIGHT, "JawRight"),
makeAxisPair(Action::MOUTHLEFT, "MouthLeft"),
makeAxisPair(Action::MOUTHRIGHT, "MouthRight"),
makeAxisPair(Action::MOUTHFROWN_L, "MouthFrown_L"),
makeAxisPair(Action::MOUTHFROWN_R, "MouthFrown_R"),
makeAxisPair(Action::MOUTHSMILE_L, "MouthSmile_L"),
makeAxisPair(Action::MOUTHSMILE_R, "MouthSmile_R"),
makeAxisPair(Action::MOUTHDIMPLE_L, "MouthDimple_L"),
makeAxisPair(Action::MOUTHDIMPLE_R, "MouthDimple_R"),
makeAxisPair(Action::LIPSSTRETCH_L, "LipsStretch_L"),
makeAxisPair(Action::LIPSSTRETCH_R, "LipsStretch_R"),
makeAxisPair(Action::LIPSUPPERCLOSE, "LipsUpperClose"),
makeAxisPair(Action::LIPSLOWERCLOSE, "LipsLowerClose"),
makeAxisPair(Action::LIPSUPPEROPEN, "LipsUpperOpen"),
makeAxisPair(Action::LIPSLOWEROPEN, "LipsLowerOpen"),
makeAxisPair(Action::LIPSFUNNEL, "LipsFunnel"),
makeAxisPair(Action::LIPSPUCKER, "LipsPucker"),
makeAxisPair(Action::PUFF, "Puff"),
makeAxisPair(Action::CHEEKSQUINT_L, "CheekSquint_L"),
makeAxisPair(Action::CHEEKSQUINT_R, "CheekSquint_R"),
makeAxisPair(Action::LIPSTOGETHER, "LipsTogether"),
makeAxisPair(Action::MOUTHUPPERUP_L, "MouthUpperUp_L"),
makeAxisPair(Action::MOUTHUPPERUP_R, "MouthUpperUp_R"),
makeAxisPair(Action::MOUTHLOWERDOWN_L, "MouthLowerDown_L"),
makeAxisPair(Action::MOUTHLOWERDOWN_R, "MouthLowerDown_R"),
makeAxisPair(Action::MOUTHPRESS_L, "MouthPress_L"),
makeAxisPair(Action::MOUTHPRESS_R, "MouthPress_R"),
makeAxisPair(Action::MOUTHSHRUGLOWER, "MouthShrugLower"),
makeAxisPair(Action::MOUTHSHRUGUPPER, "MouthShrugUpper"),
makeAxisPair(Action::NOSESNEER_L, "NoseSneer_L"),
makeAxisPair(Action::NOSESNEER_R, "NoseSneer_R"),
makeAxisPair(Action::TONGUEOUT, "TongueOut"),
makeAxisPair(Action::USERBLENDSHAPE0, "UserBlendshape0"),
makeAxisPair(Action::USERBLENDSHAPE1, "UserBlendshape1"),
makeAxisPair(Action::USERBLENDSHAPE2, "UserBlendshape2"),
makeAxisPair(Action::USERBLENDSHAPE3, "UserBlendshape3"),
makeAxisPair(Action::USERBLENDSHAPE4, "UserBlendshape4"),
makeAxisPair(Action::USERBLENDSHAPE5, "UserBlendshape5"),
makeAxisPair(Action::USERBLENDSHAPE6, "UserBlendshape6"),
makeAxisPair(Action::USERBLENDSHAPE7, "UserBlendshape7"),
makeAxisPair(Action::USERBLENDSHAPE8, "UserBlendshape8"),
makeAxisPair(Action::USERBLENDSHAPE9, "UserBlendshape9"),
makePosePair(Action::LEFT_HAND_THUMB1, "LeftHandThumb1"),
makePosePair(Action::LEFT_HAND_THUMB2, "LeftHandThumb2"),

68
libraries/controllers/src/controllers/Actions.h
View file

@ -183,8 +183,72 @@ enum class Action {
LEFT_EYE,
RIGHT_EYE,
LEFT_EYE_BLINK,
RIGHT_EYE_BLINK,
// AJT: blendshapes
EyeBlink_L,
EyeBlink_R,
EyeSquint_L,
EyeSquint_R,
EyeDown_L,
EyeDown_R,
EyeIn_L,
EyeIn_R,
EyeOpen_L,
EyeOpen_R,
EyeOut_L,
EyeOut_R,
EyeUp_L,
EyeUp_R,
BrowsD_L,
BrowsD_R,
BrowsU_C,
BrowsU_L,
BrowsU_R,
JawFwd,
JawLeft,
JawOpen,
JawRight,
MouthLeft,
MouthRight,
MouthFrown_L,
MouthFrown_R,
MouthSmile_L,
MouthSmile_R,
MouthDimple_L,
MouthDimple_R,
LipsStretch_L,
LipsStretch_R,
LipsUpperClose,
LipsLowerClose,
LipsUpperOpen,
LipsLowerOpen,
LipsFunnel,
LipsPucker,
Puff,
CheekSquint_L,
CheekSquint_R,
LipsTogether,
MouthUpperUp_L,
MouthUpperUp_R,
MouthLowerDown_L,
MouthLowerDown_R,
MouthPress_L,
MouthPress_R,
MouthShrugLower,
MouthShrugUpper,
NoseSneer_L,
NoseSneer_R,
TongueOut,
UserBlendshape0,
UserBlendshape1,
UserBlendshape2,
UserBlendshape3,
UserBlendshape4,
UserBlendshape5,
UserBlendshape6,
UserBlendshape7,
UserBlendshape8,
UserBlendshape9,
NUM_ACTIONS
};

68
libraries/controllers/src/controllers/StandardController.cpp
View file

@ -355,8 +355,72 @@ Input::NamedVector StandardController::getAvailableInputs() const {
makePair(HEAD, "Head"),
makePair(LEFT_EYE, "LeftEye"),
makePair(RIGHT_EYE, "RightEye"),
makePair(LEFT_EYE_BLINK, "LeftEyeBlink"),
makePair(RIGHT_EYE_BLINK, "RightEyeBlink"),
// AJT: blendshapes
makePair(EYEBLINK_L, "EyeBlink_L"),
makePair(EYEBLINK_R, "EyeBlink_R"),
makePair(EYESQUINT_L, "EyeSquint_L"),
makePair(EYESQUINT_R, "EyeSquint_R"),
makePair(EYEDOWN_L, "EyeDown_L"),
makePair(EYEDOWN_R, "EyeDown_R"),
makePair(EYEIN_L, "EyeIn_L"),
makePair(EYEIN_R, "EyeIn_R"),
makePair(EYEOPEN_L, "EyeOpen_L"),
makePair(EYEOPEN_R, "EyeOpen_R"),
makePair(EYEOUT_L, "EyeOut_L"),
makePair(EYEOUT_R, "EyeOut_R"),
makePair(EYEUP_L, "EyeUp_L"),
makePair(EYEUP_R, "EyeUp_R"),
makePair(BROWSD_L, "BrowsD_L"),
makePair(BROWSD_R, "BrowsD_R"),
makePair(BROWSU_C, "BrowsU_C"),
makePair(BROWSU_L, "BrowsU_L"),
makePair(BROWSU_R, "BrowsU_R"),
makePair(JAWFWD, "JawFwd"),
makePair(JAWLEFT, "JawLeft"),
makePair(JAWOPEN, "JawOpen"),
makePair(JAWRIGHT, "JawRight"),
makePair(MOUTHLEFT, "MouthLeft"),
makePair(MOUTHRIGHT, "MouthRight"),
makePair(MOUTHFROWN_L, "MouthFrown_L"),
makePair(MOUTHFROWN_R, "MouthFrown_R"),
makePair(MOUTHSMILE_L, "MouthSmile_L"),
makePair(MOUTHSMILE_R, "MouthSmile_R"),
makePair(MOUTHDIMPLE_L, "MouthDimple_L"),
makePair(MOUTHDIMPLE_R, "MouthDimple_R"),
makePair(LIPSSTRETCH_L, "LipsStretch_L"),
makePair(LIPSSTRETCH_R, "LipsStretch_R"),
makePair(LIPSUPPERCLOSE, "LipsUpperClose"),
makePair(LIPSLOWERCLOSE, "LipsLowerClose"),
makePair(LIPSUPPEROPEN, "LipsUpperOpen"),
makePair(LIPSLOWEROPEN, "LipsLowerOpen"),
makePair(LIPSFUNNEL, "LipsFunnel"),
makePair(LIPSPUCKER, "LipsPucker"),
makePair(PUFF, "Puff"),
makePair(CHEEKSQUINT_L, "CheekSquint_L"),
makePair(CHEEKSQUINT_R, "CheekSquint_R"),
makePair(LIPSTOGETHER, "LipsTogether"),
makePair(MOUTHUPPERUP_L, "MouthUpperUp_L"),
makePair(MOUTHUPPERUP_R, "MouthUpperUp_R"),
makePair(MOUTHLOWERDOWN_L, "MouthLowerDown_L"),
makePair(MOUTHLOWERDOWN_R, "MouthLowerDown_R"),
makePair(MOUTHPRESS_L, "MouthPress_L"),
makePair(MOUTHPRESS_R, "MouthPress_R"),
makePair(MOUTHSHRUGLOWER, "MouthShrugLower"),
makePair(MOUTHSHRUGUPPER, "MouthShrugUpper"),
makePair(NOSESNEER_L, "NoseSneer_L"),
makePair(NOSESNEER_R, "NoseSneer_R"),
makePair(TONGUEOUT, "TongueOut"),
makePair(USERBLENDSHAPE0, "UserBlendshape0"),
makePair(USERBLENDSHAPE1, "UserBlendshape1"),
makePair(USERBLENDSHAPE2, "UserBlendshape2"),
makePair(USERBLENDSHAPE3, "UserBlendshape3"),
makePair(USERBLENDSHAPE4, "UserBlendshape4"),
makePair(USERBLENDSHAPE5, "UserBlendshape5"),
makePair(USERBLENDSHAPE6, "UserBlendshape6"),
makePair(USERBLENDSHAPE7, "UserBlendshape7"),
makePair(USERBLENDSHAPE8, "UserBlendshape8"),
makePair(USERBLENDSHAPE9, "UserBlendshape9"),
// Aliases, PlayStation style names
makePair(LB, "L1"),

3
libraries/controllers/src/controllers/StandardControls.h
View file

@ -90,8 +90,11 @@ namespace controller {
// Grips
LEFT_GRIP,
RIGHT_GRIP,
// AJT: blendshapes
LEFT_EYE_BLINK,
RIGHT_EYE_BLINK,
NUM_STANDARD_AXES,
LZ = LT,
RZ = RT

51
libraries/shared/src/BlendshapeConstants.cpp
View file

@ -34,7 +34,6 @@ const char* FACESHIFT_BLENDSHAPES[] = {
"JawFwd",
"JawLeft",
"JawOpen",
"JawChew",
"JawRight",
"MouthLeft",
"MouthRight",
@ -48,40 +47,34 @@ const char* FACESHIFT_BLENDSHAPES[] = {
"LipsStretch_R",
"LipsUpperClose",
"LipsLowerClose",
"LipsUpperUp",
"LipsLowerDown",
"LipsUpperOpen",
"LipsLowerOpen",
"LipsFunnel",
"LipsPucker",
"ChinLowerRaise",
"ChinUpperRaise",
"Sneer",
"Puff",
"CheekSquint_L",
"CheekSquint_R",
"LipsTogether",
"MouthUpperUp_L",
"MouthUpperUp_R",
"MouthLowerDown_L",
"MouthLowerDown_R",
"MouthPress_L",
"MouthPress_R",
"MouthShrugLower",
"MouthShrugUpper",
"NoseSneer_L",
"NoseSneer_R",
"TongueOut",
"UserBlendshape0",
"UserBlendshape1",
"UserBlendshape2",
"UserBlendshape3",
"UserBlendshape4",
"UserBlendshape5",
"UserBlendshape6",
"UserBlendshape7",
"UserBlendshape8",
"UserBlendshape9",
""
};
// new in ARKit
// LipsTogether
// MouthPressLeft
// MouthPressRight
// MouthShrugLower
// MouthShrugUpper
// TongueOut
const int EYE_BLINK_L_INDEX = 0;
const int EYE_BLINK_R_INDEX = 1;
const int EYE_SQUINT_L_INDEX = 2;
const int EYE_SQUINT_R_INDEX = 3;
const int EYE_OPEN_L_INDEX = 8;
const int EYE_OPEN_R_INDEX = 9;
const int BROWS_U_L_INDEX = 17;
const int BROWS_U_R_INDEX = 18;
const int EYE_BLINK_INDICES[] = { EYE_BLINK_L_INDEX, EYE_BLINK_R_INDEX };
const int EYE_SQUINT_INDICES[] = { EYE_SQUINT_L_INDEX, EYE_SQUINT_R_INDEX };
const int EYE_OPEN_INDICES[] = { EYE_OPEN_L_INDEX, EYE_OPEN_R_INDEX };
const int BROWS_U_INDICES[] = { BROWS_U_L_INDEX, BROWS_U_R_INDEX };

74
libraries/shared/src/BlendshapeConstants.h
View file

@ -15,12 +15,6 @@
/// The names of the blendshapes expected by Faceshift, terminated with an empty string.
extern const char* FACESHIFT_BLENDSHAPES[];
// Eyes and Brows indices
extern const int EYE_BLINK_INDICES[];
extern const int EYE_OPEN_INDICES[];
extern const int BROWS_U_INDICES[];
extern const int EYE_SQUINT_INDICES[];
enum class Blendshapes : int {
EyeBlink_L = 0,
EyeBlink_R,
@ -32,7 +26,7 @@ enum class Blendshapes : int {
EyeIn_R,
EyeOpen_L,
EyeOpen_R,
EyeOut_L, // 10
EyeOut_L,
EyeOut_R,
EyeUp_L,
EyeUp_R,
@ -42,9 +36,8 @@ enum class Blendshapes : int {
BrowsU_L,
BrowsU_R,
JawFwd,
JawLeft, // 20
JawLeft,
JawOpen,
JawChew, // legacy not in ARKit
JawRight,
MouthLeft,
MouthRight,
@ -52,25 +45,74 @@ enum class Blendshapes : int {
MouthFrown_R,
MouthSmile_L,
MouthSmile_R,
MouthDimple_L, // 30
MouthDimple_L,
MouthDimple_R,
LipsStretch_L,
LipsStretch_R,
LipsUpperClose,
LipsLowerClose,
LipsUpperUp, // legacy, split in ARKit
LipsLowerDown, // legacy, split in ARKit
LipsUpperOpen,
LipsLowerOpen,
LipsFunnel, // 40
LipsFunnel,
LipsPucker,
ChinLowerRaise,
ChinUpperRaise,
Sneer, // legacy, split in ARKit
Puff,
CheekSquint_L,
CheekSquint_R,
LipsTogether,
MouthUpperUp_L,
MouthUpperUp_R,
MouthLowerDown_L,
MouthLowerDown_R,
MouthPress_L,
MouthPress_R,
MouthShrugLower,
MouthShrugUpper,
NoseSneer_L,
NoseSneer_R,
TongueOut,
UserBlendshape0,
UserBlendshape1,
UserBlendshape2,
UserBlendshape3,
UserBlendshape4,
UserBlendshape5,
UserBlendshape6,
UserBlendshape7,
UserBlendshape8,
UserBlendshape9,
BlendshapeCount
};
enum class LegacyBlendshpaes : int {
JawChew, // not in ARKit
LipsUpperUp, // split in ARKit
LipsLowerDown, // split in ARKit
ChinLowerRaise, // not in ARKit
ChinUpperRaise, // not in ARKit
Sneer, // split in ARKit
LegacyBlendshapeCount
}
// NEW in ARKit
// * LipsTogether
// * MouthUpperUp_L
// * MouthUpperUp_R
// * MouthLowerDown_L
// * MouthLowerDown_R
// * MouthPress_L
// * MouthPress_R
// * MouthShrugLower
// * MouthShrugUpper
// * NoseSneer_L
// * NoseSneer_R
// * TongueOut
// Legacy shapes
// * JawChew (not in ARKit)
// * MouthUpperUp (split in ARKit)
// * MouthLowerDown (split in ARKit)
// * Sneer (split in ARKit)
// * ChinLowerRaise (not in ARKit)
// * ChinUpperRase (not in ARKit)
#endif // hifi_BlendshapeConstants_h