From 943348ba5171b5c19c27df09911b4be3ed207beb Mon Sep 17 00:00:00 2001 From: Seth Alves Date: Sat, 7 Sep 2019 18:11:04 -0700 Subject: [PATCH] input action channels for eyes and eyelids and fingers. connect eyelid input actions to blendshapes. --- interface/resources/controllers/standard.json | 9 +++- .../controllers/standard_nomovement.json | 8 +++- interface/resources/controllers/vive.json | 44 +++++++++++++++++++ interface/src/avatar/MyHead.cpp | 37 +++++++++++----- .../controllers/src/controllers/Actions.cpp | 4 ++ .../controllers/src/controllers/Actions.h | 5 +++ 6 files changed, 93 insertions(+), 14 deletions(-) diff --git a/interface/resources/controllers/standard.json b/interface/resources/controllers/standard.json index 0a5bd12460..195f909942 100644 --- a/interface/resources/controllers/standard.json +++ b/interface/resources/controllers/standard.json @@ -162,7 +162,14 @@ { "from": "Standard.Head", "to": "Actions.Head" }, { "from": "Standard.LeftArm", "to": "Actions.LeftArm" }, { "from": "Standard.RightArm", "to": "Actions.RightArm" }, - + + { "from": "Standard.LeftEye", "to": "Actions.LeftEye" }, + { "from": "Standard.RightEye", "to": "Actions.RightEye" }, + + { "from": "Standard.LeftEyeBlink", "to": "Actions.LeftEyeBlink" }, + { "from": "Standard.RightEyeBlink", "to": "Actions.RightEyeBlink" }, + + { "from": "Standard.TrackedObject00", "to" : "Actions.TrackedObject00" }, { "from": "Standard.TrackedObject01", "to" : "Actions.TrackedObject01" }, { "from": "Standard.TrackedObject02", "to" : "Actions.TrackedObject02" }, diff --git a/interface/resources/controllers/standard_nomovement.json b/interface/resources/controllers/standard_nomovement.json index 015bc33056..602d3bb798 100644 --- a/interface/resources/controllers/standard_nomovement.json +++ b/interface/resources/controllers/standard_nomovement.json @@ -57,7 +57,13 @@ { "from": "Standard.Head", "to": "Actions.Head" }, { "from": "Standard.LeftArm", "to": "Actions.LeftArm" }, { "from": "Standard.RightArm", "to": "Actions.RightArm" }, - + + { "from": "Standard.LeftEye", "to": "Actions.LeftEye" }, + { "from": "Standard.RightEye", "to": "Actions.RightEye" }, + + { "from": "Standard.LeftEyeBlink", "to": "Actions.LeftEyeBlink" }, + { "from": "Standard.RightEyeBlink", "to": "Actions.RightEyeBlink" }, + { "from": "Standard.TrackedObject00", "to" : "Actions.TrackedObject00" }, { "from": "Standard.TrackedObject01", "to" : "Actions.TrackedObject01" }, { "from": "Standard.TrackedObject02", "to" : "Actions.TrackedObject02" }, diff --git a/interface/resources/controllers/vive.json b/interface/resources/controllers/vive.json index 730e1bcb58..b6fae1dd79 100644 --- a/interface/resources/controllers/vive.json +++ b/interface/resources/controllers/vive.json @@ -54,8 +54,52 @@ { "from": "Vive.RightApplicationMenu", "to": "Standard.RightSecondaryThumb" }, { "from": "Vive.LeftHand", "to": "Standard.LeftHand" }, + { "from": "Vive.LeftHandThumb1", "to": "Standard.LeftHandThumb1"}, + { "from": "Vive.LeftHandThumb2", "to": "Standard.LeftHandThumb2"}, + { "from": "Vive.LeftHandThumb3", "to": "Standard.LeftHandThumb3"}, + { "from": "Vive.LeftHandThumb4", "to": "Standard.LeftHandThumb4"}, + { "from": "Vive.LeftHandIndex1", "to": "Standard.LeftHandIndex1"}, + { "from": "Vive.LeftHandIndex2", "to": "Standard.LeftHandIndex2"}, + { "from": "Vive.LeftHandIndex3", "to": "Standard.LeftHandIndex3"}, + { "from": "Vive.LeftHandIndex4", "to": "Standard.LeftHandIndex4"}, + { "from": "Vive.LeftHandMiddle1", "to": "Standard.LeftHandMiddle1"}, + { "from": "Vive.LeftHandMiddle2", "to": "Standard.LeftHandMiddle2"}, + { "from": "Vive.LeftHandMiddle3", "to": "Standard.LeftHandMiddle3"}, + { "from": "Vive.LeftHandMiddle4", "to": "Standard.LeftHandMiddle4"}, + { "from": "Vive.LeftHandRing1", "to": "Standard.LeftHandRing1"}, + { "from": "Vive.LeftHandRing2", "to": "Standard.LeftHandRing2"}, + { "from": "Vive.LeftHandRing3", "to": "Standard.LeftHandRing3"}, + { "from": "Vive.LeftHandRing4", "to": "Standard.LeftHandRing4"}, + { "from": "Vive.LeftHandPinky1", "to": "Standard.LeftHandPinky1"}, + { "from": "Vive.LeftHandPinky2", "to": "Standard.LeftHandPinky2"}, + { "from": "Vive.LeftHandPinky3", "to": "Standard.LeftHandPinky3"}, + { "from": "Vive.LeftHandPinky4", "to": "Standard.LeftHandPinky4"}, { "from": "Vive.RightHand", "to": "Standard.RightHand" }, + { "from": "Vive.RightHandThumb1", "to": "Standard.RightHandThumb1"}, + { "from": "Vive.RightHandThumb2", "to": "Standard.RightHandThumb2"}, + { "from": "Vive.RightHandThumb3", "to": "Standard.RightHandThumb3"}, + { "from": "Vive.RightHandThumb4", "to": "Standard.RightHandThumb4"}, + { "from": "Vive.RightHandIndex1", "to": "Standard.RightHandIndex1"}, + { "from": "Vive.RightHandIndex2", "to": "Standard.RightHandIndex2"}, + { "from": "Vive.RightHandIndex3", "to": "Standard.RightHandIndex3"}, + { "from": "Vive.RightHandIndex4", "to": "Standard.RightHandIndex4"}, + { "from": "Vive.RightHandMiddle1", "to": "Standard.RightHandMiddle1"}, + { "from": "Vive.RightHandMiddle2", "to": "Standard.RightHandMiddle2"}, + { "from": "Vive.RightHandMiddle3", "to": "Standard.RightHandMiddle3"}, + { "from": "Vive.RightHandMiddle4", "to": "Standard.RightHandMiddle4"}, + { "from": "Vive.RightHandRing1", "to": "Standard.RightHandRing1"}, + { "from": "Vive.RightHandRing2", "to": "Standard.RightHandRing2"}, + { "from": "Vive.RightHandRing3", "to": "Standard.RightHandRing3"}, + { "from": "Vive.RightHandRing4", "to": "Standard.RightHandRing4"}, + { "from": "Vive.RightHandPinky1", "to": "Standard.RightHandPinky1"}, + { "from": "Vive.RightHandPinky2", "to": "Standard.RightHandPinky2"}, + { "from": "Vive.RightHandPinky3", "to": "Standard.RightHandPinky3"}, + { "from": "Vive.RightHandPinky4", "to": "Standard.RightHandPinky4"}, { "from": "Vive.Head", "to" : "Standard.Head" }, + { "from": "Vive.LeftEye", "to" : "Standard.LeftEye" }, + { "from": "Vive.RightEye", "to" : "Standard.RightEye" }, + { "from": "Vive.LeftEyeBlink", "to" : "Standard.LeftEyeBlink" }, + { "from": "Vive.RightEyeBlink", "to" : "Standard.RightEyeBlink" }, { "from": "Vive.LeftFoot", "to" : "Standard.LeftFoot", diff --git a/interface/src/avatar/MyHead.cpp b/interface/src/avatar/MyHead.cpp index 9b05a26c76..b5e8bc2171 100644 --- a/interface/src/avatar/MyHead.cpp +++ b/interface/src/avatar/MyHead.cpp @@ -15,7 +15,6 @@ #include #include #include -#include #include "devices/DdeFaceTracker.h" #include "Application.h" @@ -46,18 +45,32 @@ void MyHead::simulate(float deltaTime) { auto player = DependencyManager::get(); // Only use face trackers when not playing back a recording. if (!player->isPlaying()) { - auto faceTracker = qApp->getActiveFaceTracker(); - const bool hasActualFaceTrackerConnected = faceTracker && !faceTracker->isMuted(); - _isFaceTrackerConnected = hasActualFaceTrackerConnected || _owningAvatar->getHasScriptedBlendshapes(); - if (_isFaceTrackerConnected) { - if (hasActualFaceTrackerConnected) { - _blendshapeCoefficients = faceTracker->getBlendshapeCoefficients(); - } - } + // auto faceTracker = qApp->getActiveFaceTracker(); + // const bool hasActualFaceTrackerConnected = faceTracker && !faceTracker->isMuted(); + // _isFaceTrackerConnected = hasActualFaceTrackerConnected || _owningAvatar->getHasScriptedBlendshapes(); + // if (_isFaceTrackerConnected) { + // if (hasActualFaceTrackerConnected) { + // _blendshapeCoefficients = faceTracker->getBlendshapeCoefficients(); + // } + // } - auto eyeTracker = DependencyManager::get(); - _isEyeTrackerConnected = eyeTracker->isTracking(); - // if eye tracker is connected we should get the data here. + auto userInputMapper = DependencyManager::get(); + bool eyeLidsTracked = + userInputMapper->getActionStateValid(controller::Action::LEFT_EYE_BLINK) && + userInputMapper->getActionStateValid(controller::Action::RIGHT_EYE_BLINK); + setFaceTrackerConnected(eyeLidsTracked); + if (eyeLidsTracked) { + float leftEyeBlink = userInputMapper->getActionState(controller::Action::LEFT_EYE_BLINK); + float rightEyeBlink = userInputMapper->getActionState(controller::Action::RIGHT_EYE_BLINK); + _blendshapeCoefficients.resize(std::max(_blendshapeCoefficients.size(), 2)); + _blendshapeCoefficients[0] = leftEyeBlink; + _blendshapeCoefficients[1] = rightEyeBlink; + } else { + const float FULLY_OPEN = 0.0f; + _blendshapeCoefficients.resize(std::max(_blendshapeCoefficients.size(), 2)); + _blendshapeCoefficients[0] = FULLY_OPEN; + _blendshapeCoefficients[1] = FULLY_OPEN; + } } Parent::simulate(deltaTime); } diff --git a/libraries/controllers/src/controllers/Actions.cpp b/libraries/controllers/src/controllers/Actions.cpp index 9f9d92fed7..40011f2682 100644 --- a/libraries/controllers/src/controllers/Actions.cpp +++ b/libraries/controllers/src/controllers/Actions.cpp @@ -347,6 +347,10 @@ namespace controller { makePosePair(Action::HIPS, "Hips"), makePosePair(Action::SPINE2, "Spine2"), makePosePair(Action::HEAD, "Head"), + makePosePair(Action::LEFT_EYE, "LeftEye"), + makePosePair(Action::RIGHT_EYE, "RightEye"), + makeAxisPair(Action::LEFT_EYE_BLINK, "LeftEyeBlink"), + makeAxisPair(Action::RIGHT_EYE_BLINK, "RightEyeBlink"), makePosePair(Action::LEFT_HAND_THUMB1, "LeftHandThumb1"), makePosePair(Action::LEFT_HAND_THUMB2, "LeftHandThumb2"), diff --git a/libraries/controllers/src/controllers/Actions.h b/libraries/controllers/src/controllers/Actions.h index 3e99d8d147..f91d9f2522 100644 --- a/libraries/controllers/src/controllers/Actions.h +++ b/libraries/controllers/src/controllers/Actions.h @@ -181,6 +181,11 @@ enum class Action { TRACKED_OBJECT_15, SPRINT, + LEFT_EYE, + RIGHT_EYE, + LEFT_EYE_BLINK, + RIGHT_EYE_BLINK, + NUM_ACTIONS };