From 3bdf267900f44f3c0cef2d24cf53279cba5738ac Mon Sep 17 00:00:00 2001 From: David Rowe Date: Thu, 22 Jun 2017 23:32:06 +1200 Subject: [PATCH] Implement fingers control --- .../resources/controllers/leapmotion.json | 43 +++- interface/resources/controllers/standard.json | 41 ++++ interface/src/Application.cpp | 10 + interface/src/avatar/MyAvatar.cpp | 13 ++ interface/src/avatar/MyAvatar.h | 7 + interface/src/avatar/MySkeletonModel.cpp | 195 +++++++++++++++++- .../controllers/src/controllers/Actions.cpp | 42 ++++ .../controllers/src/controllers/Actions.h | 41 ++++ .../src/controllers/StandardController.cpp | 40 ++++ .../hifiLeapMotion/src/LeapMotionPlugin.cpp | 128 ++++++++++-- 10 files changed, 546 insertions(+), 14 deletions(-) diff --git a/interface/resources/controllers/leapmotion.json b/interface/resources/controllers/leapmotion.json index 2ee4c98009..25cb575946 100644 --- a/interface/resources/controllers/leapmotion.json +++ b/interface/resources/controllers/leapmotion.json @@ -2,6 +2,47 @@ "name": "Leap Motion to Standard", "channels": [ { "from": "LeapMotion.LeftHand", "to": "Standard.LeftHand" }, - { "from": "LeapMotion.RightHand", "to": "Standard.RightHand" } + { "from": "LeapMotion.LeftHandThumb1", "to": "Standard.LeftHandThumb1"}, + { "from": "LeapMotion.LeftHandThumb2", "to": "Standard.LeftHandThumb2"}, + { "from": "LeapMotion.LeftHandThumb3", "to": "Standard.LeftHandThumb3"}, + { "from": "LeapMotion.LeftHandThumb4", "to": "Standard.LeftHandThumb4"}, + { "from": "LeapMotion.LeftHandIndex1", "to": "Standard.LeftHandIndex1"}, + { "from": "LeapMotion.LeftHandIndex2", "to": "Standard.LeftHandIndex2"}, + { "from": "LeapMotion.LeftHandIndex3", "to": "Standard.LeftHandIndex3"}, + { "from": "LeapMotion.LeftHandIndex4", "to": "Standard.LeftHandIndex4"}, + { "from": "LeapMotion.LeftHandMiddle1", "to": "Standard.LeftHandMiddle1"}, + { "from": "LeapMotion.LeftHandMiddle2", "to": "Standard.LeftHandMiddle2"}, + { "from": "LeapMotion.LeftHandMiddle3", "to": "Standard.LeftHandMiddle3"}, + { "from": "LeapMotion.LeftHandMiddle4", "to": "Standard.LeftHandMiddle4"}, + { "from": "LeapMotion.LeftHandRing1", "to": "Standard.LeftHandRing1"}, + { "from": "LeapMotion.LeftHandRing2", "to": "Standard.LeftHandRing2"}, + { "from": "LeapMotion.LeftHandRing3", "to": "Standard.LeftHandRing3"}, + { "from": "LeapMotion.LeftHandRing4", "to": "Standard.LeftHandRing4"}, + { "from": "LeapMotion.LeftHandPinky1", "to": "Standard.LeftHandPinky1"}, + { "from": "LeapMotion.LeftHandPinky2", "to": "Standard.LeftHandPinky2"}, + { "from": "LeapMotion.LeftHandPinky3", "to": "Standard.LeftHandPinky3"}, + { "from": "LeapMotion.LeftHandPinky4", "to": "Standard.LeftHandPinky4"}, + + { "from": "LeapMotion.RightHand", "to": "Standard.RightHand" }, + { "from": "LeapMotion.RightHandThumb1", "to": "Standard.RightHandThumb1"}, + { "from": "LeapMotion.RightHandThumb2", "to": "Standard.RightHandThumb2"}, + { "from": "LeapMotion.RightHandThumb3", "to": "Standard.RightHandThumb3"}, + { "from": "LeapMotion.RightHandThumb4", "to": "Standard.RightHandThumb4"}, + { "from": "LeapMotion.RightHandIndex1", "to": "Standard.RightHandIndex1"}, + { "from": "LeapMotion.RightHandIndex2", "to": "Standard.RightHandIndex2"}, + { "from": "LeapMotion.RightHandIndex3", "to": "Standard.RightHandIndex3"}, + { "from": "LeapMotion.RightHandIndex4", "to": "Standard.RightHandIndex4"}, + { "from": "LeapMotion.RightHandMiddle1", "to": "Standard.RightHandMiddle1"}, + { "from": "LeapMotion.RightHandMiddle2", "to": "Standard.RightHandMiddle2"}, + { "from": "LeapMotion.RightHandMiddle3", "to": "Standard.RightHandMiddle3"}, + { "from": "LeapMotion.RightHandMiddle4", "to": "Standard.RightHandMiddle4"}, + { "from": "LeapMotion.RightHandRing1", "to": "Standard.RightHandRing1"}, + { "from": "LeapMotion.RightHandRing2", "to": "Standard.RightHandRing2"}, + { "from": "LeapMotion.RightHandRing3", "to": "Standard.RightHandRing3"}, + { "from": "LeapMotion.RightHandRing4", "to": "Standard.RightHandRing4"}, + { "from": "LeapMotion.RightHandPinky1", "to": "Standard.RightHandPinky1"}, + { "from": "LeapMotion.RightHandPinky2", "to": "Standard.RightHandPinky2"}, + { "from": "LeapMotion.RightHandPinky3", "to": "Standard.RightHandPinky3"}, + { "from": "LeapMotion.RightHandPinky4", "to": "Standard.RightHandPinky4"} ] } diff --git a/interface/resources/controllers/standard.json b/interface/resources/controllers/standard.json index 75b4821118..166f1a6869 100644 --- a/interface/resources/controllers/standard.json +++ b/interface/resources/controllers/standard.json @@ -58,7 +58,48 @@ { "from": "Standard.RT", "to": "Actions.RightHandClick" }, { "from": "Standard.LeftHand", "to": "Actions.LeftHand" }, + { "from": "Standard.LeftHandThumb1", "to": "Actions.LeftHandThumb1"}, + { "from": "Standard.LeftHandThumb2", "to": "Actions.LeftHandThumb2"}, + { "from": "Standard.LeftHandThumb3", "to": "Actions.LeftHandThumb3"}, + { "from": "Standard.LeftHandThumb4", "to": "Actions.LeftHandThumb4"}, + { "from": "Standard.LeftHandIndex1", "to": "Actions.LeftHandIndex1"}, + { "from": "Standard.LeftHandIndex2", "to": "Actions.LeftHandIndex2"}, + { "from": "Standard.LeftHandIndex3", "to": "Actions.LeftHandIndex3"}, + { "from": "Standard.LeftHandIndex4", "to": "Actions.LeftHandIndex4"}, + { "from": "Standard.LeftHandMiddle1", "to": "Actions.LeftHandMiddle1"}, + { "from": "Standard.LeftHandMiddle2", "to": "Actions.LeftHandMiddle2"}, + { "from": "Standard.LeftHandMiddle3", "to": "Actions.LeftHandMiddle3"}, + { "from": "Standard.LeftHandMiddle4", "to": "Actions.LeftHandMiddle4"}, + { "from": "Standard.LeftHandRing1", "to": "Actions.LeftHandRing1"}, + { "from": "Standard.LeftHandRing2", "to": "Actions.LeftHandRing2"}, + { "from": "Standard.LeftHandRing3", "to": "Actions.LeftHandRing3"}, + { "from": "Standard.LeftHandRing4", "to": "Actions.LeftHandRing4"}, + { "from": "Standard.LeftHandPinky1", "to": "Actions.LeftHandPinky1"}, + { "from": "Standard.LeftHandPinky2", "to": "Actions.LeftHandPinky2"}, + { "from": "Standard.LeftHandPinky3", "to": "Actions.LeftHandPinky3"}, + { "from": "Standard.LeftHandPinky4", "to": "Actions.LeftHandPinky4"}, + { "from": "Standard.RightHand", "to": "Actions.RightHand" }, + { "from": "Standard.RightHandThumb1", "to": "Actions.RightHandThumb1"}, + { "from": "Standard.RightHandThumb2", "to": "Actions.RightHandThumb2"}, + { "from": "Standard.RightHandThumb3", "to": "Actions.RightHandThumb3"}, + { "from": "Standard.RightHandThumb4", "to": "Actions.RightHandThumb4"}, + { "from": "Standard.RightHandIndex1", "to": "Actions.RightHandIndex1"}, + { "from": "Standard.RightHandIndex2", "to": "Actions.RightHandIndex2"}, + { "from": "Standard.RightHandIndex3", "to": "Actions.RightHandIndex3"}, + { "from": "Standard.RightHandIndex4", "to": "Actions.RightHandIndex4"}, + { "from": "Standard.RightHandMiddle1", "to": "Actions.RightHandMiddle1"}, + { "from": "Standard.RightHandMiddle2", "to": "Actions.RightHandMiddle2"}, + { "from": "Standard.RightHandMiddle3", "to": "Actions.RightHandMiddle3"}, + { "from": "Standard.RightHandMiddle4", "to": "Actions.RightHandMiddle4"}, + { "from": "Standard.RightHandRing1", "to": "Actions.RightHandRing1"}, + { "from": "Standard.RightHandRing2", "to": "Actions.RightHandRing2"}, + { "from": "Standard.RightHandRing3", "to": "Actions.RightHandRing3"}, + { "from": "Standard.RightHandRing4", "to": "Actions.RightHandRing4"}, + { "from": "Standard.RightHandPinky1", "to": "Actions.RightHandPinky1"}, + { "from": "Standard.RightHandPinky2", "to": "Actions.RightHandPinky2"}, + { "from": "Standard.RightHandPinky3", "to": "Actions.RightHandPinky3"}, + { "from": "Standard.RightHandPinky4", "to": "Actions.RightHandPinky4"}, { "from": "Standard.LeftFoot", "to": "Actions.LeftFoot" }, { "from": "Standard.RightFoot", "to": "Actions.RightFoot" }, diff --git a/interface/src/Application.cpp b/interface/src/Application.cpp index c67ba9c81f..2fa16c4225 100644 --- a/interface/src/Application.cpp +++ b/interface/src/Application.cpp @@ -4537,6 +4537,16 @@ void Application::update(float deltaTime) { auto avatarToSensorMatrix = worldToSensorMatrix * myAvatarMatrix; myAvatar->setHandControllerPosesInSensorFrame(leftHandPose.transform(avatarToSensorMatrix), rightHandPose.transform(avatarToSensorMatrix)); + MyAvatar::FingerPosesMap leftHandFingerPoses; + for (int i = (int)controller::Action::LEFT_HAND_THUMB1; i <= (int)controller::Action::LEFT_HAND_PINKY4; i++) { + leftHandFingerPoses[i] = userInputMapper->getPoseState((controller::Action)i).transform(avatarToSensorMatrix); + } + MyAvatar::FingerPosesMap rightHandFingerPoses; + for (int i = (int)controller::Action::RIGHT_HAND_THUMB1; i <= (int)controller::Action::RIGHT_HAND_PINKY4; i++) { + rightHandFingerPoses[i] = userInputMapper->getPoseState((controller::Action)i).transform(avatarToSensorMatrix); + } + myAvatar->setFingerControllerPosesInSensorFrame(leftHandFingerPoses, rightHandFingerPoses); + controller::Pose leftFootPose = userInputMapper->getPoseState(controller::Action::LEFT_FOOT); controller::Pose rightFootPose = userInputMapper->getPoseState(controller::Action::RIGHT_FOOT); myAvatar->setFootControllerPosesInSensorFrame(leftFootPose.transform(avatarToSensorMatrix), rightFootPose.transform(avatarToSensorMatrix)); diff --git a/interface/src/avatar/MyAvatar.cpp b/interface/src/avatar/MyAvatar.cpp index 1adcfbd345..6194ff154b 100755 --- a/interface/src/avatar/MyAvatar.cpp +++ b/interface/src/avatar/MyAvatar.cpp @@ -1447,6 +1447,19 @@ controller::Pose MyAvatar::getRightHandControllerPoseInAvatarFrame() const { return getRightHandControllerPoseInWorldFrame().transform(invAvatarMatrix); } +void MyAvatar::setFingerControllerPosesInSensorFrame(const FingerPosesMap& left, const FingerPosesMap& right) { + _leftHandFingerPosesInSensorFramceCache.set(left); + _rightHandFingerPosesInSensorFramceCache.set(right); +} + +MyAvatar::FingerPosesMap MyAvatar::getLeftHandFingerControllerPosesInSensorFrame() const { + return _leftHandFingerPosesInSensorFramceCache.get(); +} + +MyAvatar::FingerPosesMap MyAvatar::getRightHandFingerControllerPosesInSensorFrame() const { + return _rightHandFingerPosesInSensorFramceCache.get(); +} + void MyAvatar::setFootControllerPosesInSensorFrame(const controller::Pose& left, const controller::Pose& right) { _leftFootControllerPoseInSensorFrameCache.set(left); _rightFootControllerPoseInSensorFrameCache.set(right); diff --git a/interface/src/avatar/MyAvatar.h b/interface/src/avatar/MyAvatar.h index f61f24fb11..4d42b17301 100644 --- a/interface/src/avatar/MyAvatar.h +++ b/interface/src/avatar/MyAvatar.h @@ -469,6 +469,11 @@ public: controller::Pose getLeftHandControllerPoseInAvatarFrame() const; controller::Pose getRightHandControllerPoseInAvatarFrame() const; + typedef std::map FingerPosesMap; + void setFingerControllerPosesInSensorFrame(const FingerPosesMap& left, const FingerPosesMap& right); + FingerPosesMap getLeftHandFingerControllerPosesInSensorFrame() const; + FingerPosesMap getRightHandFingerControllerPosesInSensorFrame() const; + void setFootControllerPosesInSensorFrame(const controller::Pose& left, const controller::Pose& right); controller::Pose getLeftFootControllerPoseInSensorFrame() const; controller::Pose getRightFootControllerPoseInSensorFrame() const; @@ -783,6 +788,8 @@ private: ThreadSafeValueCache _headControllerPoseInSensorFrameCache{ controller::Pose() }; ThreadSafeValueCache _leftArmControllerPoseInSensorFrameCache{ controller::Pose() }; ThreadSafeValueCache _rightArmControllerPoseInSensorFrameCache{ controller::Pose() }; + ThreadSafeValueCache _leftHandFingerPosesInSensorFramceCache { }; + ThreadSafeValueCache _rightHandFingerPosesInSensorFramceCache { }; bool _hmdLeanRecenterEnabled = true; diff --git a/interface/src/avatar/MySkeletonModel.cpp b/interface/src/avatar/MySkeletonModel.cpp index e74df4cf0f..25d8427ccd 100644 --- a/interface/src/avatar/MySkeletonModel.cpp +++ b/interface/src/avatar/MySkeletonModel.cpp @@ -171,5 +171,198 @@ void MySkeletonModel::updateRig(float deltaTime, glm::mat4 parentTransform) { eyeParams.rightEyeJointIndex = geometry.rightEyeJointIndex; _rig.updateFromEyeParameters(eyeParams); -} + auto fingerPoses = myAvatar->getLeftHandFingerControllerPosesInSensorFrame(); + if (leftHandPose.isValid() && fingerPoses.size() > 0) { + // Can just check the first finger pose because either all finger poses will be valid or none of them will. + if (fingerPoses[(int)controller::Action::LEFT_HAND_INDEX1].isValid()) { + glm::quat handJointRotation = myAvatar->getLeftHandControllerPoseInSensorFrame().getRotation(); + + glm::quat previousJointRotation; + glm::quat thisJointRotation; + + previousJointRotation = handJointRotation; + thisJointRotation = fingerPoses[(int)controller::Action::LEFT_HAND_THUMB1].getRotation(); + myAvatar->setJointRotation(_rig.indexOfJoint("LeftHandThumb1"), glm::inverse(previousJointRotation) * thisJointRotation); + previousJointRotation = thisJointRotation; + thisJointRotation = fingerPoses[(int)controller::Action::LEFT_HAND_THUMB2].getRotation(); + myAvatar->setJointRotation(_rig.indexOfJoint("LeftHandThumb2"), glm::inverse(previousJointRotation) * thisJointRotation); + previousJointRotation = thisJointRotation; + thisJointRotation = fingerPoses[(int)controller::Action::LEFT_HAND_THUMB3].getRotation(); + myAvatar->setJointRotation(_rig.indexOfJoint("LeftHandThumb3"), glm::inverse(previousJointRotation) * thisJointRotation); + previousJointRotation = thisJointRotation; + thisJointRotation = fingerPoses[(int)controller::Action::LEFT_HAND_THUMB4].getRotation(); + myAvatar->setJointRotation(_rig.indexOfJoint("LeftHandThumb4"), glm::inverse(previousJointRotation) * thisJointRotation); + + previousJointRotation = handJointRotation; + thisJointRotation = fingerPoses[(int)controller::Action::LEFT_HAND_INDEX1].getRotation(); + myAvatar->setJointRotation(_rig.indexOfJoint("LeftHandIndex1"), glm::inverse(previousJointRotation) * thisJointRotation); + previousJointRotation = thisJointRotation; + thisJointRotation = fingerPoses[(int)controller::Action::LEFT_HAND_INDEX2].getRotation(); + myAvatar->setJointRotation(_rig.indexOfJoint("LeftHandIndex2"), glm::inverse(previousJointRotation) * thisJointRotation); + previousJointRotation = thisJointRotation; + thisJointRotation = fingerPoses[(int)controller::Action::LEFT_HAND_INDEX3].getRotation(); + myAvatar->setJointRotation(_rig.indexOfJoint("LeftHandIndex3"), glm::inverse(previousJointRotation) * thisJointRotation); + previousJointRotation = thisJointRotation; + thisJointRotation = fingerPoses[(int)controller::Action::LEFT_HAND_INDEX4].getRotation(); + myAvatar->setJointRotation(_rig.indexOfJoint("LeftHandIndex4"), glm::inverse(previousJointRotation) * thisJointRotation); + + previousJointRotation = handJointRotation; + thisJointRotation = fingerPoses[(int)controller::Action::LEFT_HAND_MIDDLE1].getRotation(); + myAvatar->setJointRotation(_rig.indexOfJoint("LeftHandMiddle1"), glm::inverse(previousJointRotation) * thisJointRotation); + previousJointRotation = thisJointRotation; + thisJointRotation = fingerPoses[(int)controller::Action::LEFT_HAND_MIDDLE2].getRotation(); + myAvatar->setJointRotation(_rig.indexOfJoint("LeftHandMiddle2"), glm::inverse(previousJointRotation) * thisJointRotation); + previousJointRotation = thisJointRotation; + thisJointRotation = fingerPoses[(int)controller::Action::LEFT_HAND_MIDDLE3].getRotation(); + myAvatar->setJointRotation(_rig.indexOfJoint("LeftHandMiddle3"), glm::inverse(previousJointRotation) * thisJointRotation); + previousJointRotation = thisJointRotation; + thisJointRotation = fingerPoses[(int)controller::Action::LEFT_HAND_MIDDLE4].getRotation(); + myAvatar->setJointRotation(_rig.indexOfJoint("LeftHandMiddle4"), glm::inverse(previousJointRotation) * thisJointRotation); + + previousJointRotation = handJointRotation; + thisJointRotation = fingerPoses[(int)controller::Action::LEFT_HAND_RING1].getRotation(); + myAvatar->setJointRotation(_rig.indexOfJoint("LeftHandRing1"), glm::inverse(previousJointRotation) * thisJointRotation); + previousJointRotation = thisJointRotation; + thisJointRotation = fingerPoses[(int)controller::Action::LEFT_HAND_RING2].getRotation(); + myAvatar->setJointRotation(_rig.indexOfJoint("LeftHandRing2"), glm::inverse(previousJointRotation) * thisJointRotation); + previousJointRotation = thisJointRotation; + thisJointRotation = fingerPoses[(int)controller::Action::LEFT_HAND_RING3].getRotation(); + myAvatar->setJointRotation(_rig.indexOfJoint("LeftHandRing3"), glm::inverse(previousJointRotation) * thisJointRotation); + previousJointRotation = thisJointRotation; + thisJointRotation = fingerPoses[(int)controller::Action::LEFT_HAND_RING4].getRotation(); + myAvatar->setJointRotation(_rig.indexOfJoint("LeftHandRing4"), glm::inverse(previousJointRotation) * thisJointRotation); + + previousJointRotation = handJointRotation; + thisJointRotation = fingerPoses[(int)controller::Action::LEFT_HAND_PINKY1].getRotation(); + myAvatar->setJointRotation(_rig.indexOfJoint("LeftHandPinky1"), glm::inverse(previousJointRotation) * thisJointRotation); + previousJointRotation = thisJointRotation; + thisJointRotation = fingerPoses[(int)controller::Action::LEFT_HAND_PINKY2].getRotation(); + myAvatar->setJointRotation(_rig.indexOfJoint("LeftHandPinky2"), glm::inverse(previousJointRotation) * thisJointRotation); + previousJointRotation = thisJointRotation; + thisJointRotation = fingerPoses[(int)controller::Action::LEFT_HAND_PINKY3].getRotation(); + myAvatar->setJointRotation(_rig.indexOfJoint("LeftHandPinky3"), glm::inverse(previousJointRotation) * thisJointRotation); + previousJointRotation = thisJointRotation; + thisJointRotation = fingerPoses[(int)controller::Action::LEFT_HAND_PINKY4].getRotation(); + myAvatar->setJointRotation(_rig.indexOfJoint("LeftHandPinky4"), glm::inverse(previousJointRotation) * thisJointRotation); + } else { + myAvatar->clearJointData(_rig.indexOfJoint("LeftHandThumb1")); + myAvatar->clearJointData(_rig.indexOfJoint("LeftHandThumb2")); + myAvatar->clearJointData(_rig.indexOfJoint("LeftHandThumb3")); + myAvatar->clearJointData(_rig.indexOfJoint("LeftHandThumb4")); + myAvatar->clearJointData(_rig.indexOfJoint("LeftHandIndex1")); + myAvatar->clearJointData(_rig.indexOfJoint("LeftHandIndex2")); + myAvatar->clearJointData(_rig.indexOfJoint("LeftHandIndex3")); + myAvatar->clearJointData(_rig.indexOfJoint("LeftHandIndex4")); + myAvatar->clearJointData(_rig.indexOfJoint("LeftHandMiddle1")); + myAvatar->clearJointData(_rig.indexOfJoint("LeftHandMiddle2")); + myAvatar->clearJointData(_rig.indexOfJoint("LeftHandMiddle3")); + myAvatar->clearJointData(_rig.indexOfJoint("LeftHandMiddle4")); + myAvatar->clearJointData(_rig.indexOfJoint("LeftHandRing1")); + myAvatar->clearJointData(_rig.indexOfJoint("LeftHandRing2")); + myAvatar->clearJointData(_rig.indexOfJoint("LeftHandRing3")); + myAvatar->clearJointData(_rig.indexOfJoint("LeftHandRing4")); + myAvatar->clearJointData(_rig.indexOfJoint("LeftHandPinky1")); + myAvatar->clearJointData(_rig.indexOfJoint("LeftHandPinky2")); + myAvatar->clearJointData(_rig.indexOfJoint("LeftHandPinky3")); + myAvatar->clearJointData(_rig.indexOfJoint("LeftHandPinky4")); + } + } + + fingerPoses = myAvatar->getRightHandFingerControllerPosesInSensorFrame(); + if (rightHandPose.isValid() && fingerPoses.size() > 0) { + // Can just check the first finger pose because either all finger poses will be valid or none of them will. + if (fingerPoses[(int)controller::Action::RIGHT_HAND_INDEX1].isValid()) { + glm::quat handJointRotation = myAvatar->getRightHandControllerPoseInSensorFrame().getRotation(); + + glm::quat previousJointRotation; + glm::quat thisJointRotation; + + previousJointRotation = handJointRotation; + thisJointRotation = fingerPoses[(int)controller::Action::RIGHT_HAND_THUMB1].getRotation(); + myAvatar->setJointRotation(_rig.indexOfJoint("RightHandThumb1"), glm::inverse(previousJointRotation) * thisJointRotation); + previousJointRotation = thisJointRotation; + thisJointRotation = fingerPoses[(int)controller::Action::RIGHT_HAND_THUMB2].getRotation(); + myAvatar->setJointRotation(_rig.indexOfJoint("RightHandThumb2"), glm::inverse(previousJointRotation) * thisJointRotation); + previousJointRotation = thisJointRotation; + thisJointRotation = fingerPoses[(int)controller::Action::RIGHT_HAND_THUMB3].getRotation(); + myAvatar->setJointRotation(_rig.indexOfJoint("RightHandThumb3"), glm::inverse(previousJointRotation) * thisJointRotation); + previousJointRotation = thisJointRotation; + thisJointRotation = fingerPoses[(int)controller::Action::RIGHT_HAND_THUMB4].getRotation(); + myAvatar->setJointRotation(_rig.indexOfJoint("RightHandThumb4"), glm::inverse(previousJointRotation) * thisJointRotation); + + previousJointRotation = handJointRotation; + thisJointRotation = fingerPoses[(int)controller::Action::RIGHT_HAND_INDEX1].getRotation(); + myAvatar->setJointRotation(_rig.indexOfJoint("RightHandIndex1"), glm::inverse(previousJointRotation) * thisJointRotation); + previousJointRotation = thisJointRotation; + thisJointRotation = fingerPoses[(int)controller::Action::RIGHT_HAND_INDEX2].getRotation(); + myAvatar->setJointRotation(_rig.indexOfJoint("RightHandIndex2"), glm::inverse(previousJointRotation) * thisJointRotation); + previousJointRotation = thisJointRotation; + thisJointRotation = fingerPoses[(int)controller::Action::RIGHT_HAND_INDEX3].getRotation(); + myAvatar->setJointRotation(_rig.indexOfJoint("RightHandIndex3"), glm::inverse(previousJointRotation) * thisJointRotation); + previousJointRotation = thisJointRotation; + thisJointRotation = fingerPoses[(int)controller::Action::RIGHT_HAND_INDEX4].getRotation(); + myAvatar->setJointRotation(_rig.indexOfJoint("RightHandIndex4"), glm::inverse(previousJointRotation) * thisJointRotation); + + previousJointRotation = handJointRotation; + thisJointRotation = fingerPoses[(int)controller::Action::RIGHT_HAND_MIDDLE1].getRotation(); + myAvatar->setJointRotation(_rig.indexOfJoint("RightHandMiddle1"), glm::inverse(previousJointRotation) * thisJointRotation); + previousJointRotation = thisJointRotation; + thisJointRotation = fingerPoses[(int)controller::Action::RIGHT_HAND_MIDDLE2].getRotation(); + myAvatar->setJointRotation(_rig.indexOfJoint("RightHandMiddle2"), glm::inverse(previousJointRotation) * thisJointRotation); + previousJointRotation = thisJointRotation; + thisJointRotation = fingerPoses[(int)controller::Action::RIGHT_HAND_MIDDLE3].getRotation(); + myAvatar->setJointRotation(_rig.indexOfJoint("RightHandMiddle3"), glm::inverse(previousJointRotation) * thisJointRotation); + previousJointRotation = thisJointRotation; + thisJointRotation = fingerPoses[(int)controller::Action::RIGHT_HAND_MIDDLE4].getRotation(); + myAvatar->setJointRotation(_rig.indexOfJoint("RightHandMiddle4"), glm::inverse(previousJointRotation) * thisJointRotation); + + previousJointRotation = handJointRotation; + thisJointRotation = fingerPoses[(int)controller::Action::RIGHT_HAND_RING1].getRotation(); + myAvatar->setJointRotation(_rig.indexOfJoint("RightHandRing1"), glm::inverse(previousJointRotation) * thisJointRotation); + previousJointRotation = thisJointRotation; + thisJointRotation = fingerPoses[(int)controller::Action::RIGHT_HAND_RING2].getRotation(); + myAvatar->setJointRotation(_rig.indexOfJoint("RightHandRing2"), glm::inverse(previousJointRotation) * thisJointRotation); + previousJointRotation = thisJointRotation; + thisJointRotation = fingerPoses[(int)controller::Action::RIGHT_HAND_RING3].getRotation(); + myAvatar->setJointRotation(_rig.indexOfJoint("RightHandRing3"), glm::inverse(previousJointRotation) * thisJointRotation); + previousJointRotation = thisJointRotation; + thisJointRotation = fingerPoses[(int)controller::Action::RIGHT_HAND_RING4].getRotation(); + myAvatar->setJointRotation(_rig.indexOfJoint("RightHandRing4"), glm::inverse(previousJointRotation) * thisJointRotation); + + previousJointRotation = handJointRotation; + thisJointRotation = fingerPoses[(int)controller::Action::RIGHT_HAND_PINKY1].getRotation(); + myAvatar->setJointRotation(_rig.indexOfJoint("RightHandPinky1"), glm::inverse(previousJointRotation) * thisJointRotation); + previousJointRotation = thisJointRotation; + thisJointRotation = fingerPoses[(int)controller::Action::RIGHT_HAND_PINKY2].getRotation(); + myAvatar->setJointRotation(_rig.indexOfJoint("RightHandPinky2"), glm::inverse(previousJointRotation) * thisJointRotation); + previousJointRotation = thisJointRotation; + thisJointRotation = fingerPoses[(int)controller::Action::RIGHT_HAND_PINKY3].getRotation(); + myAvatar->setJointRotation(_rig.indexOfJoint("RightHandPinky3"), glm::inverse(previousJointRotation) * thisJointRotation); + previousJointRotation = thisJointRotation; + thisJointRotation = fingerPoses[(int)controller::Action::RIGHT_HAND_PINKY4].getRotation(); + myAvatar->setJointRotation(_rig.indexOfJoint("RightHandPinky4"), glm::inverse(previousJointRotation) * thisJointRotation); + } else { + myAvatar->clearJointData(_rig.indexOfJoint("RightHandThumb1")); + myAvatar->clearJointData(_rig.indexOfJoint("RightHandThumb2")); + myAvatar->clearJointData(_rig.indexOfJoint("RightHandThumb3")); + myAvatar->clearJointData(_rig.indexOfJoint("RightHandThumb4")); + myAvatar->clearJointData(_rig.indexOfJoint("RightHandIndex1")); + myAvatar->clearJointData(_rig.indexOfJoint("RightHandIndex2")); + myAvatar->clearJointData(_rig.indexOfJoint("RightHandIndex3")); + myAvatar->clearJointData(_rig.indexOfJoint("RightHandIndex4")); + myAvatar->clearJointData(_rig.indexOfJoint("RightHandMiddle1")); + myAvatar->clearJointData(_rig.indexOfJoint("RightHandMiddle2")); + myAvatar->clearJointData(_rig.indexOfJoint("RightHandMiddle3")); + myAvatar->clearJointData(_rig.indexOfJoint("RightHandMiddle4")); + myAvatar->clearJointData(_rig.indexOfJoint("RightHandRing1")); + myAvatar->clearJointData(_rig.indexOfJoint("RightHandRing2")); + myAvatar->clearJointData(_rig.indexOfJoint("RightHandRing3")); + myAvatar->clearJointData(_rig.indexOfJoint("RightHandRing4")); + myAvatar->clearJointData(_rig.indexOfJoint("RightHandPinky1")); + myAvatar->clearJointData(_rig.indexOfJoint("RightHandPinky2")); + myAvatar->clearJointData(_rig.indexOfJoint("RightHandPinky3")); + myAvatar->clearJointData(_rig.indexOfJoint("RightHandPinky4")); + } + } +} diff --git a/libraries/controllers/src/controllers/Actions.cpp b/libraries/controllers/src/controllers/Actions.cpp index 96e433bcc8..b3c0ed3f05 100644 --- a/libraries/controllers/src/controllers/Actions.cpp +++ b/libraries/controllers/src/controllers/Actions.cpp @@ -59,6 +59,48 @@ namespace controller { makePosePair(Action::SPINE2, "Spine2"), makePosePair(Action::HEAD, "Head"), + makePosePair(Action::LEFT_HAND_THUMB1, "LeftHandThumb1"), + makePosePair(Action::LEFT_HAND_THUMB2, "LeftHandThumb2"), + makePosePair(Action::LEFT_HAND_THUMB3, "LeftHandThumb3"), + makePosePair(Action::LEFT_HAND_THUMB4, "LeftHandThumb4"), + makePosePair(Action::LEFT_HAND_INDEX1, "LeftHandIndex1"), + makePosePair(Action::LEFT_HAND_INDEX2, "LeftHandIndex2"), + makePosePair(Action::LEFT_HAND_INDEX3, "LeftHandIndex3"), + makePosePair(Action::LEFT_HAND_INDEX4, "LeftHandIndex4"), + makePosePair(Action::LEFT_HAND_MIDDLE1, "LeftHandMiddle1"), + makePosePair(Action::LEFT_HAND_MIDDLE2, "LeftHandMiddle2"), + makePosePair(Action::LEFT_HAND_MIDDLE3, "LeftHandMiddle3"), + makePosePair(Action::LEFT_HAND_MIDDLE4, "LeftHandMiddle4"), + makePosePair(Action::LEFT_HAND_RING1, "LeftHandRing1"), + makePosePair(Action::LEFT_HAND_RING2, "LeftHandRing2"), + makePosePair(Action::LEFT_HAND_RING3, "LeftHandRing3"), + makePosePair(Action::LEFT_HAND_RING4, "LeftHandRing4"), + makePosePair(Action::LEFT_HAND_PINKY1, "LeftHandPinky1"), + makePosePair(Action::LEFT_HAND_PINKY2, "LeftHandPinky2"), + makePosePair(Action::LEFT_HAND_PINKY3, "LeftHandPinky3"), + makePosePair(Action::LEFT_HAND_PINKY4, "LeftHandPinky4"), + + makePosePair(Action::RIGHT_HAND_THUMB1, "RightHandThumb1"), + makePosePair(Action::RIGHT_HAND_THUMB2, "RightHandThumb2"), + makePosePair(Action::RIGHT_HAND_THUMB3, "RightHandThumb3"), + makePosePair(Action::RIGHT_HAND_THUMB4, "RightHandThumb4"), + makePosePair(Action::RIGHT_HAND_INDEX1, "RightHandIndex1"), + makePosePair(Action::RIGHT_HAND_INDEX2, "RightHandIndex2"), + makePosePair(Action::RIGHT_HAND_INDEX3, "RightHandIndex3"), + makePosePair(Action::RIGHT_HAND_INDEX4, "RightHandIndex4"), + makePosePair(Action::RIGHT_HAND_MIDDLE1, "RightHandMiddle1"), + makePosePair(Action::RIGHT_HAND_MIDDLE2, "RightHandMiddle2"), + makePosePair(Action::RIGHT_HAND_MIDDLE3, "RightHandMiddle3"), + makePosePair(Action::RIGHT_HAND_MIDDLE4, "RightHandMiddle4"), + makePosePair(Action::RIGHT_HAND_RING1, "RightHandRing1"), + makePosePair(Action::RIGHT_HAND_RING2, "RightHandRing2"), + makePosePair(Action::RIGHT_HAND_RING3, "RightHandRing3"), + makePosePair(Action::RIGHT_HAND_RING4, "RightHandRing4"), + makePosePair(Action::RIGHT_HAND_PINKY1, "RightHandPinky1"), + makePosePair(Action::RIGHT_HAND_PINKY2, "RightHandPinky2"), + makePosePair(Action::RIGHT_HAND_PINKY3, "RightHandPinky3"), + makePosePair(Action::RIGHT_HAND_PINKY4, "RightHandPinky4"), + makeButtonPair(Action::LEFT_HAND_CLICK, "LeftHandClick"), makeButtonPair(Action::RIGHT_HAND_CLICK, "RightHandClick"), diff --git a/libraries/controllers/src/controllers/Actions.h b/libraries/controllers/src/controllers/Actions.h index 2cb500c42a..ec4800c9aa 100644 --- a/libraries/controllers/src/controllers/Actions.h +++ b/libraries/controllers/src/controllers/Actions.h @@ -104,6 +104,47 @@ enum class Action { LEFT_ARM, RIGHT_ARM, + LEFT_HAND_THUMB1, + LEFT_HAND_THUMB2, + LEFT_HAND_THUMB3, + LEFT_HAND_THUMB4, + LEFT_HAND_INDEX1, + LEFT_HAND_INDEX2, + LEFT_HAND_INDEX3, + LEFT_HAND_INDEX4, + LEFT_HAND_MIDDLE1, + LEFT_HAND_MIDDLE2, + LEFT_HAND_MIDDLE3, + LEFT_HAND_MIDDLE4, + LEFT_HAND_RING1, + LEFT_HAND_RING2, + LEFT_HAND_RING3, + LEFT_HAND_RING4, + LEFT_HAND_PINKY1, + LEFT_HAND_PINKY2, + LEFT_HAND_PINKY3, + LEFT_HAND_PINKY4, + + RIGHT_HAND_THUMB1, + RIGHT_HAND_THUMB2, + RIGHT_HAND_THUMB3, + RIGHT_HAND_THUMB4, + RIGHT_HAND_INDEX1, + RIGHT_HAND_INDEX2, + RIGHT_HAND_INDEX3, + RIGHT_HAND_INDEX4, + RIGHT_HAND_MIDDLE1, + RIGHT_HAND_MIDDLE2, + RIGHT_HAND_MIDDLE3, + RIGHT_HAND_MIDDLE4, + RIGHT_HAND_RING1, + RIGHT_HAND_RING2, + RIGHT_HAND_RING3, + RIGHT_HAND_RING4, + RIGHT_HAND_PINKY1, + RIGHT_HAND_PINKY2, + RIGHT_HAND_PINKY3, + RIGHT_HAND_PINKY4, NUM_ACTIONS, }; diff --git a/libraries/controllers/src/controllers/StandardController.cpp b/libraries/controllers/src/controllers/StandardController.cpp index 8e49bb0ebf..40b87bc6b2 100644 --- a/libraries/controllers/src/controllers/StandardController.cpp +++ b/libraries/controllers/src/controllers/StandardController.cpp @@ -101,7 +101,47 @@ Input::NamedVector StandardController::getAvailableInputs() const { // Poses makePair(LEFT_HAND, "LeftHand"), + makePair(LEFT_HAND_THUMB1, "LeftHandThumb1"), + makePair(LEFT_HAND_THUMB2, "LeftHandThumb2"), + makePair(LEFT_HAND_THUMB3, "LeftHandThumb3"), + makePair(LEFT_HAND_THUMB4, "LeftHandThumb4"), + makePair(LEFT_HAND_INDEX1, "LeftHandIndex1"), + makePair(LEFT_HAND_INDEX2, "LeftHandIndex2"), + makePair(LEFT_HAND_INDEX3, "LeftHandIndex3"), + makePair(LEFT_HAND_INDEX4, "LeftHandIndex4"), + makePair(LEFT_HAND_MIDDLE1, "LeftHandMiddle1"), + makePair(LEFT_HAND_MIDDLE2, "LeftHandMiddle2"), + makePair(LEFT_HAND_MIDDLE3, "LeftHandMiddle3"), + makePair(LEFT_HAND_MIDDLE4, "LeftHandMiddle4"), + makePair(LEFT_HAND_RING1, "LeftHandRing1"), + makePair(LEFT_HAND_RING2, "LeftHandRing2"), + makePair(LEFT_HAND_RING3, "LeftHandRing3"), + makePair(LEFT_HAND_RING4, "LeftHandRing4"), + makePair(LEFT_HAND_PINKY1, "LeftHandPinky1"), + makePair(LEFT_HAND_PINKY2, "LeftHandPinky2"), + makePair(LEFT_HAND_PINKY3, "LeftHandPinky3"), + makePair(LEFT_HAND_PINKY4, "LeftHandPinky4"), makePair(RIGHT_HAND, "RightHand"), + makePair(RIGHT_HAND_THUMB1, "RightHandThumb1"), + makePair(RIGHT_HAND_THUMB2, "RightHandThumb2"), + makePair(RIGHT_HAND_THUMB3, "RightHandThumb3"), + makePair(RIGHT_HAND_THUMB4, "RightHandThumb4"), + makePair(RIGHT_HAND_INDEX1, "RightHandIndex1"), + makePair(RIGHT_HAND_INDEX2, "RightHandIndex2"), + makePair(RIGHT_HAND_INDEX3, "RightHandIndex3"), + makePair(RIGHT_HAND_INDEX4, "RightHandIndex4"), + makePair(RIGHT_HAND_MIDDLE1, "RightHandMiddle1"), + makePair(RIGHT_HAND_MIDDLE2, "RightHandMiddle2"), + makePair(RIGHT_HAND_MIDDLE3, "RightHandMiddle3"), + makePair(RIGHT_HAND_MIDDLE4, "RightHandMiddle4"), + makePair(RIGHT_HAND_RING1, "RightHandRing1"), + makePair(RIGHT_HAND_RING2, "RightHandRing2"), + makePair(RIGHT_HAND_RING3, "RightHandRing3"), + makePair(RIGHT_HAND_RING4, "RightHandRing4"), + makePair(RIGHT_HAND_PINKY1, "RightHandPinky1"), + makePair(RIGHT_HAND_PINKY2, "RightHandPinky2"), + makePair(RIGHT_HAND_PINKY3, "RightHandPinky3"), + makePair(RIGHT_HAND_PINKY4, "RightHandPinky4"), makePair(LEFT_FOOT, "LeftFoot"), makePair(RIGHT_FOOT, "RightFoot"), makePair(RIGHT_ARM, "RightArm"), diff --git a/plugins/hifiLeapMotion/src/LeapMotionPlugin.cpp b/plugins/hifiLeapMotion/src/LeapMotionPlugin.cpp index 35e2f8d8e0..bc9ff5745c 100644 --- a/plugins/hifiLeapMotion/src/LeapMotionPlugin.cpp +++ b/plugins/hifiLeapMotion/src/LeapMotionPlugin.cpp @@ -31,13 +31,94 @@ const char* DEFAULT_SENSOR_LOCATION = SENSOR_ON_DESKTOP; enum LeapMotionJointIndex { LeftHand = 0, + LeftHandThumb1, + LeftHandThumb2, + LeftHandThumb3, + LeftHandThumb4, + LeftHandIndex1, + LeftHandIndex2, + LeftHandIndex3, + LeftHandIndex4, + LeftHandMiddle1, + LeftHandMiddle2, + LeftHandMiddle3, + LeftHandMiddle4, + LeftHandRing1, + LeftHandRing2, + LeftHandRing3, + LeftHandRing4, + LeftHandPinky1, + LeftHandPinky2, + LeftHandPinky3, + LeftHandPinky4, RightHand, + RightHandThumb1, + RightHandThumb2, + RightHandThumb3, + RightHandThumb4, + RightHandIndex1, + RightHandIndex2, + RightHandIndex3, + RightHandIndex4, + RightHandMiddle1, + RightHandMiddle2, + RightHandMiddle3, + RightHandMiddle4, + RightHandRing1, + RightHandRing2, + RightHandRing3, + RightHandRing4, + RightHandPinky1, + RightHandPinky2, + RightHandPinky3, + RightHandPinky4, + Size }; static controller::StandardPoseChannel LeapMotionJointIndexToPoseIndexMap[LeapMotionJointIndex::Size] = { controller::LEFT_HAND, - controller::RIGHT_HAND + controller::LEFT_HAND_THUMB1, + controller::LEFT_HAND_THUMB2, + controller::LEFT_HAND_THUMB3, + controller::LEFT_HAND_THUMB4, + controller::LEFT_HAND_INDEX1, + controller::LEFT_HAND_INDEX2, + controller::LEFT_HAND_INDEX3, + controller::LEFT_HAND_INDEX4, + controller::LEFT_HAND_MIDDLE1, + controller::LEFT_HAND_MIDDLE2, + controller::LEFT_HAND_MIDDLE3, + controller::LEFT_HAND_MIDDLE4, + controller::LEFT_HAND_RING1, + controller::LEFT_HAND_RING2, + controller::LEFT_HAND_RING3, + controller::LEFT_HAND_RING4, + controller::LEFT_HAND_PINKY1, + controller::LEFT_HAND_PINKY2, + controller::LEFT_HAND_PINKY3, + controller::LEFT_HAND_PINKY4, + controller::RIGHT_HAND, + controller::RIGHT_HAND_THUMB1, + controller::RIGHT_HAND_THUMB2, + controller::RIGHT_HAND_THUMB3, + controller::RIGHT_HAND_THUMB4, + controller::RIGHT_HAND_INDEX1, + controller::RIGHT_HAND_INDEX2, + controller::RIGHT_HAND_INDEX3, + controller::RIGHT_HAND_INDEX4, + controller::RIGHT_HAND_MIDDLE1, + controller::RIGHT_HAND_MIDDLE2, + controller::RIGHT_HAND_MIDDLE3, + controller::RIGHT_HAND_MIDDLE4, + controller::RIGHT_HAND_RING1, + controller::RIGHT_HAND_RING2, + controller::RIGHT_HAND_RING3, + controller::RIGHT_HAND_RING4, + controller::RIGHT_HAND_PINKY1, + controller::RIGHT_HAND_PINKY2, + controller::RIGHT_HAND_PINKY3, + controller::RIGHT_HAND_PINKY4 }; #define UNKNOWN_JOINT (controller::StandardPoseChannel)0 @@ -374,21 +455,44 @@ glm::vec3 LeapVectorToVec3(const Leap::Vector& vec) { void LeapMotionPlugin::processFrame(const Leap::Frame& frame) { // Default to uncontrolled. - _joints[LeapMotionJointIndex::LeftHand].position = glm::vec3(); - _joints[LeapMotionJointIndex::RightHand].position = glm::vec3(); + for (int i = 0; i < _joints.size(); i++) { + _joints[i].position = glm::vec3(); + } auto hands = frame.hands(); - - for (int i = 0; i < hands.count() && i < 2; i++) { + const int MAX_NUMBER_OF_HANDS = 2; + for (int i = 0; i < hands.count() && i < MAX_NUMBER_OF_HANDS; i++) { auto hand = hands[i]; - auto arm = hand.arm(); - if (hands[i].isLeft()) { - _joints[LeapMotionJointIndex::LeftHand].position = LeapVectorToVec3(hand.wristPosition()); - _joints[LeapMotionJointIndex::LeftHand].orientation = LeapBasisToQuat(LEFT_SIDE_SIGN, hand.basis()); - } else { - _joints[LeapMotionJointIndex::RightHand].position = LeapVectorToVec3(hand.wristPosition()); - _joints[LeapMotionJointIndex::RightHand].orientation = LeapBasisToQuat(RIGHT_SIDE_SIGN, hand.basis()); + int sideSign = hand.isLeft() ? LEFT_SIDE_SIGN : RIGHT_SIDE_SIGN; + int jointIndex = hand.isLeft() ? LeapMotionJointIndex::LeftHand : LeapMotionJointIndex::RightHand; + + // Hand. + _joints[jointIndex].position = LeapVectorToVec3(hand.wristPosition()); + _joints[jointIndex].orientation = LeapBasisToQuat(sideSign, hand.basis()); + + // Fingers. + // Leap Motion SDK guarantees full set of fingers and finger joints so can straightforwardly process them all. + Leap::FingerList fingers = hand.fingers(); + for (int j = Leap::Finger::Type::TYPE_THUMB; j <= Leap::Finger::Type::TYPE_PINKY; j++) { + Leap::Finger finger; + finger = fingers[j]; + Leap::Bone bone; + bone = finger.bone(Leap::Bone::Type::TYPE_PROXIMAL); + jointIndex++; + _joints[jointIndex].position = LeapVectorToVec3(bone.prevJoint()); + _joints[jointIndex].orientation = LeapBasisToQuat(sideSign, bone.basis()); + bone = finger.bone(Leap::Bone::Type::TYPE_INTERMEDIATE); + jointIndex++; + _joints[jointIndex].position = LeapVectorToVec3(bone.prevJoint()); + _joints[jointIndex].orientation = LeapBasisToQuat(sideSign, bone.basis()); + bone = finger.bone(Leap::Bone::Type::TYPE_DISTAL); + jointIndex++; + _joints[jointIndex].position = LeapVectorToVec3(bone.prevJoint()); + _joints[jointIndex].orientation = LeapBasisToQuat(sideSign, bone.basis()); + jointIndex++; + _joints[jointIndex].position = LeapVectorToVec3(bone.nextJoint()); + _joints[jointIndex].orientation = LeapBasisToQuat(sideSign, bone.basis()); } } }