From da023bfbe36d86c52a14b435651160459e3314fe Mon Sep 17 00:00:00 2001 From: "James B. Pollack" Date: Fri, 3 Jun 2016 15:40:04 -0700 Subject: [PATCH] leap --- scripts/system/controllers/leapHands.js | 527 ++++++++++++++++++++++++ 1 file changed, 527 insertions(+) create mode 100644 scripts/system/controllers/leapHands.js diff --git a/scripts/system/controllers/leapHands.js b/scripts/system/controllers/leapHands.js new file mode 100644 index 0000000000..1be0b1e5f6 --- /dev/null +++ b/scripts/system/controllers/leapHands.js @@ -0,0 +1,527 @@ +// +// leapHands.js +// examples +// +// Created by David Rowe on 8 Sep 2014. +// Copyright 2014 High Fidelity, Inc. +// +// This is an example script that uses the Leap Motion to make the avatar's hands replicate the user's hand actions. +// +// Distributed under the Apache License, Version 2.0. +// See the accompanying file LICENSE or http://www.apache.org/licenses/LICENSE-2.0.html +// + +var leftTriggerValue = 0; +var rightTriggerValue = 0; + +var LEAP_TRIGGER_START_ANGLE = 15.0; +var LEAP_TRIGGER_END_ANGLE = 40.0; + +function getLeapMotionLeftTrigger() { + //print("left trigger = " + leftTriggerValue); + return leftTriggerValue; +} +function getLeapMotionRightTrigger() { + //print("right trigger = " + rightTriggerValue); + return rightTriggerValue; +} + +var leapHands = (function () { + + var isOnHMD, + LEAP_ON_HMD_MENU_ITEM = "Leap Motion on HMD", + LEAP_OFFSET = 0.019, // Thickness of Leap Motion plus HMD clip + HMD_OFFSET = 0.070, // Eyeballs to front surface of Oculus DK2 TODO: Confirm and make depend on device and eye relief + hasHandAndWristJoints, + handToWristOffset = [], // For avatars without a wrist joint we control an estimate of a proper hand joint position + HAND_OFFSET = 0.4, // Relative distance of wrist to hand versus wrist to index finger knuckle + handAnimationStateHandlers, + handAnimationStateFunctions, + handAnimationStateProperties, + hands, + wrists, + NUM_HANDS = 2, // 0 = left; 1 = right + fingers, + NUM_FINGERS = 5, // 0 = thumb; ...; 4 = pinky + THUMB = 0, + MIDDLE_FINGER = 2, + NUM_FINGER_JOINTS = 3, // 0 = metacarpal(hand)-proximal(finger) joint; ...; 2 = intermediate-distal joint + MAX_HAND_INACTIVE_COUNT = 20, + calibrationStatus, + UNCALIBRATED = 0, + CALIBRATING = 1, + CALIBRATED = 2, + CALIBRATION_TIME = 1000, // milliseconds + avatarScale, + avatarFaceModelURL, + avatarSkeletonModelURL, + settingsTimer, + HMD_CAMERA_TO_AVATAR_ROTATION = [ + Quat.angleAxis(180.0, { x: 0, y: 0, z: 1 }), + Quat.angleAxis(-180.0, { x: 0, y: 0, z: 1 }) + ], + DESKTOP_CAMERA_TO_AVATAR_ROTATION = + Quat.multiply(Quat.angleAxis(180.0, { x: 0, y: 1, z: 0 }), Quat.angleAxis(90.0, { x: 0, y: 0, z: 1 })), + LEAP_THUMB_ROOT_ADJUST = [Quat.fromPitchYawRollDegrees(0, 0, 20), Quat.fromPitchYawRollDegrees(0, 0, -20)]; + + function printSkeletonJointNames() { + var jointNames, + i; + + print(MyAvatar.skeletonModelURL); + + print("Skeleton joint names ..."); + jointNames = MyAvatar.getJointNames(); + for (i = 0; i < jointNames.length; i += 1) { + print(i + ": " + jointNames[i]); + } + print("... skeleton joint names"); + } + + function animateLeftHand() { + var ROTATION_AND_POSITION = 0; + + return { + leftHandType: ROTATION_AND_POSITION, + leftHandPosition: hands[0].position, + leftHandRotation: hands[0].rotation + }; + } + + function animateRightHand() { + var ROTATION_AND_POSITION = 0; + + return { + rightHandType: ROTATION_AND_POSITION, + rightHandPosition: hands[1].position, + rightHandRotation: hands[1].rotation + }; + } + + function finishCalibration() { + var avatarPosition, + handPosition, + middleFingerPosition, + leapHandHeight, + h; + + if (!isOnHMD) { + if (hands[0].controller.isActive() && hands[1].controller.isActive()) { + leapHandHeight = (hands[0].controller.getAbsTranslation().y + hands[1].controller.getAbsTranslation().y) / 2.0; + } else { + calibrationStatus = UNCALIBRATED; + return; + } + } + + avatarPosition = MyAvatar.position; + + for (h = 0; h < NUM_HANDS; h += 1) { + handPosition = MyAvatar.getJointPosition(hands[h].jointName); + if (!hasHandAndWristJoints) { + middleFingerPosition = MyAvatar.getJointPosition(fingers[h][MIDDLE_FINGER][0].jointName); + handToWristOffset[h] = Vec3.multiply(Vec3.subtract(handPosition, middleFingerPosition), 1.0 - HAND_OFFSET); + } + + if (isOnHMD) { + // Offset of Leap Motion origin from physical eye position + hands[h].zeroPosition = { x: 0.0, y: 0.0, z: HMD_OFFSET + LEAP_OFFSET }; + } else { + hands[h].zeroPosition = { + x: handPosition.x - avatarPosition.x, + y: handPosition.y - avatarPosition.y, + z: avatarPosition.z - handPosition.z + }; + hands[h].zeroPosition = Vec3.multiplyQbyV(MyAvatar.orientation, hands[h].zeroPosition); + hands[h].zeroPosition.y = hands[h].zeroPosition.y - leapHandHeight; + } + } + + MyAvatar.clearJointData("LeftHand"); + MyAvatar.clearJointData("LeftForeArm"); + MyAvatar.clearJointData("RightHand"); + MyAvatar.clearJointData("RightForeArm"); + + calibrationStatus = CALIBRATED; + print("Leap Motion: Calibrated"); + } + + function calibrate() { + var jointNames, + i; + + calibrationStatus = CALIBRATING; + + avatarScale = MyAvatar.scale; + avatarFaceModelURL = MyAvatar.faceModelURL; + avatarSkeletonModelURL = MyAvatar.skeletonModelURL; + + // Does this skeleton have both wrist and hand joints? + hasHandAndWristJoints = false; + jointNames = MyAvatar.getJointNames(); + for (i = 0; i < jointNames.length; i += 1) { + hasHandAndWristJoints = hasHandAndWristJoints || jointNames[i].toLowerCase() === "leftwrist"; + } + + // Set avatar arms vertical, forearms horizontal, as "zero" position for calibration + MyAvatar.setJointRotation("LeftForeArm", Quat.fromPitchYawRollDegrees(0.0, 0.0, 90.0)); + MyAvatar.setJointRotation("LeftHand", Quat.fromPitchYawRollDegrees(0.0, 90.0, 0.0)); + MyAvatar.setJointRotation("RightForeArm", Quat.fromPitchYawRollDegrees(0.0, 0.0, -90.0)); + MyAvatar.setJointRotation("RightHand", Quat.fromPitchYawRollDegrees(0.0, -90.0, 0.0)); + + // Wait for arms to assume their positions before calculating + Script.setTimeout(finishCalibration, CALIBRATION_TIME); + } + + function checkCalibration() { + + if (calibrationStatus === CALIBRATED) { + return true; + } + + if (calibrationStatus !== CALIBRATING) { + calibrate(); + } + + return false; + } + + function setIsOnHMD() { + isOnHMD = Menu.isOptionChecked(LEAP_ON_HMD_MENU_ITEM); + print("Leap Motion: " + (isOnHMD ? "Is on HMD" : "Is on desk")); + } + + function checkSettings() { + if (calibrationStatus > UNCALIBRATED && (MyAvatar.scale !== avatarScale + || MyAvatar.faceModelURL !== avatarFaceModelURL + || MyAvatar.skeletonModelURL !== avatarSkeletonModelURL + || Menu.isOptionChecked(LEAP_ON_HMD_MENU_ITEM) !== isOnHMD)) { + print("Leap Motion: Recalibrate..."); + calibrationStatus = UNCALIBRATED; + + setIsOnHMD(); + } + } + + function setUp() { + + wrists = [ + { + jointName: "LeftWrist", + controller: Controller.createInputController("Spatial", "joint_L_wrist") + }, + { + jointName: "RightWrist", + controller: Controller.createInputController("Spatial", "joint_R_wrist") + } + ]; + + hands = [ + { + jointName: "LeftHand", + controller: Controller.createInputController("Spatial", "joint_L_hand"), + inactiveCount: 0 + }, + { + jointName: "RightHand", + controller: Controller.createInputController("Spatial", "joint_R_hand"), + inactiveCount: 0 + } + ]; + + // The Leap controller's first joint is the hand-metacarpal joint but this joint's data is not used because it's too + // dependent on the model skeleton exactly matching the Leap skeleton; using just the second and subsequent joints + // seems to work better over all. + fingers = [{}, {}]; + fingers[0] = [ + [ + { jointName: "LeftHandThumb1", controller: Controller.createInputController("Spatial", "joint_L_thumb2") }, + { jointName: "LeftHandThumb2", controller: Controller.createInputController("Spatial", "joint_L_thumb3") }, + { jointName: "LeftHandThumb3", controller: Controller.createInputController("Spatial", "joint_L_thumb4") } + ], + [ + { jointName: "LeftHandIndex1", controller: Controller.createInputController("Spatial", "joint_L_index2") }, + { jointName: "LeftHandIndex2", controller: Controller.createInputController("Spatial", "joint_L_index3") }, + { jointName: "LeftHandIndex3", controller: Controller.createInputController("Spatial", "joint_L_index4") } + ], + [ + { jointName: "LeftHandMiddle1", controller: Controller.createInputController("Spatial", "joint_L_middle2") }, + { jointName: "LeftHandMiddle2", controller: Controller.createInputController("Spatial", "joint_L_middle3") }, + { jointName: "LeftHandMiddle3", controller: Controller.createInputController("Spatial", "joint_L_middle4") } + ], + [ + { jointName: "LeftHandRing1", controller: Controller.createInputController("Spatial", "joint_L_ring2") }, + { jointName: "LeftHandRing2", controller: Controller.createInputController("Spatial", "joint_L_ring3") }, + { jointName: "LeftHandRing3", controller: Controller.createInputController("Spatial", "joint_L_ring4") } + ], + [ + { jointName: "LeftHandPinky1", controller: Controller.createInputController("Spatial", "joint_L_pinky2") }, + { jointName: "LeftHandPinky2", controller: Controller.createInputController("Spatial", "joint_L_pinky3") }, + { jointName: "LeftHandPinky3", controller: Controller.createInputController("Spatial", "joint_L_pinky4") } + ] + ]; + fingers[1] = [ + [ + { jointName: "RightHandThumb1", controller: Controller.createInputController("Spatial", "joint_R_thumb2") }, + { jointName: "RightHandThumb2", controller: Controller.createInputController("Spatial", "joint_R_thumb3") }, + { jointName: "RightHandThumb3", controller: Controller.createInputController("Spatial", "joint_R_thumb4") } + ], + [ + { jointName: "RightHandIndex1", controller: Controller.createInputController("Spatial", "joint_R_index2") }, + { jointName: "RightHandIndex2", controller: Controller.createInputController("Spatial", "joint_R_index3") }, + { jointName: "RightHandIndex3", controller: Controller.createInputController("Spatial", "joint_R_index4") } + ], + [ + { jointName: "RightHandMiddle1", controller: Controller.createInputController("Spatial", "joint_R_middle2") }, + { jointName: "RightHandMiddle2", controller: Controller.createInputController("Spatial", "joint_R_middle3") }, + { jointName: "RightHandMiddle3", controller: Controller.createInputController("Spatial", "joint_R_middle4") } + ], + [ + { jointName: "RightHandRing1", controller: Controller.createInputController("Spatial", "joint_R_ring2") }, + { jointName: "RightHandRing2", controller: Controller.createInputController("Spatial", "joint_R_ring3") }, + { jointName: "RightHandRing3", controller: Controller.createInputController("Spatial", "joint_R_ring4") } + ], + [ + { jointName: "RightHandPinky1", controller: Controller.createInputController("Spatial", "joint_R_pinky2") }, + { jointName: "RightHandPinky2", controller: Controller.createInputController("Spatial", "joint_R_pinky3") }, + { jointName: "RightHandPinky3", controller: Controller.createInputController("Spatial", "joint_R_pinky4") } + ] + ]; + + handAnimationStateHandlers = [null, null]; + handAnimationStateFunctions = [animateLeftHand, animateRightHand]; + handAnimationStateProperties = [ + ["leftHandType", "leftHandPosition", "leftHandRotation"], + ["rightHandType", "rightHandPosition", "rightHandPosition"] + ]; + + setIsOnHMD(); + + settingsTimer = Script.setInterval(checkSettings, 2000); + + calibrationStatus = UNCALIBRATED; + + { + var mapping = Controller.newMapping("LeapmotionTrigger"); + mapping.from(getLeapMotionLeftTrigger).to(Controller.Standard.LT); + mapping.from(getLeapMotionRightTrigger).to(Controller.Standard.RT); + mapping.enable(); + } + } + + function moveHands() { + var h, + i, + j, + side, + handOffset, + wristOffset, + handRotation, + locRotation, + cameraOrientation, + inverseAvatarOrientation; + + for (h = 0; h < NUM_HANDS; h += 1) { + side = h === 0 ? -1.0 : 1.0; + + if (hands[h].controller.isActive()) { + + // Calibrate if necessary. + if (!checkCalibration()) { + return; + } + + // Hand animation handlers ... + if (handAnimationStateHandlers[h] === null) { + handAnimationStateHandlers[h] = MyAvatar.addAnimationStateHandler(handAnimationStateFunctions[h], + handAnimationStateProperties[h]); + } + + // Hand position ... + handOffset = hands[h].controller.getAbsTranslation(); + handRotation = hands[h].controller.getAbsRotation(); + + if (isOnHMD) { + + // Adjust to control wrist position if "hand" joint is at wrist ... + if (!hasHandAndWristJoints) { + wristOffset = Vec3.multiplyQbyV(handRotation, handToWristOffset[h]); + handOffset = Vec3.sum(handOffset, wristOffset); + } + + // Hand offset in camera coordinates ... + handOffset = { + x: -handOffset.x, + y: -handOffset.z, + z: -handOffset.y - hands[h].zeroPosition.z + }; + + // Hand offset in world coordinates ... + cameraOrientation = Camera.getOrientation(); + handOffset = Vec3.sum(Camera.getPosition(), Vec3.multiplyQbyV(cameraOrientation, handOffset)); + + // Hand offset in avatar coordinates ... + inverseAvatarOrientation = Quat.inverse(MyAvatar.orientation); + handOffset = Vec3.subtract(handOffset, MyAvatar.position); + handOffset = Vec3.multiplyQbyV(inverseAvatarOrientation, handOffset); + handOffset.z = -handOffset.z; + handOffset.x = -handOffset.x; + + + // Hand rotation in camera coordinates ... + handRotation = { + x: -handRotation.y, + y: -handRotation.z, + z: -handRotation.x, + w: handRotation.w + }; + + // Hand rotation in avatar coordinates ... + handRotation = Quat.multiply(HMD_CAMERA_TO_AVATAR_ROTATION[h], handRotation); + cameraOrientation = { + x: cameraOrientation.z, + y: cameraOrientation.y, + z: cameraOrientation.x, + w: cameraOrientation.w + }; + cameraOrientation = Quat.multiply(cameraOrientation, Quat.inverse(MyAvatar.orientation)); + handRotation = Quat.multiply(handRotation, cameraOrientation); // Works!!! + + } else { + + // Adjust to control wrist position if "hand" joint is at wrist ... + if (!hasHandAndWristJoints) { + wristOffset = Vec3.multiplyQbyV(handRotation, handToWristOffset[h]); + handOffset = Vec3.sum(handOffset, wristOffset); + } + + // Hand offset in camera coordinates ... + handOffset = { + x: -handOffset.x, + y: hands[h].zeroPosition.y + handOffset.y, + z: hands[h].zeroPosition.z - handOffset.z + }; + + // Hand rotation in camera coordinates ... + handRotation = { + x: handRotation.z, + y: handRotation.y, + z: handRotation.x, + w: handRotation.w + }; + + // Hand rotation in avatar coordinates ... + handRotation = Quat.multiply(DESKTOP_CAMERA_TO_AVATAR_ROTATION, handRotation); + } + + // Set hand position and orientation for animation state handler ... + hands[h].position = handOffset; + hands[h].rotation = handRotation; + + // Set finger joints ... + var summed = 0; + var closeAngle = 0; + for (i = 0; i < NUM_FINGERS; i += 1) { + for (j = 0; j < NUM_FINGER_JOINTS; j += 1) { + if (fingers[h][i][j].controller !== null) { + locRotation = fingers[h][i][j].controller.getLocRotation(); + var eulers = Quat.safeEulerAngles(locRotation); + closeAngle += eulers.x; + + summed++; + + if (i === THUMB) { + locRotation = { + x: side * locRotation.y, + y: side * -locRotation.z, + z: side * -locRotation.x, + w: locRotation.w + }; + if (j === 0) { + // Adjust avatar thumb root joint rotation to make avatar hands look better + locRotation = Quat.multiply(LEAP_THUMB_ROOT_ADJUST[h], locRotation); + } + } else { + locRotation = { + x: -locRotation.x, + y: -locRotation.z, + z: -locRotation.y, + w: locRotation.w + }; + } + MyAvatar.setJointRotation(fingers[h][i][j].jointName, locRotation); + } + } + } + + hands[h].inactiveCount = 0; + if (summed > 0) { + closeAngle /= summed; + } + + var triggerValue = (-closeAngle - LEAP_TRIGGER_START_ANGLE) / (LEAP_TRIGGER_END_ANGLE - LEAP_TRIGGER_START_ANGLE); + triggerValue = Math.max(0.0, Math.min(triggerValue, 1.0)); + + if (h == 0) { + leftTriggerValue = triggerValue; + } else { + rightTriggerValue = triggerValue; + + } + + } else { + + if (hands[h].inactiveCount < MAX_HAND_INACTIVE_COUNT) { + + hands[h].inactiveCount += 1; + + if (hands[h].inactiveCount === MAX_HAND_INACTIVE_COUNT) { + if (handAnimationStateHandlers[h] !== null) { + MyAvatar.removeAnimationStateHandler(handAnimationStateHandlers[h]); + handAnimationStateHandlers[h] = null; + leftTriggerValue = 0.0; + rightTriggerValue = 0.0; + } + } + } + } + } + } + + function tearDown() { + var h, + i, + j; + + Script.clearInterval(settingsTimer); + + for (h = 0; h < NUM_HANDS; h += 1) { + Controller.releaseInputController(hands[h].controller); + Controller.releaseInputController(wrists[h].controller); + if (handAnimationStateHandlers[h] !== null) { + MyAvatar.removeAnimationStateHandler(handAnimationStateHandlers[h]); + } + for (i = 0; i < NUM_FINGERS; i += 1) { + for (j = 0; j < NUM_FINGER_JOINTS; j += 1) { + if (fingers[h][i][j].controller !== null) { + Controller.releaseInputController(fingers[h][i][j].controller); + } + } + } + } + } + + return { + printSkeletonJointNames: printSkeletonJointNames, + setUp : setUp, + moveHands : moveHands, + tearDown : tearDown + }; +}()); + + +//leapHands.printSkeletonJointNames(); + +leapHands.setUp(); +Script.update.connect(leapHands.moveHands); +Script.scriptEnding.connect(leapHands.tearDown);