From 16eca79cfa064995cf955a13fc126a4d0821d9e5 Mon Sep 17 00:00:00 2001 From: David Rowe Date: Wed, 10 Sep 2014 14:25:28 -0700 Subject: [PATCH 01/14] Add leapHands.js that makes avatar fingers mirror real finger movements --- examples/leapHands.js | 280 ++++++++++++++++++++++++++++++++++++++++++ 1 file changed, 280 insertions(+) create mode 100644 examples/leapHands.js diff --git a/examples/leapHands.js b/examples/leapHands.js new file mode 100644 index 0000000000..73c66736e0 --- /dev/null +++ b/examples/leapHands.js @@ -0,0 +1,280 @@ +// +// leapHands.js +// examples +// +// Created by David Rowe on 8 Sep 2014. +// Copyright 2014 High Fidelity, Inc. +// +// This is an example script that uses the Leap Motion to make the avatar's hands replicate the user's hand actions. +// +// Distributed under the Apache License, Version 2.0. +// See the accompanying file LICENSE or http://www.apache.org/licenses/LICENSE-2.0.html +// + +var leapHands = (function () { + + var leftHand, + rightHand, + NUM_FINGERS = 5, // 0 = thumb; ...; 4 = pinky + THUMB = 0, + leftHandFingers, + rightHandFingers, + NUM_FINGER_JOINTS = 3, // 0 = hand-finger joint; ...; 2 = fingertip joint + leftHandInactiveCount, + rightHandInactiveCount, + MAX_HAND_INACTIVE_COUNT = 20; + + function printSkeletonJointNames() { + var jointNames, + i; + + print(MyAvatar.skeletonModelURL); + + print("Skeleton joint names ..."); + jointNames = MyAvatar.getJointNames(); + for (i = 0; i < jointNames.length; i += 1) { + print(i + ": " + jointNames[i]); + } + print("... skeleton joint names"); + + /* + http://public.highfidelity.io/models/skeletons/ron_standing.fst + Skeleton joint names ... + 0: Hips + 1: RightUpLeg + 2: RightLeg + 3: RightFoot + 4: RightToeBase + 5: RightToe_End + 6: LeftUpLeg + 7: LeftLeg + 8: LeftFoot + 9: LeftToeBase + 10: LeftToe_End + 11: Spine + 12: Spine1 + 13: Spine2 + 14: RightShoulder + 15: RightArm + 16: RightForeArm + 17: RightHand + 18: RightHandPinky1 + 19: RightHandPinky2 + 20: RightHandPinky3 + 21: RightHandPinky4 + 22: RightHandRing1 + 23: RightHandRing2 + 24: RightHandRing3 + 25: RightHandRing4 + 26: RightHandMiddle1 + 27: RightHandMiddle2 + 28: RightHandMiddle3 + 29: RightHandMiddle4 + 30: RightHandIndex1 + 31: RightHandIndex2 + 32: RightHandIndex3 + 33: RightHandIndex4 + 34: RightHandThumb1 + 35: RightHandThumb2 + 36: RightHandThumb3 + 37: RightHandThumb4 + 38: LeftShoulder + 39: LeftArm + 40: LeftForeArm + 41: LeftHand + 42: LeftHandPinky1 + 43: LeftHandPinky2 + 44: LeftHandPinky3 + 45: LeftHandPinky4 + 46: LeftHandRing1 + 47: LeftHandRing2 + 48: LeftHandRing3 + 49: LeftHandRing4 + 50: LeftHandMiddle1 + 51: LeftHandMiddle2 + 52: LeftHandMiddle3 + 53: LeftHandMiddle4 + 54: LeftHandIndex1 + 55: LeftHandIndex2 + 56: LeftHandIndex3 + 57: LeftHandIndex4 + 58: LeftHandThumb1 + 59: LeftHandThumb2 + 60: LeftHandThumb3 + 61: LeftHandThumb4 + 62: Neck + 63: Head + 64: HeadTop_End + 65: body + ... skeleton joint names + */ + } + + function setUp() { + + leftHand = Controller.createInputController("Spatial", "joint_L_hand"); + leftHandFingers = [ + [ + { jointName: "LeftHandThumb1", controller: Controller.createInputController("Spatial", "joint_L_thumb2") }, + { jointName: "LeftHandThumb2", controller: Controller.createInputController("Spatial", "joint_L_thumb3") }, + { jointName: "LeftHandThumb3", controller: Controller.createInputController("Spatial", "joint_L_thumb4") } + ], + [ + { jointName: "LeftHandIndex1", controller: Controller.createInputController("Spatial", "joint_L_index2") }, + { jointName: "LeftHandIndex2", controller: Controller.createInputController("Spatial", "joint_L_index3") }, + { jointName: "LeftHandIndex3", controller: Controller.createInputController("Spatial", "joint_L_index4") } + ], + [ + { jointName: "LeftHandMiddle1", controller: Controller.createInputController("Spatial", "joint_L_middle2") }, + { jointName: "LeftHandMiddle2", controller: Controller.createInputController("Spatial", "joint_L_middle3") }, + { jointName: "LeftHandMiddle3", controller: Controller.createInputController("Spatial", "joint_L_middle4") } + ], + [ + { jointName: "LeftHandRing1", controller: Controller.createInputController("Spatial", "joint_L_ring2") }, + { jointName: "LeftHandRing2", controller: Controller.createInputController("Spatial", "joint_L_ring3") }, + { jointName: "LeftHandRing3", controller: Controller.createInputController("Spatial", "joint_L_ring4") } + ], + [ + { jointName: "LeftHandPinky1", controller: Controller.createInputController("Spatial", "joint_L_pinky2") }, + { jointName: "LeftHandPinky2", controller: Controller.createInputController("Spatial", "joint_L_pinky3") }, + { jointName: "LeftHandPinky3", controller: Controller.createInputController("Spatial", "joint_L_pinky4") } + ] + ]; + leftHandInactiveCount = MAX_HAND_INACTIVE_COUNT; + + rightHand = Controller.createInputController("Spatial", "joint_R_hand"); + rightHandFingers = [ + [ + { jointName: "RightHandThumb1", controller: Controller.createInputController("Spatial", "joint_R_thumb2") }, + { jointName: "RightHandThumb2", controller: Controller.createInputController("Spatial", "joint_R_thumb3") }, + { jointName: "RightHandThumb3", controller: Controller.createInputController("Spatial", "joint_R_thumb4") } + ], + [ + { jointName: "RightHandIndex1", controller: Controller.createInputController("Spatial", "joint_R_index2") }, + { jointName: "RightHandIndex2", controller: Controller.createInputController("Spatial", "joint_R_index3") }, + { jointName: "RightHandIndex3", controller: Controller.createInputController("Spatial", "joint_R_index4") } + ], + [ + { jointName: "RightHandMiddle1", controller: Controller.createInputController("Spatial", "joint_R_middle2") }, + { jointName: "RightHandMiddle2", controller: Controller.createInputController("Spatial", "joint_R_middle3") }, + { jointName: "RightHandMiddle3", controller: Controller.createInputController("Spatial", "joint_R_middle4") } + ], + [ + { jointName: "RightHandRing1", controller: Controller.createInputController("Spatial", "joint_R_ring2") }, + { jointName: "RightHandRing2", controller: Controller.createInputController("Spatial", "joint_R_ring3") }, + { jointName: "RightHandRing3", controller: Controller.createInputController("Spatial", "joint_R_ring4") } + ], + [ + { jointName: "RightHandPinky1", controller: Controller.createInputController("Spatial", "joint_R_pinky2") }, + { jointName: "RightHandPinky2", controller: Controller.createInputController("Spatial", "joint_R_pinky3") }, + { jointName: "RightHandPinky3", controller: Controller.createInputController("Spatial", "joint_R_pinky4") } + ] + ]; + rightHandInactiveCount = MAX_HAND_INACTIVE_COUNT; + } + + function moveHands() { + var i, + j, + locRotation; + + if (leftHand.isActive()) { + leftHandInactiveCount = 0; + + // Fixed hand location for starters ... + MyAvatar.setJointData("LeftArm", Quat.fromPitchYawRollDegrees(90.0, 0.0, -90.0)); + MyAvatar.setJointData("LeftForeArm", Quat.fromPitchYawRollDegrees(90.0, 0.0, 180.0)); + MyAvatar.setJointData("LeftHand", Quat.fromPitchYawRollDegrees(0.0, 0.0, 0.0)); + + // Finger joints ... + for (i = 0; i < NUM_FINGERS; i += 1) { + for (j = 0; j < NUM_FINGER_JOINTS; j += 1) { + if (leftHandFingers[i][j].controller !== null) { + locRotation = leftHandFingers[i][j].controller.getLocRotation(); + if (i === THUMB) { + MyAvatar.setJointData(leftHandFingers[i][j].jointName, + Quat.fromPitchYawRollRadians(-locRotation.y, -locRotation.z, locRotation.x)); + + } else { + MyAvatar.setJointData(leftHandFingers[i][j].jointName, + Quat.fromPitchYawRollRadians(-locRotation.x, 0.0, -locRotation.y)); + } + } + } + } + + leftHandInactiveCount = 0; + } else { + leftHandInactiveCount += 1; + } + + if (rightHand.isActive()) { + + // Fixed hand location for starters ... + MyAvatar.setJointData("RightArm", Quat.fromPitchYawRollDegrees(90.0, 0.0, 90.0)); + MyAvatar.setJointData("RightForeArm", Quat.fromPitchYawRollDegrees(90.0, 0.0, 180.0)); + MyAvatar.setJointData("RightHand", Quat.fromPitchYawRollDegrees(0.0, 0.0, 0.0)); + + // Finger joints ... + for (i = 0; i < NUM_FINGERS; i += 1) { + for (j = 0; j < NUM_FINGER_JOINTS; j += 1) { + if (rightHandFingers[i][j].controller !== null) { + locRotation = rightHandFingers[i][j].controller.getLocRotation(); + if (i === THUMB) { + MyAvatar.setJointData(rightHandFingers[i][j].jointName, + Quat.fromPitchYawRollRadians(locRotation.y, -locRotation.z, -locRotation.x)); + } else { + MyAvatar.setJointData(rightHandFingers[i][j].jointName, + Quat.fromPitchYawRollRadians(-locRotation.x, 0.0, -locRotation.y)); + } + } + } + } + + rightHandInactiveCount = 0; + } else { + rightHandInactiveCount += 1; + } + + if (leftHandInactiveCount >= MAX_HAND_INACTIVE_COUNT && rightHandInactiveCount >= MAX_HAND_INACTIVE_COUNT) { + MyAvatar.clearJointsData(); + } + } + + function tearDown() { + var i, + j; + + Controller.releaseInputController(leftHand); + for (i = 0; i < NUM_FINGERS; i += 1) { + for (j = 0; j < NUM_FINGER_JOINTS; j += 1) { + if (leftHandFingers[i][j].controller !== null) { + Controller.releaseInputController(leftHandFingers[i][j].controller); + } + } + } + + Controller.releaseInputController(rightHand); + for (i = 0; i < NUM_FINGERS; i += 1) { + for (j = 0; j < NUM_FINGER_JOINTS; j += 1) { + if (rightHandFingers[i][j].controller !== null) { + Controller.releaseInputController(rightHandFingers[i][j].controller); + } + } + } + } + + return { + printSkeletonJointNames: printSkeletonJointNames, + setUp : setUp, + moveHands : moveHands, + tearDown : tearDown + }; +}()); + + +//leapHands.printSkeletonJointNames(); + +leapHands.setUp(); +Script.update.connect(leapHands.moveHands); +Script.scriptEnding.connect(leapHands.tearDown); From b4592aceb67d4fa567087df0c4c05281c93dd1d8 Mon Sep 17 00:00:00 2001 From: David Rowe Date: Wed, 10 Sep 2014 20:44:35 -0700 Subject: [PATCH 02/14] Add interim work-around and TODOs for Leap Motion controller issues --- examples/leapHands.js | 15 +++++++++------ 1 file changed, 9 insertions(+), 6 deletions(-) diff --git a/examples/leapHands.js b/examples/leapHands.js index 73c66736e0..5af3e72756 100644 --- a/examples/leapHands.js +++ b/examples/leapHands.js @@ -19,7 +19,7 @@ var leapHands = (function () { THUMB = 0, leftHandFingers, rightHandFingers, - NUM_FINGER_JOINTS = 3, // 0 = hand-finger joint; ...; 2 = fingertip joint + NUM_FINGER_JOINTS = 3, // 0 = metacarpal(hand)-proximal(finger) joint; ...; 2 = intermediate-distal(tip) joint leftHandInactiveCount, rightHandInactiveCount, MAX_HAND_INACTIVE_COUNT = 20; @@ -112,6 +112,8 @@ var leapHands = (function () { function setUp() { + // TODO: Leap Motion controller joint naming doesn't match up with skeleton joint naming; numbers are out by 1. + leftHand = Controller.createInputController("Spatial", "joint_L_hand"); leftHandFingers = [ [ @@ -187,17 +189,17 @@ var leapHands = (function () { MyAvatar.setJointData("LeftHand", Quat.fromPitchYawRollDegrees(0.0, 0.0, 0.0)); // Finger joints ... + // TODO: 2.0 * scale factors should not be necessary; Leap Motion controller code needs investigating. for (i = 0; i < NUM_FINGERS; i += 1) { for (j = 0; j < NUM_FINGER_JOINTS; j += 1) { if (leftHandFingers[i][j].controller !== null) { locRotation = leftHandFingers[i][j].controller.getLocRotation(); if (i === THUMB) { MyAvatar.setJointData(leftHandFingers[i][j].jointName, - Quat.fromPitchYawRollRadians(-locRotation.y, -locRotation.z, locRotation.x)); - + Quat.fromPitchYawRollRadians(2.0 * -locRotation.y, 2.0 * -locRotation.z, 2.0 * locRotation.x)); } else { MyAvatar.setJointData(leftHandFingers[i][j].jointName, - Quat.fromPitchYawRollRadians(-locRotation.x, 0.0, -locRotation.y)); + Quat.fromPitchYawRollRadians(2.0 * -locRotation.x, 0.0, 2.0 * -locRotation.y)); } } } @@ -216,16 +218,17 @@ var leapHands = (function () { MyAvatar.setJointData("RightHand", Quat.fromPitchYawRollDegrees(0.0, 0.0, 0.0)); // Finger joints ... + // TODO: 2.0 * scale factors should not be necessary; Leap Motion controller code needs investigating. for (i = 0; i < NUM_FINGERS; i += 1) { for (j = 0; j < NUM_FINGER_JOINTS; j += 1) { if (rightHandFingers[i][j].controller !== null) { locRotation = rightHandFingers[i][j].controller.getLocRotation(); if (i === THUMB) { MyAvatar.setJointData(rightHandFingers[i][j].jointName, - Quat.fromPitchYawRollRadians(locRotation.y, -locRotation.z, -locRotation.x)); + Quat.fromPitchYawRollRadians(2.0 * locRotation.y, 2.0 * -locRotation.z, 2.0 * -locRotation.x)); } else { MyAvatar.setJointData(rightHandFingers[i][j].jointName, - Quat.fromPitchYawRollRadians(-locRotation.x, 0.0, -locRotation.y)); + Quat.fromPitchYawRollRadians(2.0 * -locRotation.x, 0.0, 2.0 * -locRotation.y)); } } } From c7b34b4f54dcf385cad6433a463e4e965c608356 Mon Sep 17 00:00:00 2001 From: David Rowe Date: Wed, 10 Sep 2014 22:29:40 -0700 Subject: [PATCH 03/14] Lower left and right hands independently when no longer being tracked --- examples/leapHands.js | 14 +++++++++++--- interface/src/avatar/MyAvatar.cpp | 1 + 2 files changed, 12 insertions(+), 3 deletions(-) diff --git a/examples/leapHands.js b/examples/leapHands.js index 5af3e72756..71f4c4449b 100644 --- a/examples/leapHands.js +++ b/examples/leapHands.js @@ -208,6 +208,12 @@ var leapHands = (function () { leftHandInactiveCount = 0; } else { leftHandInactiveCount += 1; + + if (leftHandInactiveCount === MAX_HAND_INACTIVE_COUNT) { + MyAvatar.clearJointData("LeftHand"); + MyAvatar.clearJointData("LeftForeArm"); + MyAvatar.clearJointData("LeftArm"); + } } if (rightHand.isActive()) { @@ -237,10 +243,12 @@ var leapHands = (function () { rightHandInactiveCount = 0; } else { rightHandInactiveCount += 1; - } - if (leftHandInactiveCount >= MAX_HAND_INACTIVE_COUNT && rightHandInactiveCount >= MAX_HAND_INACTIVE_COUNT) { - MyAvatar.clearJointsData(); + if (rightHandInactiveCount === MAX_HAND_INACTIVE_COUNT) { + MyAvatar.clearJointData("RightHand"); + MyAvatar.clearJointData("RightForeArm"); + MyAvatar.clearJointData("RightArm"); + } } } diff --git a/interface/src/avatar/MyAvatar.cpp b/interface/src/avatar/MyAvatar.cpp index 48d58fb02c..1da046d5fc 100644 --- a/interface/src/avatar/MyAvatar.cpp +++ b/interface/src/avatar/MyAvatar.cpp @@ -997,6 +997,7 @@ void MyAvatar::clearJointData(int index) { if (QThread::currentThread() == thread()) { // HACK: ATM only JS scripts call clearJointData() on MyAvatar so we hardcode the priority _skeletonModel.setJointState(index, false, glm::quat(), 0.0f); + _skeletonModel.clearJointAnimationPriority(index); } } From 1f159a5ce52441366a12dfdd94a3ed36c2c817ee Mon Sep 17 00:00:00 2001 From: David Rowe Date: Wed, 10 Sep 2014 23:17:33 -0700 Subject: [PATCH 04/14] Refactor w.r.t. hands --- examples/leapHands.js | 149 ++++++++++++++++++------------------------ 1 file changed, 63 insertions(+), 86 deletions(-) diff --git a/examples/leapHands.js b/examples/leapHands.js index 71f4c4449b..346c9daede 100644 --- a/examples/leapHands.js +++ b/examples/leapHands.js @@ -13,15 +13,12 @@ var leapHands = (function () { - var leftHand, - rightHand, + var hands, + NUM_HANDS = 2, // 0 = left; 1 = right + fingers, NUM_FINGERS = 5, // 0 = thumb; ...; 4 = pinky THUMB = 0, - leftHandFingers, - rightHandFingers, NUM_FINGER_JOINTS = 3, // 0 = metacarpal(hand)-proximal(finger) joint; ...; 2 = intermediate-distal(tip) joint - leftHandInactiveCount, - rightHandInactiveCount, MAX_HAND_INACTIVE_COUNT = 20; function printSkeletonJointNames() { @@ -114,8 +111,13 @@ var leapHands = (function () { // TODO: Leap Motion controller joint naming doesn't match up with skeleton joint naming; numbers are out by 1. - leftHand = Controller.createInputController("Spatial", "joint_L_hand"); - leftHandFingers = [ + hands = [ + { controller: Controller.createInputController("Spatial", "joint_L_hand"), inactiveCount: 0 }, + { controller: Controller.createInputController("Spatial", "joint_R_hand"), inactiveCount: 0 } + ]; + + fingers = [ {}, {} ]; + fingers[0] = [ [ { jointName: "LeftHandThumb1", controller: Controller.createInputController("Spatial", "joint_L_thumb2") }, { jointName: "LeftHandThumb2", controller: Controller.createInputController("Spatial", "joint_L_thumb3") }, @@ -142,10 +144,7 @@ var leapHands = (function () { { jointName: "LeftHandPinky3", controller: Controller.createInputController("Spatial", "joint_L_pinky4") } ] ]; - leftHandInactiveCount = MAX_HAND_INACTIVE_COUNT; - - rightHand = Controller.createInputController("Spatial", "joint_R_hand"); - rightHandFingers = [ + fingers[1] = [ [ { jointName: "RightHandThumb1", controller: Controller.createInputController("Spatial", "joint_R_thumb2") }, { jointName: "RightHandThumb2", controller: Controller.createInputController("Spatial", "joint_R_thumb3") }, @@ -172,104 +171,82 @@ var leapHands = (function () { { jointName: "RightHandPinky3", controller: Controller.createInputController("Spatial", "joint_R_pinky4") } ] ]; - rightHandInactiveCount = MAX_HAND_INACTIVE_COUNT; } function moveHands() { - var i, + var h, + i, j, + side, locRotation; - if (leftHand.isActive()) { - leftHandInactiveCount = 0; + for (h = 0; h < NUM_HANDS; h += 1) { + side = h === 0 ? -1.0 : 1.0; - // Fixed hand location for starters ... - MyAvatar.setJointData("LeftArm", Quat.fromPitchYawRollDegrees(90.0, 0.0, -90.0)); - MyAvatar.setJointData("LeftForeArm", Quat.fromPitchYawRollDegrees(90.0, 0.0, 180.0)); - MyAvatar.setJointData("LeftHand", Quat.fromPitchYawRollDegrees(0.0, 0.0, 0.0)); + if (hands[h].controller.isActive()) { - // Finger joints ... - // TODO: 2.0 * scale factors should not be necessary; Leap Motion controller code needs investigating. - for (i = 0; i < NUM_FINGERS; i += 1) { - for (j = 0; j < NUM_FINGER_JOINTS; j += 1) { - if (leftHandFingers[i][j].controller !== null) { - locRotation = leftHandFingers[i][j].controller.getLocRotation(); - if (i === THUMB) { - MyAvatar.setJointData(leftHandFingers[i][j].jointName, - Quat.fromPitchYawRollRadians(2.0 * -locRotation.y, 2.0 * -locRotation.z, 2.0 * locRotation.x)); - } else { - MyAvatar.setJointData(leftHandFingers[i][j].jointName, - Quat.fromPitchYawRollRadians(2.0 * -locRotation.x, 0.0, 2.0 * -locRotation.y)); + // Fixed hand location for starters ... + if (h === 0) { + MyAvatar.setJointData("LeftArm", Quat.fromPitchYawRollDegrees(90.0, 0.0, -90.0)); + MyAvatar.setJointData("LeftForeArm", Quat.fromPitchYawRollDegrees(90.0, 0.0, 180.0)); + MyAvatar.setJointData("LeftHand", Quat.fromPitchYawRollDegrees(0.0, 0.0, 0.0)); + } else { + MyAvatar.setJointData("RightArm", Quat.fromPitchYawRollDegrees(90.0, 0.0, 90.0)); + MyAvatar.setJointData("RightForeArm", Quat.fromPitchYawRollDegrees(90.0, 0.0, 180.0)); + MyAvatar.setJointData("RightHand", Quat.fromPitchYawRollDegrees(0.0, 0.0, 0.0)); + } + + // Finger joints ... + // TODO: 2.0 * scale factors should not be necessary; Leap Motion controller code needs investigating. + for (i = 0; i < NUM_FINGERS; i += 1) { + for (j = 0; j < NUM_FINGER_JOINTS; j += 1) { + if (fingers[h][i][j].controller !== null) { + locRotation = fingers[h][i][j].controller.getLocRotation(); + if (i === THUMB) { + MyAvatar.setJointData(fingers[h][i][j].jointName, + Quat.fromPitchYawRollRadians(2.0 * side * locRotation.y, 2.0 * -locRotation.z, + 2.0 * side * -locRotation.x)); + } else { + MyAvatar.setJointData(fingers[h][i][j].jointName, + Quat.fromPitchYawRollRadians(2.0 * -locRotation.x, 0.0, 2.0 * -locRotation.y)); + } } } } - } - leftHandInactiveCount = 0; - } else { - leftHandInactiveCount += 1; + hands[h].inactiveCount = 0; - if (leftHandInactiveCount === MAX_HAND_INACTIVE_COUNT) { - MyAvatar.clearJointData("LeftHand"); - MyAvatar.clearJointData("LeftForeArm"); - MyAvatar.clearJointData("LeftArm"); - } - } + } else { - if (rightHand.isActive()) { + hands[h].inactiveCount += 1; - // Fixed hand location for starters ... - MyAvatar.setJointData("RightArm", Quat.fromPitchYawRollDegrees(90.0, 0.0, 90.0)); - MyAvatar.setJointData("RightForeArm", Quat.fromPitchYawRollDegrees(90.0, 0.0, 180.0)); - MyAvatar.setJointData("RightHand", Quat.fromPitchYawRollDegrees(0.0, 0.0, 0.0)); - - // Finger joints ... - // TODO: 2.0 * scale factors should not be necessary; Leap Motion controller code needs investigating. - for (i = 0; i < NUM_FINGERS; i += 1) { - for (j = 0; j < NUM_FINGER_JOINTS; j += 1) { - if (rightHandFingers[i][j].controller !== null) { - locRotation = rightHandFingers[i][j].controller.getLocRotation(); - if (i === THUMB) { - MyAvatar.setJointData(rightHandFingers[i][j].jointName, - Quat.fromPitchYawRollRadians(2.0 * locRotation.y, 2.0 * -locRotation.z, 2.0 * -locRotation.x)); - } else { - MyAvatar.setJointData(rightHandFingers[i][j].jointName, - Quat.fromPitchYawRollRadians(2.0 * -locRotation.x, 0.0, 2.0 * -locRotation.y)); - } + if (hands[h].inactiveCount === MAX_HAND_INACTIVE_COUNT) { + if (h === 0) { + MyAvatar.clearJointData("LeftHand"); + MyAvatar.clearJointData("LeftForeArm"); + MyAvatar.clearJointData("LeftArm"); + } else { + MyAvatar.clearJointData("RightHand"); + MyAvatar.clearJointData("RightForeArm"); + MyAvatar.clearJointData("RightArm"); } } } - - rightHandInactiveCount = 0; - } else { - rightHandInactiveCount += 1; - - if (rightHandInactiveCount === MAX_HAND_INACTIVE_COUNT) { - MyAvatar.clearJointData("RightHand"); - MyAvatar.clearJointData("RightForeArm"); - MyAvatar.clearJointData("RightArm"); - } } } function tearDown() { - var i, + var h, + i, j; - Controller.releaseInputController(leftHand); - for (i = 0; i < NUM_FINGERS; i += 1) { - for (j = 0; j < NUM_FINGER_JOINTS; j += 1) { - if (leftHandFingers[i][j].controller !== null) { - Controller.releaseInputController(leftHandFingers[i][j].controller); - } - } - } - - Controller.releaseInputController(rightHand); - for (i = 0; i < NUM_FINGERS; i += 1) { - for (j = 0; j < NUM_FINGER_JOINTS; j += 1) { - if (rightHandFingers[i][j].controller !== null) { - Controller.releaseInputController(rightHandFingers[i][j].controller); + for (h = 0; h < NUM_HANDS; h += 1) { + Controller.releaseInputController(hands[h].controller); + for (i = 0; i < NUM_FINGERS; i += 1) { + for (j = 0; j < NUM_FINGER_JOINTS; j += 1) { + if (fingers[h][i][j].controller !== null) { + Controller.releaseInputController(fingers[h][i][j].controller); + } } } } From be20ec9f4afa8a65205097269538275f6a36342f Mon Sep 17 00:00:00 2001 From: David Rowe Date: Thu, 11 Sep 2014 09:18:37 -0700 Subject: [PATCH 05/14] Construct controllers in inactive state --- interface/src/scripting/ControllerScriptingInterface.cpp | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/interface/src/scripting/ControllerScriptingInterface.cpp b/interface/src/scripting/ControllerScriptingInterface.cpp index 50d07c1108..f2e65a6e28 100644 --- a/interface/src/scripting/ControllerScriptingInterface.cpp +++ b/interface/src/scripting/ControllerScriptingInterface.cpp @@ -312,7 +312,8 @@ void ControllerScriptingInterface::updateInputControllers() { InputController::InputController(int deviceTrackerId, int subTrackerId, QObject* parent) : AbstractInputController(), _deviceTrackerId(deviceTrackerId), - _subTrackerId(subTrackerId) + _subTrackerId(subTrackerId), + _isActive(false) { } From 1b48d5c040812cef35bb120ce7dc33d2699297f6 Mon Sep 17 00:00:00 2001 From: David Rowe Date: Thu, 11 Sep 2014 20:14:07 -0700 Subject: [PATCH 06/14] Add hand pitch, roll, and yaw from Leap Motion --- examples/leapHands.js | 26 +++++++++++++++++++++----- 1 file changed, 21 insertions(+), 5 deletions(-) diff --git a/examples/leapHands.js b/examples/leapHands.js index 346c9daede..37d8ed6822 100644 --- a/examples/leapHands.js +++ b/examples/leapHands.js @@ -14,12 +14,14 @@ var leapHands = (function () { var hands, + wrists, NUM_HANDS = 2, // 0 = left; 1 = right fingers, NUM_FINGERS = 5, // 0 = thumb; ...; 4 = pinky THUMB = 0, NUM_FINGER_JOINTS = 3, // 0 = metacarpal(hand)-proximal(finger) joint; ...; 2 = intermediate-distal(tip) joint - MAX_HAND_INACTIVE_COUNT = 20; + MAX_HAND_INACTIVE_COUNT = 20, + PI = 3.141593; function printSkeletonJointNames() { var jointNames, @@ -116,7 +118,12 @@ var leapHands = (function () { { controller: Controller.createInputController("Spatial", "joint_R_hand"), inactiveCount: 0 } ]; - fingers = [ {}, {} ]; + wrists = [ + { controller: Controller.createInputController("Spatial", "joint_L_wrist") }, + { controller: Controller.createInputController("Spatial", "joint_R_wrist") } + ] + + fingers = [{}, {}]; fingers[0] = [ [ { jointName: "LeftHandThumb1", controller: Controller.createInputController("Spatial", "joint_L_thumb2") }, @@ -178,22 +185,30 @@ var leapHands = (function () { i, j, side, - locRotation; + locRotation, + handRoll, + handPitch, + handYaw; for (h = 0; h < NUM_HANDS; h += 1) { side = h === 0 ? -1.0 : 1.0; if (hands[h].controller.isActive()) { + // TODO: 2.0* scale factor should not be necessary; Leap Motion controller code needs investigating. + handRoll = 2.0 * -hands[h].controller.getAbsRotation().z; + handPitch = 2.0 * -wrists[h].controller.getAbsRotation().x; + handYaw = 2.0 * -wrists[h].controller.getAbsRotation().y; + // Fixed hand location for starters ... if (h === 0) { MyAvatar.setJointData("LeftArm", Quat.fromPitchYawRollDegrees(90.0, 0.0, -90.0)); MyAvatar.setJointData("LeftForeArm", Quat.fromPitchYawRollDegrees(90.0, 0.0, 180.0)); - MyAvatar.setJointData("LeftHand", Quat.fromPitchYawRollDegrees(0.0, 0.0, 0.0)); + MyAvatar.setJointData("LeftHand", Quat.fromPitchYawRollRadians(handPitch, handRoll, handYaw)); } else { MyAvatar.setJointData("RightArm", Quat.fromPitchYawRollDegrees(90.0, 0.0, 90.0)); MyAvatar.setJointData("RightForeArm", Quat.fromPitchYawRollDegrees(90.0, 0.0, 180.0)); - MyAvatar.setJointData("RightHand", Quat.fromPitchYawRollDegrees(0.0, 0.0, 0.0)); + MyAvatar.setJointData("RightHand", Quat.fromPitchYawRollRadians(handPitch, PI / 2.0 + handRoll, handYaw)); } // Finger joints ... @@ -242,6 +257,7 @@ var leapHands = (function () { for (h = 0; h < NUM_HANDS; h += 1) { Controller.releaseInputController(hands[h].controller); + Controller.releaseInputController(wrists[h].controller); for (i = 0; i < NUM_FINGERS; i += 1) { for (j = 0; j < NUM_FINGER_JOINTS; j += 1) { if (fingers[h][i][j].controller !== null) { From 4382b6261318cea3e92f183a11e305855b7c7b32 Mon Sep 17 00:00:00 2001 From: David Rowe Date: Sun, 14 Sep 2014 14:02:03 -0700 Subject: [PATCH 07/14] Add TODO for reviewing Leap Motion right hand roll calculation --- examples/leapHands.js | 7 ++++++- 1 file changed, 6 insertions(+), 1 deletion(-) diff --git a/examples/leapHands.js b/examples/leapHands.js index 37d8ed6822..b44c21401e 100644 --- a/examples/leapHands.js +++ b/examples/leapHands.js @@ -200,6 +200,11 @@ var leapHands = (function () { handPitch = 2.0 * -wrists[h].controller.getAbsRotation().x; handYaw = 2.0 * -wrists[h].controller.getAbsRotation().y; + // TODO: Leap Motion controller's right-hand roll calculation is off by 90 degrees. + if (h === 1) { + handRoll = handRoll + PI / 2.0; + } + // Fixed hand location for starters ... if (h === 0) { MyAvatar.setJointData("LeftArm", Quat.fromPitchYawRollDegrees(90.0, 0.0, -90.0)); @@ -208,7 +213,7 @@ var leapHands = (function () { } else { MyAvatar.setJointData("RightArm", Quat.fromPitchYawRollDegrees(90.0, 0.0, 90.0)); MyAvatar.setJointData("RightForeArm", Quat.fromPitchYawRollDegrees(90.0, 0.0, 180.0)); - MyAvatar.setJointData("RightHand", Quat.fromPitchYawRollRadians(handPitch, PI / 2.0 + handRoll, handYaw)); + MyAvatar.setJointData("RightHand", Quat.fromPitchYawRollRadians(handPitch, handRoll, handYaw)); } // Finger joints ... From 3ddd5854cdcb8b4b0e1f487166f65a4a6668cc65 Mon Sep 17 00:00:00 2001 From: David Rowe Date: Sun, 14 Sep 2014 15:03:22 -0700 Subject: [PATCH 08/14] Position avatar hands with Leap Motion with no roll, pitch or yaw --- examples/leapHands.js | 53 +++++++++++++++++++++++---------- interface/src/avatar/Avatar.cpp | 32 ++++++++++++++++++++ interface/src/avatar/Avatar.h | 5 ++++ interface/src/renderer/Model.h | 24 +++++++-------- 4 files changed, 87 insertions(+), 27 deletions(-) diff --git a/examples/leapHands.js b/examples/leapHands.js index b44c21401e..64d6708cc0 100644 --- a/examples/leapHands.js +++ b/examples/leapHands.js @@ -21,6 +21,7 @@ var leapHands = (function () { THUMB = 0, NUM_FINGER_JOINTS = 3, // 0 = metacarpal(hand)-proximal(finger) joint; ...; 2 = intermediate-distal(tip) joint MAX_HAND_INACTIVE_COUNT = 20, + LEAP_HEIGHT_OFFSET = 0.15, PI = 3.141593; function printSkeletonJointNames() { @@ -114,14 +115,22 @@ var leapHands = (function () { // TODO: Leap Motion controller joint naming doesn't match up with skeleton joint naming; numbers are out by 1. hands = [ - { controller: Controller.createInputController("Spatial", "joint_L_hand"), inactiveCount: 0 }, - { controller: Controller.createInputController("Spatial", "joint_R_hand"), inactiveCount: 0 } + { + jointName: "LeftHand", + controller: Controller.createInputController("Spatial", "joint_L_hand"), + inactiveCount: 0 + }, + { + jointName: "RightHand", + controller: Controller.createInputController("Spatial", "joint_R_hand"), + inactiveCount: 0 + } ]; wrists = [ { controller: Controller.createInputController("Spatial", "joint_L_wrist") }, { controller: Controller.createInputController("Spatial", "joint_R_wrist") } - ] + ]; fingers = [{}, {}]; fingers[0] = [ @@ -178,6 +187,19 @@ var leapHands = (function () { { jointName: "RightHandPinky3", controller: Controller.createInputController("Spatial", "joint_R_pinky4") } ] ]; + + // TODO: Add automatic calibration of avatar's hands. + // Calibration values for Ron. + hands[0].zeroPosition = { + x: -0.157, + y: 0.204, + z: 0.336 + }; + hands[1].zeroPosition = { + x: 0.234, + y: 0.204, + z: 0.339 + }; } function moveHands() { @@ -185,16 +207,25 @@ var leapHands = (function () { i, j, side, - locRotation, + handOffset, handRoll, handPitch, - handYaw; + handYaw, + locRotation; for (h = 0; h < NUM_HANDS; h += 1) { side = h === 0 ? -1.0 : 1.0; if (hands[h].controller.isActive()) { + // Hand position ... + handOffset = hands[h].controller.getAbsTranslation(); + handOffset = { + x: -handOffset.x, + y: hands[h].zeroPosition.y + handOffset.y - LEAP_HEIGHT_OFFSET, + z: hands[h].zeroPosition.z - handOffset.z + }; + // TODO: 2.0* scale factor should not be necessary; Leap Motion controller code needs investigating. handRoll = 2.0 * -hands[h].controller.getAbsRotation().z; handPitch = 2.0 * -wrists[h].controller.getAbsRotation().x; @@ -205,16 +236,8 @@ var leapHands = (function () { handRoll = handRoll + PI / 2.0; } - // Fixed hand location for starters ... - if (h === 0) { - MyAvatar.setJointData("LeftArm", Quat.fromPitchYawRollDegrees(90.0, 0.0, -90.0)); - MyAvatar.setJointData("LeftForeArm", Quat.fromPitchYawRollDegrees(90.0, 0.0, 180.0)); - MyAvatar.setJointData("LeftHand", Quat.fromPitchYawRollRadians(handPitch, handRoll, handYaw)); - } else { - MyAvatar.setJointData("RightArm", Quat.fromPitchYawRollDegrees(90.0, 0.0, 90.0)); - MyAvatar.setJointData("RightForeArm", Quat.fromPitchYawRollDegrees(90.0, 0.0, 180.0)); - MyAvatar.setJointData("RightHand", Quat.fromPitchYawRollRadians(handPitch, handRoll, handYaw)); - } + // Hand position ... + MyAvatar.setJointModelPositionAndOrientation(hands[h].jointName, handOffset); // Finger joints ... // TODO: 2.0 * scale factors should not be necessary; Leap Motion controller code needs investigating. diff --git a/interface/src/avatar/Avatar.cpp b/interface/src/avatar/Avatar.cpp index c7d69cff7a..301020f1c5 100644 --- a/interface/src/avatar/Avatar.cpp +++ b/interface/src/avatar/Avatar.cpp @@ -840,6 +840,38 @@ glm::quat Avatar::getJointCombinedRotation(const QString& name) const { return rotation; } +bool Avatar::setJointModelPositionAndOrientation(int index, glm::vec3 position, const glm::quat& rotation, + bool useRotation) { + bool success; + if (QThread::currentThread() != thread()) { + QMetaObject::invokeMethod(const_cast(this), "setJointModelPositionAndOrientation", + Qt::BlockingQueuedConnection, Q_RETURN_ARG(bool, success), Q_ARG(const int, index), + Q_ARG(const glm::vec3, position), + Q_ARG(const glm::quat&, rotation), Q_ARG(bool, useRotation)); + } else { + qDebug() << "setJointModelPositionAndOrientation()"; + success = _skeletonModel.setJointPosition(index, position, rotation, useRotation, -1, true, + glm::vec3(0.0, -1.0, 0.0), DEFAULT_PRIORITY + 2.0f); + } + return success; +} + +bool Avatar::setJointModelPositionAndOrientation(const QString& name, glm::vec3 position, const glm::quat& rotation, + bool useRotation) { + bool success; + if (QThread::currentThread() != thread()) { + QMetaObject::invokeMethod(const_cast(this), "setJointModelPositionAndOrientation", + Qt::BlockingQueuedConnection, Q_RETURN_ARG(bool, success), Q_ARG(const QString&, name), + Q_ARG(const glm::vec3, position), + Q_ARG(const glm::quat&, rotation), Q_ARG(bool, useRotation)); + } else { + qDebug() << "setJointModelPositionAndOrientation()"; + success = _skeletonModel.setJointPosition(getJointIndex(name), position, rotation, useRotation, -1, true, + glm::vec3(0.0, -1.0, 0.0), DEFAULT_PRIORITY + 2.0f); + } + return success; +} + void Avatar::scaleVectorRelativeToPosition(glm::vec3 &positionToScale) const { //Scale a world space vector as if it was relative to the position positionToScale = _position + _scale * (positionToScale - _position); diff --git a/interface/src/avatar/Avatar.h b/interface/src/avatar/Avatar.h index cf51ffb8b8..3a29f04d4c 100755 --- a/interface/src/avatar/Avatar.h +++ b/interface/src/avatar/Avatar.h @@ -151,6 +151,11 @@ public: Q_INVOKABLE glm::quat getJointCombinedRotation(int index) const; Q_INVOKABLE glm::quat getJointCombinedRotation(const QString& name) const; + Q_INVOKABLE bool setJointModelPositionAndOrientation(int index, const glm::vec3 position, + const glm::quat& rotation = glm::quat(), bool useRotation = false); + Q_INVOKABLE bool setJointModelPositionAndOrientation(const QString& name, const glm::vec3 position, + const glm::quat& rotation = glm::quat(), bool useRotation = false); + Q_INVOKABLE glm::vec3 getVelocity() const { return _velocity; } Q_INVOKABLE glm::vec3 getAcceleration() const { return _acceleration; } Q_INVOKABLE glm::vec3 getAngularVelocity() const { return _angularVelocity; } diff --git a/interface/src/renderer/Model.h b/interface/src/renderer/Model.h index c41fd7d5de..39090c7e1b 100644 --- a/interface/src/renderer/Model.h +++ b/interface/src/renderer/Model.h @@ -183,6 +183,18 @@ public: QVector& getJointStates() { return _jointStates; } const QVector& getJointStates() const { return _jointStates; } + /// \param jointIndex index of joint in model structure + /// \param position position of joint in model-frame + /// \param rotation rotation of joint in model-frame + /// \param useRotation false if rotation should be ignored + /// \param lastFreeIndex + /// \param allIntermediatesFree + /// \param alignment + /// \return true if joint exists + bool setJointPosition(int jointIndex, const glm::vec3& position, const glm::quat& rotation = glm::quat(), + bool useRotation = false, int lastFreeIndex = -1, bool allIntermediatesFree = false, + const glm::vec3& alignment = glm::vec3(0.0f, -1.0f, 0.0f), float priority = 1.0f); + protected: QSharedPointer _geometry; @@ -226,18 +238,6 @@ protected: virtual void updateVisibleJointStates(); - /// \param jointIndex index of joint in model structure - /// \param position position of joint in model-frame - /// \param rotation rotation of joint in model-frame - /// \param useRotation false if rotation should be ignored - /// \param lastFreeIndex - /// \param allIntermediatesFree - /// \param alignment - /// \return true if joint exists - bool setJointPosition(int jointIndex, const glm::vec3& position, const glm::quat& rotation = glm::quat(), - bool useRotation = false, int lastFreeIndex = -1, bool allIntermediatesFree = false, - const glm::vec3& alignment = glm::vec3(0.0f, -1.0f, 0.0f), float priority = 1.0f); - void inverseKinematics(int jointIndex, glm::vec3 position, const glm::quat& rotation, float priority); /// Restores the indexed joint to its default position. From 1527c40e1daf5322b22e521d8d6baa6b5213027e Mon Sep 17 00:00:00 2001 From: David Rowe Date: Sun, 14 Sep 2014 15:25:02 -0700 Subject: [PATCH 09/14] Add hand roll, pitch, and yaw --- examples/leapHands.js | 14 ++++++++++++-- 1 file changed, 12 insertions(+), 2 deletions(-) diff --git a/examples/leapHands.js b/examples/leapHands.js index 64d6708cc0..db4988dcc4 100644 --- a/examples/leapHands.js +++ b/examples/leapHands.js @@ -236,8 +236,18 @@ var leapHands = (function () { handRoll = handRoll + PI / 2.0; } - // Hand position ... - MyAvatar.setJointModelPositionAndOrientation(hands[h].jointName, handOffset); + // Hand position and orientation ... + if (h === 0) { + MyAvatar.setJointModelPositionAndOrientation(hands[h].jointName, handOffset, Quat.fromPitchYawRollRadians( + handPitch, + -PI / 2.0 - handYaw, + handRoll), true); + } else { + MyAvatar.setJointModelPositionAndOrientation(hands[h].jointName, handOffset, Quat.fromPitchYawRollRadians( + handPitch, + PI / 2.0 - handYaw, + handRoll), true); + } // Finger joints ... // TODO: 2.0 * scale factors should not be necessary; Leap Motion controller code needs investigating. From a594136a9af9494749984d644f08f13a188e2ef9 Mon Sep 17 00:00:00 2001 From: David Rowe Date: Mon, 15 Sep 2014 20:12:13 -0700 Subject: [PATCH 10/14] Fix up hand roll, pitch, and yaw as best able for now --- examples/leapHands.js | 23 ++++++++++++----------- interface/src/avatar/Avatar.cpp | 30 ++++++++++-------------------- interface/src/avatar/Avatar.h | 7 +++---- interface/src/renderer/Model.h | 26 +++++++++++++------------- 4 files changed, 38 insertions(+), 48 deletions(-) diff --git a/examples/leapHands.js b/examples/leapHands.js index db4988dcc4..08d4a918bf 100644 --- a/examples/leapHands.js +++ b/examples/leapHands.js @@ -211,6 +211,7 @@ var leapHands = (function () { handRoll, handPitch, handYaw, + handRotation, locRotation; for (h = 0; h < NUM_HANDS; h += 1) { @@ -229,25 +230,25 @@ var leapHands = (function () { // TODO: 2.0* scale factor should not be necessary; Leap Motion controller code needs investigating. handRoll = 2.0 * -hands[h].controller.getAbsRotation().z; handPitch = 2.0 * -wrists[h].controller.getAbsRotation().x; - handYaw = 2.0 * -wrists[h].controller.getAbsRotation().y; + handYaw = 2.0 * wrists[h].controller.getAbsRotation().y; - // TODO: Leap Motion controller's right-hand roll calculation is off by 90 degrees. + // TODO: Leap Motion controller's right-hand roll calculation only works if physical hand is upside down. + // Approximate fix is to add a fudge factor. if (h === 1) { - handRoll = handRoll + PI / 2.0; + handRoll = handRoll + 0.6 * PI; } // Hand position and orientation ... if (h === 0) { - MyAvatar.setJointModelPositionAndOrientation(hands[h].jointName, handOffset, Quat.fromPitchYawRollRadians( - handPitch, - -PI / 2.0 - handYaw, - handRoll), true); + handRotation = Quat.multiply(Quat.angleAxis(-90.0, { x: 0, y: 1, z: 0 }), + Quat.fromVec3Radians({ x: handRoll, y: handYaw, z: -handPitch })); + + } else { - MyAvatar.setJointModelPositionAndOrientation(hands[h].jointName, handOffset, Quat.fromPitchYawRollRadians( - handPitch, - PI / 2.0 - handYaw, - handRoll), true); + handRotation = Quat.multiply(Quat.angleAxis(90.0, { x: 0, y: 1, z: 0 }), + Quat.fromVec3Radians({ x: -handRoll, y: handYaw, z: handPitch })); } + MyAvatar.setJointModelPositionAndOrientation(hands[h].jointName, handOffset, handRotation, true); // Finger joints ... // TODO: 2.0 * scale factors should not be necessary; Leap Motion controller code needs investigating. diff --git a/interface/src/avatar/Avatar.cpp b/interface/src/avatar/Avatar.cpp index 301020f1c5..1eaa717585 100644 --- a/interface/src/avatar/Avatar.cpp +++ b/interface/src/avatar/Avatar.cpp @@ -840,36 +840,26 @@ glm::quat Avatar::getJointCombinedRotation(const QString& name) const { return rotation; } -bool Avatar::setJointModelPositionAndOrientation(int index, glm::vec3 position, const glm::quat& rotation, - bool useRotation) { - bool success; +const float SCRIPT_PRIORITY = DEFAULT_PRIORITY + 1.0f; + +void Avatar::setJointModelPositionAndOrientation(int index, glm::vec3 position, const glm::quat& rotation) { if (QThread::currentThread() != thread()) { QMetaObject::invokeMethod(const_cast(this), "setJointModelPositionAndOrientation", - Qt::BlockingQueuedConnection, Q_RETURN_ARG(bool, success), Q_ARG(const int, index), - Q_ARG(const glm::vec3, position), - Q_ARG(const glm::quat&, rotation), Q_ARG(bool, useRotation)); + Qt::BlockingQueuedConnection, Q_ARG(const int, index), Q_ARG(const glm::vec3, position), + Q_ARG(const glm::quat&, rotation)); } else { - qDebug() << "setJointModelPositionAndOrientation()"; - success = _skeletonModel.setJointPosition(index, position, rotation, useRotation, -1, true, - glm::vec3(0.0, -1.0, 0.0), DEFAULT_PRIORITY + 2.0f); + _skeletonModel.inverseKinematics(index, position, rotation, SCRIPT_PRIORITY); } - return success; } -bool Avatar::setJointModelPositionAndOrientation(const QString& name, glm::vec3 position, const glm::quat& rotation, - bool useRotation) { - bool success; +void Avatar::setJointModelPositionAndOrientation(const QString& name, glm::vec3 position, const glm::quat& rotation) { if (QThread::currentThread() != thread()) { QMetaObject::invokeMethod(const_cast(this), "setJointModelPositionAndOrientation", - Qt::BlockingQueuedConnection, Q_RETURN_ARG(bool, success), Q_ARG(const QString&, name), - Q_ARG(const glm::vec3, position), - Q_ARG(const glm::quat&, rotation), Q_ARG(bool, useRotation)); + Qt::BlockingQueuedConnection, Q_ARG(const QString&, name), Q_ARG(const glm::vec3, position), + Q_ARG(const glm::quat&, rotation)); } else { - qDebug() << "setJointModelPositionAndOrientation()"; - success = _skeletonModel.setJointPosition(getJointIndex(name), position, rotation, useRotation, -1, true, - glm::vec3(0.0, -1.0, 0.0), DEFAULT_PRIORITY + 2.0f); + _skeletonModel.inverseKinematics(getJointIndex(name), position, rotation, SCRIPT_PRIORITY); } - return success; } void Avatar::scaleVectorRelativeToPosition(glm::vec3 &positionToScale) const { diff --git a/interface/src/avatar/Avatar.h b/interface/src/avatar/Avatar.h index 3a29f04d4c..20f8f0c9a2 100755 --- a/interface/src/avatar/Avatar.h +++ b/interface/src/avatar/Avatar.h @@ -151,10 +151,9 @@ public: Q_INVOKABLE glm::quat getJointCombinedRotation(int index) const; Q_INVOKABLE glm::quat getJointCombinedRotation(const QString& name) const; - Q_INVOKABLE bool setJointModelPositionAndOrientation(int index, const glm::vec3 position, - const glm::quat& rotation = glm::quat(), bool useRotation = false); - Q_INVOKABLE bool setJointModelPositionAndOrientation(const QString& name, const glm::vec3 position, - const glm::quat& rotation = glm::quat(), bool useRotation = false); + Q_INVOKABLE void setJointModelPositionAndOrientation(int index, const glm::vec3 position, const glm::quat& rotation); + Q_INVOKABLE void setJointModelPositionAndOrientation(const QString& name, const glm::vec3 position, + const glm::quat& rotation); Q_INVOKABLE glm::vec3 getVelocity() const { return _velocity; } Q_INVOKABLE glm::vec3 getAcceleration() const { return _acceleration; } diff --git a/interface/src/renderer/Model.h b/interface/src/renderer/Model.h index 39090c7e1b..9d02168c47 100644 --- a/interface/src/renderer/Model.h +++ b/interface/src/renderer/Model.h @@ -183,17 +183,7 @@ public: QVector& getJointStates() { return _jointStates; } const QVector& getJointStates() const { return _jointStates; } - /// \param jointIndex index of joint in model structure - /// \param position position of joint in model-frame - /// \param rotation rotation of joint in model-frame - /// \param useRotation false if rotation should be ignored - /// \param lastFreeIndex - /// \param allIntermediatesFree - /// \param alignment - /// \return true if joint exists - bool setJointPosition(int jointIndex, const glm::vec3& position, const glm::quat& rotation = glm::quat(), - bool useRotation = false, int lastFreeIndex = -1, bool allIntermediatesFree = false, - const glm::vec3& alignment = glm::vec3(0.0f, -1.0f, 0.0f), float priority = 1.0f); + void inverseKinematics(int jointIndex, glm::vec3 position, const glm::quat& rotation, float priority); protected: QSharedPointer _geometry; @@ -238,8 +228,18 @@ protected: virtual void updateVisibleJointStates(); - void inverseKinematics(int jointIndex, glm::vec3 position, const glm::quat& rotation, float priority); - + /// \param jointIndex index of joint in model structure + /// \param position position of joint in model-frame + /// \param rotation rotation of joint in model-frame + /// \param useRotation false if rotation should be ignored + /// \param lastFreeIndex + /// \param allIntermediatesFree + /// \param alignment + /// \return true if joint exists + bool setJointPosition(int jointIndex, const glm::vec3& position, const glm::quat& rotation = glm::quat(), + bool useRotation = false, int lastFreeIndex = -1, bool allIntermediatesFree = false, + const glm::vec3& alignment = glm::vec3(0.0f, -1.0f, 0.0f), float priority = 1.0f); + /// Restores the indexed joint to its default position. /// \param fraction the fraction of the default position to apply (i.e., 0.25f to slerp one fourth of the way to /// the original position From 543e9e3eb5c1a74c3854e7f30af14cb7dbd089ab Mon Sep 17 00:00:00 2001 From: David Rowe Date: Mon, 15 Sep 2014 20:22:53 -0700 Subject: [PATCH 11/14] Access wrist controller rotation only once per frame --- examples/leapHands.js | 6 ++++-- 1 file changed, 4 insertions(+), 2 deletions(-) diff --git a/examples/leapHands.js b/examples/leapHands.js index 08d4a918bf..e7080f0cbb 100644 --- a/examples/leapHands.js +++ b/examples/leapHands.js @@ -212,6 +212,7 @@ var leapHands = (function () { handPitch, handYaw, handRotation, + wristAbsRotation, locRotation; for (h = 0; h < NUM_HANDS; h += 1) { @@ -229,8 +230,9 @@ var leapHands = (function () { // TODO: 2.0* scale factor should not be necessary; Leap Motion controller code needs investigating. handRoll = 2.0 * -hands[h].controller.getAbsRotation().z; - handPitch = 2.0 * -wrists[h].controller.getAbsRotation().x; - handYaw = 2.0 * wrists[h].controller.getAbsRotation().y; + wristAbsRotation = wrists[h].controller.getAbsRotation(); + handPitch = 2.0 * -wristAbsRotation.x; + handYaw = 2.0 * wristAbsRotation.y; // TODO: Leap Motion controller's right-hand roll calculation only works if physical hand is upside down. // Approximate fix is to add a fudge factor. From f1d7ab149fc4ff551734bc1cac7878a4536a5fd5 Mon Sep 17 00:00:00 2001 From: David Rowe Date: Mon, 15 Sep 2014 21:25:50 -0700 Subject: [PATCH 12/14] Add calibration of "zero" position according to avatar body --- examples/leapHands.js | 84 ++++++++++++++++++++++++++++++++++++------- 1 file changed, 71 insertions(+), 13 deletions(-) diff --git a/examples/leapHands.js b/examples/leapHands.js index e7080f0cbb..8339ac6272 100644 --- a/examples/leapHands.js +++ b/examples/leapHands.js @@ -21,6 +21,11 @@ var leapHands = (function () { THUMB = 0, NUM_FINGER_JOINTS = 3, // 0 = metacarpal(hand)-proximal(finger) joint; ...; 2 = intermediate-distal(tip) joint MAX_HAND_INACTIVE_COUNT = 20, + avatarCalibrationStatus, + AVATAR_UNCALIBRATED = 0, + AVATAR_CALIBRATING = 1, + AVATAR_CALIBRATED = 2, + AVATAR_CALIBRATION_TIME = 1000, // milliseconds LEAP_HEIGHT_OFFSET = 0.15, PI = 3.141593; @@ -110,8 +115,69 @@ var leapHands = (function () { */ } + function calibrateAvatarFinish() { + var avatarPosition, + avatarOrientation, + handPosition, + h; + + avatarPosition = MyAvatar.position; + avatarOrientation = MyAvatar.orientation; + + for (h = 0; h < NUM_HANDS; h += 1) { + handPosition = MyAvatar.getJointPosition(hands[h].jointName); + hands[h].zeroPosition = { + x: handPosition.x - avatarPosition.x, + y: handPosition.y - avatarPosition.y, + z: avatarPosition.z - handPosition.z + }; + hands[h].zeroPosition = Vec3.multiplyQbyV(MyAvatar.orientation, hands[h].zeroPosition); + } + + MyAvatar.clearJointData("LeftHand"); + MyAvatar.clearJointData("LeftForeArm"); + MyAvatar.clearJointData("LeftArm"); + MyAvatar.clearJointData("RightHand"); + MyAvatar.clearJointData("RightForeArm"); + MyAvatar.clearJointData("RightArm"); + + avatarCalibrationStatus = AVATAR_CALIBRATED; + print("Leap Motion: Calibrated avatar"); + } + + function calibrateAvatar() { + + avatarCalibrationStatus = AVATAR_CALIBRATING; + + // Set avatar arms vertical, forearms horizontal, as "zero" position for calibration + MyAvatar.setJointData("LeftArm", Quat.fromPitchYawRollDegrees(90.0, 0.0, -90.0)); + MyAvatar.setJointData("LeftForeArm", Quat.fromPitchYawRollDegrees(90.0, 0.0, 180.0)); + MyAvatar.setJointData("LeftHand", Quat.fromPitchYawRollRadians(0.0, 0.0, 0.0)); + MyAvatar.setJointData("RightArm", Quat.fromPitchYawRollDegrees(90.0, 0.0, 90.0)); + MyAvatar.setJointData("RightForeArm", Quat.fromPitchYawRollDegrees(90.0, 0.0, 180.0)); + MyAvatar.setJointData("RightHand", Quat.fromPitchYawRollRadians(0.0, 0.0, 0.0)); + + // Wait for arms to assume their positions before calculating + Script.setTimeout(calibrateAvatarFinish, AVATAR_CALIBRATION_TIME); + } + + function checkCalibration() { + + if (avatarCalibrationStatus === AVATAR_CALIBRATED) { + return true; + } + + if (avatarCalibrationStatus !== AVATAR_CALIBRATING) { + calibrateAvatar(); + } + + return false; + } + function setUp() { + avatarCalibrationStatus = AVATAR_UNCALIBRATED; + // TODO: Leap Motion controller joint naming doesn't match up with skeleton joint naming; numbers are out by 1. hands = [ @@ -187,19 +253,6 @@ var leapHands = (function () { { jointName: "RightHandPinky3", controller: Controller.createInputController("Spatial", "joint_R_pinky4") } ] ]; - - // TODO: Add automatic calibration of avatar's hands. - // Calibration values for Ron. - hands[0].zeroPosition = { - x: -0.157, - y: 0.204, - z: 0.336 - }; - hands[1].zeroPosition = { - x: 0.234, - y: 0.204, - z: 0.339 - }; } function moveHands() { @@ -220,6 +273,11 @@ var leapHands = (function () { if (hands[h].controller.isActive()) { + // Calibrate when and if a controller is first active. + if (!checkCalibration()) { + return; + } + // Hand position ... handOffset = hands[h].controller.getAbsTranslation(); handOffset = { From f890ea744b9287d3e308be6628bb4c9cfe902ba3 Mon Sep 17 00:00:00 2001 From: David Rowe Date: Mon, 15 Sep 2014 23:09:00 -0700 Subject: [PATCH 13/14] Add calibration of physical hand height --- examples/leapHands.js | 52 +++++++++++++++++++++++++------------------ 1 file changed, 30 insertions(+), 22 deletions(-) diff --git a/examples/leapHands.js b/examples/leapHands.js index 8339ac6272..cf72b5efdf 100644 --- a/examples/leapHands.js +++ b/examples/leapHands.js @@ -21,12 +21,11 @@ var leapHands = (function () { THUMB = 0, NUM_FINGER_JOINTS = 3, // 0 = metacarpal(hand)-proximal(finger) joint; ...; 2 = intermediate-distal(tip) joint MAX_HAND_INACTIVE_COUNT = 20, - avatarCalibrationStatus, - AVATAR_UNCALIBRATED = 0, - AVATAR_CALIBRATING = 1, - AVATAR_CALIBRATED = 2, - AVATAR_CALIBRATION_TIME = 1000, // milliseconds - LEAP_HEIGHT_OFFSET = 0.15, + calibrationStatus, + UNCALIBRATED = 0, + CALIBRATING = 1, + CALIBRATED = 2, + CALIBRATION_TIME = 1000, // milliseconds PI = 3.141593; function printSkeletonJointNames() { @@ -115,23 +114,32 @@ var leapHands = (function () { */ } - function calibrateAvatarFinish() { + function finishCalibration() { var avatarPosition, avatarOrientation, - handPosition, + avatarHandPosition, + leapHandHeight, h; + if (hands[0].controller.isActive() && hands[1].controller.isActive()) { + leapHandHeight = (hands[0].controller.getAbsTranslation().y + hands[1].controller.getAbsTranslation().y) / 2.0; + } else { + calibrationStatus = UNCALIBRATED; + return; + } + avatarPosition = MyAvatar.position; avatarOrientation = MyAvatar.orientation; for (h = 0; h < NUM_HANDS; h += 1) { - handPosition = MyAvatar.getJointPosition(hands[h].jointName); + avatarHandPosition = MyAvatar.getJointPosition(hands[h].jointName); hands[h].zeroPosition = { - x: handPosition.x - avatarPosition.x, - y: handPosition.y - avatarPosition.y, - z: avatarPosition.z - handPosition.z + x: avatarHandPosition.x - avatarPosition.x, + y: avatarHandPosition.y - avatarPosition.y, + z: avatarPosition.z - avatarHandPosition.z }; hands[h].zeroPosition = Vec3.multiplyQbyV(MyAvatar.orientation, hands[h].zeroPosition); + hands[h].zeroPosition.y = hands[h].zeroPosition.y - leapHandHeight; } MyAvatar.clearJointData("LeftHand"); @@ -141,13 +149,13 @@ var leapHands = (function () { MyAvatar.clearJointData("RightForeArm"); MyAvatar.clearJointData("RightArm"); - avatarCalibrationStatus = AVATAR_CALIBRATED; - print("Leap Motion: Calibrated avatar"); + calibrationStatus = CALIBRATED; + print("Leap Motion: Calibrated"); } - function calibrateAvatar() { + function calibrate() { - avatarCalibrationStatus = AVATAR_CALIBRATING; + calibrationStatus = CALIBRATING; // Set avatar arms vertical, forearms horizontal, as "zero" position for calibration MyAvatar.setJointData("LeftArm", Quat.fromPitchYawRollDegrees(90.0, 0.0, -90.0)); @@ -158,17 +166,17 @@ var leapHands = (function () { MyAvatar.setJointData("RightHand", Quat.fromPitchYawRollRadians(0.0, 0.0, 0.0)); // Wait for arms to assume their positions before calculating - Script.setTimeout(calibrateAvatarFinish, AVATAR_CALIBRATION_TIME); + Script.setTimeout(finishCalibration, CALIBRATION_TIME); } function checkCalibration() { - if (avatarCalibrationStatus === AVATAR_CALIBRATED) { + if (calibrationStatus === CALIBRATED) { return true; } - if (avatarCalibrationStatus !== AVATAR_CALIBRATING) { - calibrateAvatar(); + if (calibrationStatus !== CALIBRATING) { + calibrate(); } return false; @@ -176,7 +184,7 @@ var leapHands = (function () { function setUp() { - avatarCalibrationStatus = AVATAR_UNCALIBRATED; + calibrationStatus = UNCALIBRATED; // TODO: Leap Motion controller joint naming doesn't match up with skeleton joint naming; numbers are out by 1. @@ -282,7 +290,7 @@ var leapHands = (function () { handOffset = hands[h].controller.getAbsTranslation(); handOffset = { x: -handOffset.x, - y: hands[h].zeroPosition.y + handOffset.y - LEAP_HEIGHT_OFFSET, + y: hands[h].zeroPosition.y + handOffset.y, z: hands[h].zeroPosition.z - handOffset.z }; From ed2216d10018982bd534ead947ed637a0dcbc5e6 Mon Sep 17 00:00:00 2001 From: David Rowe Date: Tue, 16 Sep 2014 11:13:42 -0700 Subject: [PATCH 14/14] Apply right hand roll work-around only if Windows --- examples/leapHands.js | 8 ++++++-- 1 file changed, 6 insertions(+), 2 deletions(-) diff --git a/examples/leapHands.js b/examples/leapHands.js index cf72b5efdf..95d3969a08 100644 --- a/examples/leapHands.js +++ b/examples/leapHands.js @@ -26,7 +26,8 @@ var leapHands = (function () { CALIBRATING = 1, CALIBRATED = 2, CALIBRATION_TIME = 1000, // milliseconds - PI = 3.141593; + PI = 3.141593, + isWindows; function printSkeletonJointNames() { var jointNames, @@ -123,6 +124,9 @@ var leapHands = (function () { if (hands[0].controller.isActive() && hands[1].controller.isActive()) { leapHandHeight = (hands[0].controller.getAbsTranslation().y + hands[1].controller.getAbsTranslation().y) / 2.0; + + // TODO: Temporary detection of Windows to work around Leap Controller problem. + isWindows = (hands[1].controller.getAbsRotation().z > (0.25 * PI)); } else { calibrationStatus = UNCALIBRATED; return; @@ -302,7 +306,7 @@ var leapHands = (function () { // TODO: Leap Motion controller's right-hand roll calculation only works if physical hand is upside down. // Approximate fix is to add a fudge factor. - if (h === 1) { + if (h === 1 && isWindows) { handRoll = handRoll + 0.6 * PI; }