0d14e5a379
Needs a lot of cleanup. Data has been de-duplicated, and where identical copies existed, one of them has been replaced with a symlink. Some files have been excluded, such as binaries, installers and debug dumps. Some of that may still be present.
532 lines
21 KiB
JavaScript
532 lines
21 KiB
JavaScript
//
|
|
// realsense.js
|
|
// examples
|
|
//
|
|
// Created by Thijs Wenker on 17 Dec 2014.
|
|
// Copyright 2014 High Fidelity, Inc.
|
|
//
|
|
// This is an example script that uses the Intel RealSense camera to make the avatar's hands replicate the user's hand actions.
|
|
//
|
|
// Distributed under the Apache License, Version 2.0.
|
|
// See the accompanying file LICENSE or http://www.apache.org/licenses/LICENSE-2.0.html
|
|
//
|
|
|
|
var leapHands = (function () {
|
|
|
|
var isOnHMD,
|
|
MotionTracker = "RealSense",
|
|
LEAP_ON_HMD_MENU_ITEM = "Leap Motion on HMD",
|
|
LEAP_OFFSET = 0.019, // Thickness of Leap Motion plus HMD clip
|
|
HMD_OFFSET = 0.070, // Eyeballs to front surface of Oculus DK2 TODO: Confirm and make depend on device and eye relief
|
|
hasHandAndWristJoints,
|
|
handToWristOffset = [], // For avatars without a wrist joint we control an estimate of a proper hand joint position
|
|
HAND_OFFSET = 0.4, // Relative distance of wrist to hand versus wrist to index finger knuckle
|
|
hands,
|
|
wrists,
|
|
NUM_HANDS = 2, // 0 = left; 1 = right
|
|
fingers,
|
|
NUM_FINGERS = 5, // 0 = thumb; ...; 4 = pinky
|
|
THUMB = 0,
|
|
MIDDLE_FINGER = 2,
|
|
NUM_FINGER_JOINTS = 3, // 0 = metacarpal(hand)-proximal(finger) joint; ...; 2 = intermediate-distal joint
|
|
MAX_HAND_INACTIVE_COUNT = 20,
|
|
calibrationStatus,
|
|
UNCALIBRATED = 0,
|
|
CALIBRATING = 1,
|
|
CALIBRATED = 2,
|
|
CALIBRATION_TIME = 1000, // milliseconds
|
|
avatarScale,
|
|
avatarFaceModelURL,
|
|
avatarSkeletonModelURL,
|
|
settingsTimer;
|
|
|
|
function printSkeletonJointNames() {
|
|
var jointNames,
|
|
i;
|
|
|
|
print(MyAvatar.skeletonModelURL);
|
|
|
|
print("Skeleton joint names ...");
|
|
jointNames = MyAvatar.getJointNames();
|
|
for (i = 0; i < jointNames.length; i += 1) {
|
|
print(i + ": " + jointNames[i]);
|
|
}
|
|
print("... skeleton joint names");
|
|
|
|
/*
|
|
http://public.highfidelity.io/models/skeletons/ron_standing.fst
|
|
Skeleton joint names ...
|
|
0: Hips
|
|
1: RightUpLeg
|
|
2: RightLeg
|
|
3: RightFoot
|
|
4: RightToeBase
|
|
5: RightToe_End
|
|
6: LeftUpLeg
|
|
7: LeftLeg
|
|
8: LeftFoot
|
|
9: LeftToeBase
|
|
10: LeftToe_End
|
|
11: Spine
|
|
12: Spine1
|
|
13: Spine2
|
|
14: RightShoulder
|
|
15: RightArm
|
|
16: RightForeArm
|
|
17: RightHand
|
|
18: RightHandPinky1
|
|
19: RightHandPinky2
|
|
20: RightHandPinky3
|
|
21: RightHandPinky4
|
|
22: RightHandRing1
|
|
23: RightHandRing2
|
|
24: RightHandRing3
|
|
25: RightHandRing4
|
|
26: RightHandMiddle1
|
|
27: RightHandMiddle2
|
|
28: RightHandMiddle3
|
|
29: RightHandMiddle4
|
|
30: RightHandIndex1
|
|
31: RightHandIndex2
|
|
32: RightHandIndex3
|
|
33: RightHandIndex4
|
|
34: RightHandThumb1
|
|
35: RightHandThumb2
|
|
36: RightHandThumb3
|
|
37: RightHandThumb4
|
|
38: LeftShoulder
|
|
39: LeftArm
|
|
40: LeftForeArm
|
|
41: LeftHand
|
|
42: LeftHandPinky1
|
|
43: LeftHandPinky2
|
|
44: LeftHandPinky3
|
|
45: LeftHandPinky4
|
|
46: LeftHandRing1
|
|
47: LeftHandRing2
|
|
48: LeftHandRing3
|
|
49: LeftHandRing4
|
|
50: LeftHandMiddle1
|
|
51: LeftHandMiddle2
|
|
52: LeftHandMiddle3
|
|
53: LeftHandMiddle4
|
|
54: LeftHandIndex1
|
|
55: LeftHandIndex2
|
|
56: LeftHandIndex3
|
|
57: LeftHandIndex4
|
|
58: LeftHandThumb1
|
|
59: LeftHandThumb2
|
|
60: LeftHandThumb3
|
|
61: LeftHandThumb4
|
|
62: Neck
|
|
63: Head
|
|
64: HeadTop_End
|
|
65: body
|
|
... skeleton joint names
|
|
*/
|
|
}
|
|
|
|
function finishCalibration() {
|
|
var avatarPosition,
|
|
handPosition,
|
|
middleFingerPosition,
|
|
leapHandHeight,
|
|
h;
|
|
|
|
if (!isOnHMD) {
|
|
if (hands[0].controller.isActive() && hands[1].controller.isActive()) {
|
|
leapHandHeight = (hands[0].controller.getAbsTranslation().y + hands[1].controller.getAbsTranslation().y) / 2.0;
|
|
} else {
|
|
calibrationStatus = UNCALIBRATED;
|
|
return;
|
|
}
|
|
}
|
|
|
|
avatarPosition = MyAvatar.position;
|
|
|
|
for (h = 0; h < NUM_HANDS; h += 1) {
|
|
handPosition = MyAvatar.getJointPosition(hands[h].jointName);
|
|
if (!hasHandAndWristJoints) {
|
|
middleFingerPosition = MyAvatar.getJointPosition(fingers[h][MIDDLE_FINGER][0].jointName);
|
|
handToWristOffset[h] = Vec3.multiply(Vec3.subtract(handPosition, middleFingerPosition), 1.0 - HAND_OFFSET);
|
|
}
|
|
|
|
if (isOnHMD) {
|
|
// Offset of Leap Motion origin from physical eye position
|
|
hands[h].zeroPosition = { x: 0.0, y: 0.0, z: HMD_OFFSET + LEAP_OFFSET };
|
|
} else {
|
|
hands[h].zeroPosition = {
|
|
x: handPosition.x - avatarPosition.x,
|
|
y: handPosition.y - avatarPosition.y,
|
|
z: avatarPosition.z - handPosition.z
|
|
};
|
|
hands[h].zeroPosition = Vec3.multiplyQbyV(MyAvatar.orientation, hands[h].zeroPosition);
|
|
hands[h].zeroPosition.y = hands[h].zeroPosition.y - leapHandHeight;
|
|
}
|
|
}
|
|
|
|
MyAvatar.clearJointData("LeftHand");
|
|
MyAvatar.clearJointData("LeftForeArm");
|
|
MyAvatar.clearJointData("LeftArm");
|
|
MyAvatar.clearJointData("RightHand");
|
|
MyAvatar.clearJointData("RightForeArm");
|
|
MyAvatar.clearJointData("RightArm");
|
|
|
|
calibrationStatus = CALIBRATED;
|
|
print("Leap Motion: Calibrated");
|
|
}
|
|
|
|
function calibrate() {
|
|
var jointNames,
|
|
i;
|
|
|
|
calibrationStatus = CALIBRATING;
|
|
|
|
avatarScale = MyAvatar.scale;
|
|
avatarFaceModelURL = MyAvatar.faceModelURL;
|
|
avatarSkeletonModelURL = MyAvatar.skeletonModelURL;
|
|
|
|
// Does this skeleton have both wrist and hand joints?
|
|
hasHandAndWristJoints = false;
|
|
jointNames = MyAvatar.getJointNames();
|
|
for (i = 0; i < jointNames.length; i += 1) {
|
|
hasHandAndWristJoints = hasHandAndWristJoints || jointNames[i].toLowerCase() === "leftwrist";
|
|
}
|
|
|
|
// Set avatar arms vertical, forearms horizontal, as "zero" position for calibration
|
|
MyAvatar.setJointData("LeftArm", Quat.fromPitchYawRollDegrees(90.0, 0.0, -90.0));
|
|
MyAvatar.setJointData("LeftForeArm", Quat.fromPitchYawRollDegrees(90.0, 0.0, 180.0));
|
|
MyAvatar.setJointData("LeftHand", Quat.fromPitchYawRollRadians(0.0, 0.0, 0.0));
|
|
MyAvatar.setJointData("RightArm", Quat.fromPitchYawRollDegrees(90.0, 0.0, 90.0));
|
|
MyAvatar.setJointData("RightForeArm", Quat.fromPitchYawRollDegrees(90.0, 0.0, 180.0));
|
|
MyAvatar.setJointData("RightHand", Quat.fromPitchYawRollRadians(0.0, 0.0, 0.0));
|
|
|
|
|
|
// Wait for arms to assume their positions before calculating
|
|
Script.setTimeout(finishCalibration, CALIBRATION_TIME);
|
|
}
|
|
|
|
function checkCalibration() {
|
|
|
|
if (calibrationStatus === CALIBRATED) {
|
|
return true;
|
|
}
|
|
|
|
if (calibrationStatus !== CALIBRATING) {
|
|
calibrate();
|
|
}
|
|
|
|
return false;
|
|
}
|
|
|
|
function setIsOnHMD() {
|
|
isOnHMD = Menu.isOptionChecked(LEAP_ON_HMD_MENU_ITEM);
|
|
print("Leap Motion: " + (isOnHMD ? "Is on HMD" : "Is on desk"));
|
|
}
|
|
|
|
function checkSettings() {
|
|
if (calibrationStatus > UNCALIBRATED && (MyAvatar.scale !== avatarScale
|
|
|| MyAvatar.faceModelURL !== avatarFaceModelURL
|
|
|| MyAvatar.skeletonModelURL !== avatarSkeletonModelURL
|
|
|| Menu.isOptionChecked(LEAP_ON_HMD_MENU_ITEM) !== isOnHMD)) {
|
|
print("Leap Motion: Recalibrate...");
|
|
calibrationStatus = UNCALIBRATED;
|
|
|
|
setIsOnHMD();
|
|
}
|
|
}
|
|
|
|
function setUp() {
|
|
|
|
wrists = [
|
|
{
|
|
jointName: "LeftWrist",
|
|
controller: Controller.createInputController(MotionTracker, "joint_L_wrist")
|
|
},
|
|
{
|
|
jointName: "RightWrist",
|
|
controller: Controller.createInputController(MotionTracker, "joint_R_wrist")
|
|
}
|
|
];
|
|
|
|
hands = [
|
|
{
|
|
jointName: "LeftHand",
|
|
controller: Controller.createInputController(MotionTracker, "joint_L_hand"),
|
|
inactiveCount: 0
|
|
},
|
|
{
|
|
jointName: "RightHand",
|
|
controller: Controller.createInputController(MotionTracker, "joint_R_hand"),
|
|
inactiveCount: 0
|
|
}
|
|
];
|
|
|
|
// The Leap controller's first joint is the hand-metacarpal joint but this joint's data is not used because it's too
|
|
// dependent on the model skeleton exactly matching the Leap skeleton; using just the second and subsequent joints
|
|
// seems to work better over all.
|
|
fingers = [{}, {}];
|
|
fingers[0] = [
|
|
[
|
|
{ jointName: "LeftHandThumb1", controller: Controller.createInputController(MotionTracker, "joint_L_thumb2") },
|
|
{ jointName: "LeftHandThumb2", controller: Controller.createInputController(MotionTracker, "joint_L_thumb3") },
|
|
{ jointName: "LeftHandThumb3", controller: Controller.createInputController(MotionTracker, "joint_L_thumb4") }
|
|
],
|
|
[
|
|
{ jointName: "LeftHandIndex1", controller: Controller.createInputController(MotionTracker, "joint_L_index2") },
|
|
{ jointName: "LeftHandIndex2", controller: Controller.createInputController(MotionTracker, "joint_L_index3") },
|
|
{ jointName: "LeftHandIndex3", controller: Controller.createInputController(MotionTracker, "joint_L_index4") }
|
|
],
|
|
[
|
|
{ jointName: "LeftHandMiddle1", controller: Controller.createInputController(MotionTracker, "joint_L_middle2") },
|
|
{ jointName: "LeftHandMiddle2", controller: Controller.createInputController(MotionTracker, "joint_L_middle3") },
|
|
{ jointName: "LeftHandMiddle3", controller: Controller.createInputController(MotionTracker, "joint_L_middle4") }
|
|
],
|
|
[
|
|
{ jointName: "LeftHandRing1", controller: Controller.createInputController(MotionTracker, "joint_L_ring2") },
|
|
{ jointName: "LeftHandRing2", controller: Controller.createInputController(MotionTracker, "joint_L_ring3") },
|
|
{ jointName: "LeftHandRing3", controller: Controller.createInputController(MotionTracker, "joint_L_ring4") }
|
|
],
|
|
[
|
|
{ jointName: "LeftHandPinky1", controller: Controller.createInputController(MotionTracker, "joint_L_pinky2") },
|
|
{ jointName: "LeftHandPinky2", controller: Controller.createInputController(MotionTracker, "joint_L_pinky3") },
|
|
{ jointName: "LeftHandPinky3", controller: Controller.createInputController(MotionTracker, "joint_L_pinky4") }
|
|
]
|
|
];
|
|
fingers[1] = [
|
|
[
|
|
{ jointName: "RightHandThumb1", controller: Controller.createInputController(MotionTracker, "joint_R_thumb2") },
|
|
{ jointName: "RightHandThumb2", controller: Controller.createInputController(MotionTracker, "joint_R_thumb3") },
|
|
{ jointName: "RightHandThumb3", controller: Controller.createInputController(MotionTracker, "joint_R_thumb4") }
|
|
],
|
|
[
|
|
{ jointName: "RightHandIndex1", controller: Controller.createInputController(MotionTracker, "joint_R_index2") },
|
|
{ jointName: "RightHandIndex2", controller: Controller.createInputController(MotionTracker, "joint_R_index3") },
|
|
{ jointName: "RightHandIndex3", controller: Controller.createInputController(MotionTracker, "joint_R_index4") }
|
|
],
|
|
[
|
|
{ jointName: "RightHandMiddle1", controller: Controller.createInputController(MotionTracker, "joint_R_middle2") },
|
|
{ jointName: "RightHandMiddle2", controller: Controller.createInputController(MotionTracker, "joint_R_middle3") },
|
|
{ jointName: "RightHandMiddle3", controller: Controller.createInputController(MotionTracker, "joint_R_middle4") }
|
|
],
|
|
[
|
|
{ jointName: "RightHandRing1", controller: Controller.createInputController(MotionTracker, "joint_R_ring2") },
|
|
{ jointName: "RightHandRing2", controller: Controller.createInputController(MotionTracker, "joint_R_ring3") },
|
|
{ jointName: "RightHandRing3", controller: Controller.createInputController(MotionTracker, "joint_R_ring4") }
|
|
],
|
|
[
|
|
{ jointName: "RightHandPinky1", controller: Controller.createInputController(MotionTracker, "joint_R_pinky2") },
|
|
{ jointName: "RightHandPinky2", controller: Controller.createInputController(MotionTracker, "joint_R_pinky3") },
|
|
{ jointName: "RightHandPinky3", controller: Controller.createInputController(MotionTracker, "joint_R_pinky4") }
|
|
]
|
|
];
|
|
|
|
setIsOnHMD();
|
|
|
|
settingsTimer = Script.setInterval(checkSettings, 2000);
|
|
|
|
calibrationStatus = UNCALIBRATED;
|
|
}
|
|
|
|
function moveHands() {
|
|
var h,
|
|
i,
|
|
j,
|
|
side,
|
|
handOffset,
|
|
wristOffset,
|
|
handRotation,
|
|
locRotation,
|
|
cameraOrientation,
|
|
inverseAvatarOrientation;
|
|
|
|
for (h = 0; h < NUM_HANDS; h += 1) {
|
|
side = h === 0 ? -1.0 : 1.0;
|
|
|
|
if (hands[h].controller.isActive()) {
|
|
|
|
// Calibrate if necessary.
|
|
if (!checkCalibration()) {
|
|
return;
|
|
}
|
|
|
|
// Hand position ...
|
|
handOffset = hands[h].controller.getAbsTranslation();
|
|
handRotation = hands[h].controller.getAbsRotation();
|
|
|
|
if (isOnHMD) {
|
|
|
|
// Adjust to control wrist position if "hand" joint is at wrist ...
|
|
if (!hasHandAndWristJoints) {
|
|
wristOffset = Vec3.multiplyQbyV(handRotation, handToWristOffset[h]);
|
|
handOffset = Vec3.sum(handOffset, wristOffset);
|
|
}
|
|
|
|
// Hand offset in camera coordinates ...
|
|
handOffset = {
|
|
x: hands[h].zeroPosition.x - handOffset.x,
|
|
y: hands[h].zeroPosition.y - handOffset.z,
|
|
z: hands[h].zeroPosition.z + handOffset.y
|
|
};
|
|
handOffset.z = -handOffset.z;
|
|
|
|
// Hand offset in world coordinates ...
|
|
cameraOrientation = Camera.getOrientation();
|
|
handOffset = Vec3.sum(Camera.getPosition(), Vec3.multiplyQbyV(cameraOrientation, handOffset));
|
|
|
|
// Hand offset in avatar coordinates ...
|
|
inverseAvatarOrientation = Quat.inverse(MyAvatar.orientation);
|
|
handOffset = Vec3.subtract(handOffset, MyAvatar.position);
|
|
handOffset = Vec3.multiplyQbyV(inverseAvatarOrientation, handOffset);
|
|
handOffset.z = -handOffset.z;
|
|
handOffset.x = -handOffset.x;
|
|
|
|
// Hand rotation in camera coordinates ...
|
|
handRotation = {
|
|
x: handRotation.z,
|
|
y: handRotation.y,
|
|
z: handRotation.x,
|
|
w: handRotation.w
|
|
};
|
|
|
|
// Hand rotation in avatar coordinates ...
|
|
if (h === 0) {
|
|
handRotation.x = -handRotation.x;
|
|
handRotation = Quat.multiply(Quat.angleAxis(90.0, { x: 1, y: 0, z: 0 }), handRotation);
|
|
handRotation = Quat.multiply(Quat.angleAxis(90.0, { x: 0, y: 0, z: 1 }), handRotation);
|
|
} else {
|
|
handRotation.z = -handRotation.z;
|
|
handRotation = Quat.multiply(Quat.angleAxis(90.0, { x: 1, y: 0, z: 0 }), handRotation);
|
|
handRotation = Quat.multiply(Quat.angleAxis(-90.0, { x: 0, y: 0, z: 1 }), handRotation);
|
|
}
|
|
|
|
cameraOrientation.x = -cameraOrientation.x;
|
|
cameraOrientation.z = -cameraOrientation.z;
|
|
handRotation = Quat.multiply(cameraOrientation, handRotation);
|
|
handRotation = Quat.multiply(inverseAvatarOrientation, handRotation);
|
|
|
|
} else {
|
|
|
|
// Adjust to control wrist position if "hand" joint is at wrist ...
|
|
if (!hasHandAndWristJoints) {
|
|
// wristOffset = Vec3.multiplyQbyV(handRotation, handToWristOffset[h]);
|
|
// handOffset = Vec3.sum(handOffset, wristOffset);
|
|
}
|
|
|
|
// Hand offset in camera coordinates ...
|
|
handOffset = {
|
|
x: -handOffset.x,
|
|
y: hands[h].zeroPosition.y + handOffset.y,
|
|
z: hands[h].zeroPosition.z - handOffset.z
|
|
};
|
|
|
|
// Hand rotation in camera coordinates ...
|
|
handRotation = {
|
|
x: handRotation.y,//z,
|
|
y: handRotation.z,//y,
|
|
z: -handRotation.x,//x,
|
|
w: handRotation.w
|
|
};
|
|
|
|
// Hand rotation in avatar coordinates ...
|
|
if (h === 0) {
|
|
// handRotation.x = -handRotation.x;
|
|
handRotation = Quat.multiply(Quat.angleAxis(90.0, { x: 0, y: 1, z: 0 }),
|
|
handRotation);
|
|
handRotation = Quat.multiply(Quat.angleAxis(-90.0, { x: 1, y: 0, z: 0 }),
|
|
handRotation);
|
|
} else {
|
|
// handRotation.z = -handRotation.z;
|
|
handRotation = Quat.multiply(Quat.angleAxis(-90.0, { x: 0, y: 1, z: 0 }),
|
|
handRotation);
|
|
handRotation = Quat.multiply(Quat.angleAxis(-90.0, { x: 1, y: 0, z: 0 }),
|
|
handRotation);
|
|
}
|
|
}
|
|
|
|
// Set hand position and orientation ...
|
|
MyAvatar.setJointModelPositionAndOrientation(hands[h].jointName, handOffset, handRotation, true);
|
|
|
|
// Set finger joints ...
|
|
for (i = 0; i < NUM_FINGERS; i += 1) {
|
|
for (j = 0; j < NUM_FINGER_JOINTS; j += 1) {
|
|
if (fingers[h][i][j].controller !== null) {
|
|
locRotation = fingers[h][i][j].controller.getLocRotation();
|
|
if (i === THUMB) {
|
|
locRotation = {
|
|
x: side * locRotation.y,
|
|
y: side * -locRotation.z,
|
|
z: side * -locRotation.x,
|
|
w: locRotation.w
|
|
};
|
|
} else {
|
|
locRotation = {
|
|
x: -locRotation.x,
|
|
y: -locRotation.z,
|
|
z: -locRotation.y,
|
|
w: locRotation.w
|
|
};
|
|
}
|
|
MyAvatar.setJointData(fingers[h][i][j].jointName, locRotation);
|
|
}
|
|
}
|
|
}
|
|
|
|
hands[h].inactiveCount = 0;
|
|
|
|
} else {
|
|
|
|
if (hands[h].inactiveCount < MAX_HAND_INACTIVE_COUNT) {
|
|
|
|
hands[h].inactiveCount += 1;
|
|
|
|
if (hands[h].inactiveCount === MAX_HAND_INACTIVE_COUNT) {
|
|
if (h === 0) {
|
|
MyAvatar.clearJointData("LeftHand");
|
|
MyAvatar.clearJointData("LeftForeArm");
|
|
MyAvatar.clearJointData("LeftArm");
|
|
} else {
|
|
MyAvatar.clearJointData("RightHand");
|
|
MyAvatar.clearJointData("RightForeArm");
|
|
MyAvatar.clearJointData("RightArm");
|
|
}
|
|
}
|
|
}
|
|
}
|
|
}
|
|
}
|
|
|
|
function tearDown() {
|
|
var h,
|
|
i,
|
|
j;
|
|
|
|
Script.clearInterval(settingsTimer);
|
|
|
|
for (h = 0; h < NUM_HANDS; h += 1) {
|
|
Controller.releaseInputController(hands[h].controller);
|
|
Controller.releaseInputController(wrists[h].controller);
|
|
for (i = 0; i < NUM_FINGERS; i += 1) {
|
|
for (j = 0; j < NUM_FINGER_JOINTS; j += 1) {
|
|
if (fingers[h][i][j].controller !== null) {
|
|
Controller.releaseInputController(fingers[h][i][j].controller);
|
|
}
|
|
}
|
|
}
|
|
}
|
|
}
|
|
|
|
return {
|
|
printSkeletonJointNames: printSkeletonJointNames,
|
|
setUp : setUp,
|
|
moveHands : moveHands,
|
|
tearDown : tearDown
|
|
};
|
|
}());
|
|
|
|
|
|
//leapHands.printSkeletonJointNames();
|
|
|
|
leapHands.setUp();
|
|
Script.update.connect(leapHands.moveHands);
|
|
Script.scriptEnding.connect(leapHands.tearDown);
|