Merge pull request #3517 from ctrlaltdavid/20048

CR for Job #20048 - Leap - Make leapHands.js work with Leap-on-Rift

examples/leapHands.js

@@ -13,7 +13,10 @@
 
 var leapHands = (function () {
 
-    var hands,
+    var isOnHMD,
+        LEAP_OFFSET = 0.019,  // Thickness of Leap Motion plus HMD clip
+        HMD_OFFSET = 0.100,  // Eyeballs to front surface of Oculus DK2  TODO: Confirm and make depend on device and eye relief
+        hands,
         wrists,
         NUM_HANDS = 2,  // 0 = left; 1 = right
         fingers,
@@ -188,8 +191,6 @@ var leapHands = (function () {
 
     function setUp() {
 
-        calibrationStatus = UNCALIBRATED;
-
         // TODO: Leap Motion controller joint naming doesn't match up with skeleton joint naming; numbers are out by 1.
 
         hands = [
@@ -265,6 +266,20 @@ var leapHands = (function () {
                 { jointName: "RightHandPinky3", controller: Controller.createInputController("Spatial", "joint_R_pinky4") }
             ]
         ];
+
+        isOnHMD = Menu.isOptionChecked("Leap Motion on HMD");
+        if (isOnHMD) {
+            print("Leap Motion is on HMD");
+
+            // Offset of Leap Motion origin from physical eye position
+            hands[0].zeroPosition = { x: 0.0, y: 0.0, z: HMD_OFFSET + LEAP_OFFSET };
+            hands[1].zeroPosition = { x: 0.0, y: 0.0, z: HMD_OFFSET + LEAP_OFFSET };
+
+            calibrationStatus = CALIBRATED;
+        } else {
+            print("Leap Motion is on desk");
+            calibrationStatus = UNCALIBRATED;
+        }
     }
 
     function moveHands() {
@@ -278,7 +293,9 @@ var leapHands = (function () {
             handYaw,
             handRotation,
             wristAbsRotation,
-            locRotation;
+            locRotation,
+            cameraOrientation,
+            inverseAvatarOrientation;
 
         for (h = 0; h < NUM_HANDS; h += 1) {
             side = h === 0 ? -1.0 : 1.0;
@@ -291,35 +308,82 @@ var leapHands = (function () {
                 }
 
                 // Hand position ...
-                handOffset = hands[h].controller.getAbsTranslation();
+                if (isOnHMD) {
-                handOffset = {
-                    x: -handOffset.x,
-                    y: hands[h].zeroPosition.y + handOffset.y,
-                    z: hands[h].zeroPosition.z - handOffset.z
-                };
 
-                // TODO: 2.0* scale factor should not be necessary; Leap Motion controller code needs investigating.
+                    // Hand offset in camera coordinates ...
-                handRoll = 2.0 * -hands[h].controller.getAbsRotation().z;
+                    handOffset = hands[h].controller.getAbsTranslation();
-                wristAbsRotation = wrists[h].controller.getAbsRotation();
+                    handOffset = {
-                handPitch = 2.0 * -wristAbsRotation.x;
+                        x: hands[h].zeroPosition.x - handOffset.x,
-                handYaw = 2.0 * wristAbsRotation.y;
+                        y: hands[h].zeroPosition.y - handOffset.z,
+                        z: hands[h].zeroPosition.z + handOffset.y
+                    };
+                    handOffset.z = -handOffset.z;
 
-                // TODO: Leap Motion controller's right-hand roll calculation only works if physical hand is upside down.
+                    // Hand offset in world coordinates ...
-                // Approximate fix is to add a fudge factor.
+                    cameraOrientation = Camera.getOrientation();
-                if (h === 1 && isWindows) {
+                    handOffset = Vec3.sum(Camera.getPosition(), Vec3.multiplyQbyV(cameraOrientation, handOffset));
-                    handRoll = handRoll + 0.6 * PI;
-                }
 
-                // Hand position and orientation ...
+                    // Hand offset  in avatar coordinates ...
-                if (h === 0) {
+                    inverseAvatarOrientation = Quat.inverse(MyAvatar.orientation);
-                    handRotation = Quat.multiply(Quat.angleAxis(-90.0, { x: 0, y: 1, z: 0 }),
+                    handOffset = Vec3.subtract(handOffset, MyAvatar.position);
-                        Quat.fromVec3Radians({ x: handRoll, y: handYaw, z: -handPitch }));
+                    handOffset = Vec3.multiplyQbyV(inverseAvatarOrientation, handOffset);
+                    handOffset.z = -handOffset.z;
+                    handOffset.x = -handOffset.x;
 
+                    // Hand rotation in camera coordinates ...
+                    // TODO: 2.0* scale factors should not be necessary; Leap Motion controller code needs investigating.
+                    handRoll = 2.0 * -hands[h].controller.getAbsRotation().z;
+                    wristAbsRotation = wrists[h].controller.getAbsRotation();
+                    handPitch = 2.0 * wristAbsRotation.x - PI / 2.0;
+                    handYaw = 2.0 * -wristAbsRotation.y;
+                    // TODO: Roll values only work if hand is upside down; Leap Motion controller code needs investigating.
+                    handRoll = PI + handRoll;
+
+                    if (h === 0) {
+                        handRotation = Quat.multiply(Quat.angleAxis(-90.0, { x: 0, y: 1, z: 0 }),
+                            Quat.fromVec3Radians({ x: handRoll, y: handYaw, z: -handPitch }));
+                    } else {
+                        handRotation = Quat.multiply(Quat.angleAxis(90.0, { x: 0, y: 1, z: 0 }),
+                            Quat.fromVec3Radians({ x: -handRoll, y: handYaw, z: handPitch }));
+                    }
+
+                    // Hand rotation in avatar coordinates ...
+                    cameraOrientation.x = -cameraOrientation.x;
+                    cameraOrientation.z = -cameraOrientation.z;
+                    handRotation = Quat.multiply(cameraOrientation, handRotation);
+                    handRotation = Quat.multiply(inverseAvatarOrientation, handRotation);
 
                 } else {
-                    handRotation = Quat.multiply(Quat.angleAxis(90.0, { x: 0, y: 1, z: 0 }),
+
-                        Quat.fromVec3Radians({ x: -handRoll, y: handYaw, z: handPitch }));
+                    handOffset = hands[h].controller.getAbsTranslation();
+                    handOffset = {
+                        x: -handOffset.x,
+                        y: hands[h].zeroPosition.y + handOffset.y,
+                        z: hands[h].zeroPosition.z - handOffset.z
+                    };
+
+                    // TODO: 2.0* scale factors should not be necessary; Leap Motion controller code needs investigating.
+                    handRoll = 2.0 * -hands[h].controller.getAbsRotation().z;
+                    wristAbsRotation = wrists[h].controller.getAbsRotation();
+                    handPitch = 2.0 * -wristAbsRotation.x;
+                    handYaw = 2.0 * wristAbsRotation.y;
+
+                    // TODO: Leap Motion controller's right-hand roll calculation only works if physical hand is upside down.
+                    // Approximate fix is to add a fudge factor.
+                    if (h === 1 && isWindows) {
+                        handRoll = handRoll + 0.6 * PI;
+                    }
+
+                    // Hand position and orientation ...
+                    if (h === 0) {
+                        handRotation = Quat.multiply(Quat.angleAxis(-90.0, { x: 0, y: 1, z: 0 }),
+                            Quat.fromVec3Radians({ x: handRoll, y: handYaw, z: -handPitch }));
+                    } else {
+                        handRotation = Quat.multiply(Quat.angleAxis(90.0, { x: 0, y: 1, z: 0 }),
+                            Quat.fromVec3Radians({ x: -handRoll, y: handYaw, z: handPitch }));
+                    }
                 }
+
                 MyAvatar.setJointModelPositionAndOrientation(hands[h].jointName, handOffset, handRotation, true);
 
                 // Finger joints ...

interface/src/Application.cpp

@@ -599,8 +599,11 @@ void Application::paintGL() {
 
     if (_myCamera.getMode() == CAMERA_MODE_FIRST_PERSON) {
         _myCamera.setTightness(0.0f);  //  In first person, camera follows (untweaked) head exactly without delay
-        _myCamera.setTargetPosition(_myAvatar->getHead()->getEyePosition());
+        if (!OculusManager::isConnected()) {
-        _myCamera.setTargetRotation(_myAvatar->getHead()->getCameraOrientation());
+            _myCamera.setTargetPosition(_myAvatar->getHead()->getEyePosition());
+            _myCamera.setTargetRotation(_myAvatar->getHead()->getCameraOrientation());
+        }
+        // OculusManager::display() updates camera position and rotation a bit further on.
 
     } else if (_myCamera.getMode() == CAMERA_MODE_THIRD_PERSON) {
         //Note, the camera distance is set in Camera::setMode() so we dont have to do it here.
@@ -630,7 +633,9 @@ void Application::paintGL() {
     }
 
     // Update camera position
-    _myCamera.update( 1.f/_fps );
+    if (!OculusManager::isConnected()) {
+        _myCamera.update(1.f / _fps);
+    }
 
     // Note: whichCamera is used to pick between the normal camera myCamera for our
     // main camera, vs, an alternate camera. The alternate camera we support right now
@@ -640,7 +645,7 @@ void Application::paintGL() {
     // Why have two cameras? Well, one reason is that because in the case of the renderViewFrustum()
     // code, we want to keep the state of "myCamera" intact, so we can render what the view frustum of
     // myCamera is. But we also want to do meaningful camera transforms on OpenGL for the offset camera
-    Camera whichCamera = _myCamera;
+    Camera* whichCamera = &_myCamera;
 
     if (Menu::getInstance()->isOptionChecked(MenuOption::DisplayFrustum)) {
 
@@ -654,7 +659,7 @@ void Application::paintGL() {
         _viewFrustumOffsetCamera.setDistance(viewFrustumOffset.distance);
         _viewFrustumOffsetCamera.initialize(); // force immediate snap to ideal position and orientation
         _viewFrustumOffsetCamera.update(1.f/_fps);
-        whichCamera = _viewFrustumOffsetCamera;
+        whichCamera = &_viewFrustumOffsetCamera;
     }
 
     if (Menu::getInstance()->getShadowsEnabled()) {
@@ -667,15 +672,16 @@ void Application::paintGL() {
         glClear(GL_COLOR_BUFFER_BIT);
         
         //When in mirror mode, use camera rotation. Otherwise, use body rotation
-        if (whichCamera.getMode() == CAMERA_MODE_MIRROR) {
+        if (whichCamera->getMode() == CAMERA_MODE_MIRROR) {
-            OculusManager::display(whichCamera.getRotation(), whichCamera.getPosition(), whichCamera);
+            OculusManager::display(whichCamera->getRotation(), whichCamera->getPosition(), *whichCamera);
         } else {
-            OculusManager::display(_myAvatar->getWorldAlignedOrientation(), _myAvatar->getDefaultEyePosition(), whichCamera);
+            OculusManager::display(_myAvatar->getWorldAlignedOrientation(), _myAvatar->getDefaultEyePosition(), *whichCamera);
         }
+        _myCamera.update(1.f / _fps);
 
     } else if (TV3DManager::isConnected()) {
        
-        TV3DManager::display(whichCamera);
+        TV3DManager::display(*whichCamera);
 
     } else {
         _glowEffect.prepare();
@@ -683,7 +689,7 @@ void Application::paintGL() {
         glMatrixMode(GL_MODELVIEW);
         glPushMatrix();
         glLoadIdentity();
-        displaySide(whichCamera);
+        displaySide(*whichCamera);
         glPopMatrix();
 
         if (Menu::getInstance()->isOptionChecked(MenuOption::Mirror)) {

interface/src/Menu.cpp

@@ -453,6 +453,9 @@ Menu::Menu() :
     addCheckableActionToQMenuAndActionHash(sixenseOptionsMenu, MenuOption::SixenseMouseInput, 0, true);
     addCheckableActionToQMenuAndActionHash(sixenseOptionsMenu, MenuOption::SixenseLasers, 0, false);
 
+    QMenu* leapOptionsMenu = handOptionsMenu->addMenu("Leap Motion");
+    addCheckableActionToQMenuAndActionHash(leapOptionsMenu, MenuOption::LeapMotionOnHMD, 0, false);
+
     QMenu* networkMenu = developerMenu->addMenu("Network");
     addCheckableActionToQMenuAndActionHash(networkMenu, MenuOption::DisableNackPackets, 0, false);
     addCheckableActionToQMenuAndActionHash(networkMenu,

interface/src/Menu.h

@@ -403,6 +403,7 @@ namespace MenuOption {
     const QString IncreaseAvatarSize = "Increase Avatar Size";
     const QString IncreaseVoxelSize = "Increase Voxel Size";
     const QString KeyboardMotorControl = "Enable Keyboard Motor Control";
+    const QString LeapMotionOnHMD = "Leap Motion on HMD";
     const QString LoadScript = "Open and Run Script File...";
     const QString LoadScriptURL = "Open and Run Script from URL...";
     const QString LodTools = "LOD Tools";

interface/src/devices/Leapmotion.cpp

@@ -8,8 +8,9 @@
 //  Distributed under the Apache License, Version 2.0.
 //  See the accompanying file LICENSE or http://www.apache.org/licenses/LICENSE-2.0.html
 //
-#include "SharedUtil.h"
 #include "Leapmotion.h"
+#include "Menu.h"
+#include "SharedUtil.h"
 
 const int PALMROOT_NUM_JOINTS = 3;
 const int FINGER_NUM_JOINTS = 4;
@@ -101,6 +102,12 @@ Leapmotion::Leapmotion() :
             }
         }
     }
+
+#ifdef HAVE_LEAPMOTION
+    if (Menu::getInstance()->isOptionChecked(MenuOption::LeapMotionOnHMD)) {
+        _controller.setPolicyFlags(Leap::Controller::POLICY_OPTIMIZE_HMD);
+    }
+#endif
 }
 
 Leapmotion::~Leapmotion() {

interface/src/devices/OculusManager.cpp

@@ -412,6 +412,10 @@ void OculusManager::display(const glm::quat &bodyOrientation, const glm::vec3 &p
     
     glBindTexture(GL_TEXTURE_2D, 0);
     
+    // Update camera for use by rest of Interface.
+    whichCamera.setTargetPosition((_leftEyePosition + _rightEyePosition) / 2.f);
+    whichCamera.setTargetRotation(_camera->getTargetRotation());
+
 #endif
 }