Merge pull request #9054 from howard-stearns/third-person-lasers

make hand-controller ui work in third person

interface/src/avatar/MyAvatar.cpp

@@ -2333,14 +2333,49 @@ bool MyAvatar::hasDriveInput() const {
     return fabsf(_driveKeys[TRANSLATE_X]) > 0.0f || fabsf(_driveKeys[TRANSLATE_Y]) > 0.0f || fabsf(_driveKeys[TRANSLATE_Z]) > 0.0f;
 }
 
+// The resulting matrix is used to render the hand controllers, even if the camera is decoupled from the avatar.
+// Specificly, if we are rendering using a third person camera.  We would like to render the hand controllers in front of the camera,
+// not in front of the avatar.
+glm::mat4 MyAvatar::computeCameraRelativeHandControllerMatrix(const glm::mat4& controllerSensorMatrix) const {
+
+    // Fetch the current camera transform.
+    glm::mat4 cameraWorldMatrix = qApp->getCamera()->getTransform();
+    if (qApp->getCamera()->getMode() == CAMERA_MODE_MIRROR) {
+        cameraWorldMatrix *= createMatFromScaleQuatAndPos(vec3(-1.0f, 1.0f, 1.0f), glm::quat(), glm::vec3());
+    }
+
+    // compute a NEW sensorToWorldMatrix for the camera.  The equation is cameraWorldMatrix = cameraSensorToWorldMatrix * _hmdSensorMatrix.
+    // here we solve for the unknown cameraSensorToWorldMatrix.
+    glm::mat4 cameraSensorToWorldMatrix = cameraWorldMatrix * glm::inverse(_hmdSensorMatrix);
+
+    // Using the new cameraSensorToWorldMatrix, compute where the controller is in world space.
+    glm::mat4 controllerWorldMatrix = cameraSensorToWorldMatrix * controllerSensorMatrix;
+
+    // move it into avatar space
+    glm::mat4 avatarMatrix = createMatFromQuatAndPos(getOrientation(), getPosition());
+    return glm::inverse(avatarMatrix) * controllerWorldMatrix;
+}
+
 glm::quat MyAvatar::getAbsoluteJointRotationInObjectFrame(int index) const {
-    switch(index) {
+    switch (index) {
         case CONTROLLER_LEFTHAND_INDEX: {
             return getLeftHandControllerPoseInAvatarFrame().getRotation();
         }
         case CONTROLLER_RIGHTHAND_INDEX: {
             return getRightHandControllerPoseInAvatarFrame().getRotation();
         }
+        case CAMERA_RELATIVE_CONTROLLER_LEFTHAND_INDEX: {
+            auto pose = _leftHandControllerPoseInSensorFrameCache.get();
+            glm::mat4 controllerSensorMatrix = createMatFromQuatAndPos(pose.rotation, pose.translation);
+            glm::mat4 result = computeCameraRelativeHandControllerMatrix(controllerSensorMatrix);
+            return glmExtractRotation(result);
+        }
+        case CAMERA_RELATIVE_CONTROLLER_RIGHTHAND_INDEX: {
+            auto pose = _rightHandControllerPoseInSensorFrameCache.get();
+            glm::mat4 controllerSensorMatrix = createMatFromQuatAndPos(pose.rotation, pose.translation);
+            glm::mat4 result = computeCameraRelativeHandControllerMatrix(controllerSensorMatrix);
+            return glmExtractRotation(result);
+        }
         default: {
             return Avatar::getAbsoluteJointRotationInObjectFrame(index);
         }
@@ -2348,13 +2383,25 @@ glm::quat MyAvatar::getAbsoluteJointRotationInObjectFrame(int index) const {
 }
 
 glm::vec3 MyAvatar::getAbsoluteJointTranslationInObjectFrame(int index) const {
-    switch(index) {
+    switch (index) {
         case CONTROLLER_LEFTHAND_INDEX: {
             return getLeftHandControllerPoseInAvatarFrame().getTranslation();
         }
         case CONTROLLER_RIGHTHAND_INDEX: {
             return getRightHandControllerPoseInAvatarFrame().getTranslation();
         }
+        case CAMERA_RELATIVE_CONTROLLER_LEFTHAND_INDEX: {
+            auto pose = _leftHandControllerPoseInSensorFrameCache.get();
+            glm::mat4 controllerSensorMatrix = createMatFromQuatAndPos(pose.rotation, pose.translation);
+            glm::mat4 result = computeCameraRelativeHandControllerMatrix(controllerSensorMatrix);
+            return extractTranslation(result);
+        }
+        case CAMERA_RELATIVE_CONTROLLER_RIGHTHAND_INDEX: {
+            auto pose = _rightHandControllerPoseInSensorFrameCache.get();
+            glm::mat4 controllerSensorMatrix = createMatFromQuatAndPos(pose.rotation, pose.translation);
+            glm::mat4 result = computeCameraRelativeHandControllerMatrix(controllerSensorMatrix);
+            return extractTranslation(result);
+        }
         default: {
             return Avatar::getAbsoluteJointTranslationInObjectFrame(index);
         }

interface/src/avatar/MyAvatar.h

@@ -374,6 +374,7 @@ private:
 
     void clampTargetScaleToDomainLimits();
     void clampScaleChangeToDomainLimits(float desiredScale);
+    glm::mat4 computeCameraRelativeHandControllerMatrix(const glm::mat4& controllerSensorMatrix) const;
 
     float _driveKeys[MAX_DRIVE_KEYS];
     bool _wasPushing;

libraries/avatars/src/AvatarData.cpp

@@ -953,6 +953,12 @@ int AvatarData::getFauxJointIndex(const QString& name) const {
     if (name == "_CONTROLLER_RIGHTHAND") {
         return CONTROLLER_RIGHTHAND_INDEX;
     }
+    if (name == "_CAMERA_RELATIVE_CONTROLLER_LEFTHAND") {
+        return CAMERA_RELATIVE_CONTROLLER_LEFTHAND_INDEX;
+    }
+    if (name == "_CAMERA_RELATIVE_CONTROLLER_RIGHTHAND") {
+        return CAMERA_RELATIVE_CONTROLLER_RIGHTHAND_INDEX;
+    }
     return -1;
 }
 

libraries/avatars/src/AvatarData.h

@@ -539,6 +539,7 @@ void RayToAvatarIntersectionResultFromScriptValue(const QScriptValue& object, Ra
 const int SENSOR_TO_WORLD_MATRIX_INDEX = 65534; // -2
 const int CONTROLLER_RIGHTHAND_INDEX = 65533; // -3
 const int CONTROLLER_LEFTHAND_INDEX = 65532; // -4
-
+const int CAMERA_RELATIVE_CONTROLLER_RIGHTHAND_INDEX = 65531; // -5
+const int CAMERA_RELATIVE_CONTROLLER_LEFTHAND_INDEX = 65530; // -6
 
 #endif // hifi_AvatarData_h

scripts/system/controllers/controllerDisplay.js

@@ -48,7 +48,6 @@ createControllerDisplay = function(config) {
         mappingName: "mapping-display-" + Math.random(),
 
         setVisible: function(visible) {
-            debug("Setting visible", this.overlays.length);
             for (var i = 0; i < this.overlays.length; ++i) {
                 Overlays.editOverlay(this.overlays[i], {
                     visible: visible
@@ -77,9 +76,6 @@ createControllerDisplay = function(config) {
                         textures[part.textureName] = layer.defaultTextureURL;
                     }
                     for (var i = 0; i < this.partOverlays[partName].length; ++i) {
-
-                        // AJT: REMOVE
-                        print("AJT: Overlays.editOverlays(" + partName + ", " + i + ", { textures: " + JSON.stringify(textures) + " })");
                         Overlays.editOverlay(this.partOverlays[partName][i], {
                             textures: textures
                         });

scripts/system/controllers/handControllerPointer.js

@@ -489,9 +489,11 @@ function setColoredLaser() { // answer trigger state if lasers supported, else f
     var color = (activeTrigger.state === 'full') ? LASER_TRIGGER_COLOR_XYZW : LASER_SEARCH_COLOR_XYZW;
 
     if (!HMD.isHandControllerAvailable()) {
-        var position = MyAvatar.getHeadPosition();
+        // NOTE: keep this offset in sync with scripts/system/librarires/controllers.js:57
-        var direction = Quat.getUp(Quat.multiply(MyAvatar.headOrientation, Quat.angleAxis(-90, { x: 1, y: 0, z: 0 })));
+        var VERTICAL_HEAD_LASER_OFFSET = 0.1;
-        return HMD.setExtraLaser(position, true, color, direction);
+        var position = Vec3.sum(HMD.position, Vec3.multiplyQbyV(HMD.orientation, {x: 0, y: VERTICAL_HEAD_LASER_OFFSET, z: 0}));
+        var orientation = Quat.multiply(HMD.orientation, Quat.angleAxis(-90, { x: 1, y: 0, z: 0 }));
+        return HMD.setExtraLaser(position, true, color, Quat.getUp(orientation));
     }
 
     return HMD.setHandLasers(activeHudLaser, true, color, SYSTEM_LASER_DIRECTION) && activeTrigger.state;
@@ -511,10 +513,6 @@ function update() {
         return off(); // Let them use mouse in peace.
     }
 
-    if (!Menu.isOptionChecked("First Person")) {
-        return off(); // What to do? menus can be behind hand!
-    }
-
     if ((!Window.hasFocus() && !HMD.active) || !Reticle.allowMouseCapture) {
         // In desktop it's pretty clear when another app is on top. In that case we bail, because
         // hand controllers might be sputtering "valid" data and that will keep someone from deliberately

scripts/system/controllers/viveControllerConfiguration.js

@@ -69,7 +69,7 @@ VIVE_CONTROLLER_CONFIGURATION_LEFT = {
     controllers: [
         {
             modelURL: viveModelURL,
-            jointIndex: MyAvatar.getJointIndex("_CONTROLLER_LEFTHAND"),
+            jointIndex: MyAvatar.getJointIndex("_CAMERA_RELATIVE_CONTROLLER_LEFTHAND"),
             naturalPosition: viveNaturalPosition,
             rotation: leftBaseRotation,
             position: Vec3.multiplyQbyV(Quat.fromPitchYawRollDegrees(0, 0, 45), leftBasePosition),
@@ -195,8 +195,7 @@ VIVE_CONTROLLER_CONFIGURATION_RIGHT = {
     controllers: [
         {
             modelURL: viveModelURL,
-            jointIndex: MyAvatar.getJointIndex("_CONTROLLER_RIGHTHAND"),
+            jointIndex: MyAvatar.getJointIndex("_CAMERA_RELATIVE_CONTROLLER_RIGHTHAND"),
-
             rotation: rightBaseRotation,
             position: Vec3.multiplyQbyV(Quat.fromPitchYawRollDegrees(0, 0, -45), rightBasePosition),
 

scripts/system/hmd.js

@@ -22,11 +22,24 @@ var desktopMenuItemName = "Desktop";
     }
 });
 
+var controllerDisplay = false;
+function updateControllerDisplay() {
+    if (HMD.active && Menu.isOptionChecked("Third Person")) {
+        if (!controllerDisplay) {
+            HMD.requestShowHandControllers();
+            controllerDisplay = true;
+        }
+    } else if (controllerDisplay) {
+        HMD.requestHideHandControllers();
+        controllerDisplay = false;
+    }
+}
+
 var toolBar = Toolbars.getToolbar("com.highfidelity.interface.toolbar.system");
 var button;
 // Independent and Entity mode make people sick. Third Person and Mirror have traps that we need to work through.
 // Disable them in hmd.
-var desktopOnlyViews = ['Third Person', 'Mirror', 'Independent Mode', 'Entity Mode'];
+var desktopOnlyViews = ['Mirror', 'Independent Mode', 'Entity Mode'];
 function onHmdChanged(isHmd) {
     button.writeProperty('buttonState', isHmd ? 0 : 1);
     button.writeProperty('defaultState', isHmd ? 0 : 1);
@@ -34,6 +47,7 @@ function onHmdChanged(isHmd) {
     desktopOnlyViews.forEach(function (view) {
         Menu.setMenuEnabled("View>" + view, !isHmd);
     });
+    updateControllerDisplay();
 }
 function onClicked(){
     var isDesktop = Menu.isOptionChecked(desktopMenuItemName);
@@ -52,11 +66,13 @@ if (headset) {
 
     button.clicked.connect(onClicked);
     HMD.displayModeChanged.connect(onHmdChanged);
+    Camera.modeUpdated.connect(updateControllerDisplay);
 
     Script.scriptEnding.connect(function () {
         toolBar.removeButton("hmdToggle");
         button.clicked.disconnect(onClicked);
         HMD.displayModeChanged.disconnect(onHmdChanged);
+        Camera.modeUpdated.disconnect(updateControllerDisplay);
     });
 }
 

scripts/system/libraries/controllers.js

@@ -35,16 +35,27 @@ getControllerWorldLocation = function (handController, doOffset) {
     var position;
     var pose = Controller.getPoseValue(handController);
     var valid = pose.valid;
+    var controllerJointIndex;
     if (pose.valid) {
-        orientation = Quat.multiply(MyAvatar.orientation, pose.rotation);
+        if (handController === Controller.Standard.RightHand) {
-        position = Vec3.sum(Vec3.multiplyQbyV(MyAvatar.orientation, pose.translation), MyAvatar.position);
+            controllerJointIndex = MyAvatar.getJointIndex("_CAMERA_RELATIVE_CONTROLLER_RIGHTHAND");
+        } else {
+            controllerJointIndex = MyAvatar.getJointIndex("_CAMERA_RELATIVE_CONTROLLER_LEFTHAND");
+        }
+        orientation = Quat.multiply(MyAvatar.orientation, MyAvatar.getAbsoluteJointRotationInObjectFrame(controllerJointIndex));
+        position = Vec3.sum(MyAvatar.position, Vec3.multiplyQbyV(MyAvatar.orientation, MyAvatar.getAbsoluteJointTranslationInObjectFrame(controllerJointIndex)));
+
         // add to the real position so the grab-point is out in front of the hand, a bit
         if (doOffset) {
-            position = Vec3.sum(position, Vec3.multiplyQbyV(orientation, getGrabPointSphereOffset(handController)));
+            var offset = getGrabPointSphereOffset(handController);
+            position = Vec3.sum(position, Vec3.multiplyQbyV(orientation, offset));
         }
+
     } else if (!HMD.isHandControllerAvailable()) {
-        position = MyAvatar.getHeadPosition();
+        // NOTE: keep this offset in sync with scripts/system/controllers/handControllerPointer.js:493
-        orientation = Quat.multiply(MyAvatar.headOrientation, Quat.angleAxis(-90, { x: 1, y: 0, z: 0 }));
+        var VERTICAL_HEAD_LASER_OFFSET = 0.1;
+        position = Vec3.sum(Camera.position, Vec3.multiplyQbyV(Camera.orientation, {x: 0, y: VERTICAL_HEAD_LASER_OFFSET, z: 0}));
+        orientation = Quat.multiply(Camera.orientation, Quat.angleAxis(-90, { x: 1, y: 0, z: 0 }));
         valid = true;
     }