From e2fded846f5e35d1cba697e8b0f3bbc54efab64a Mon Sep 17 00:00:00 2001 From: howard-stearns Date: Fri, 11 Nov 2016 11:10:42 -0800 Subject: [PATCH 1/5] make hand-controller ui work in third person --- scripts/system/controllers/handControllerPointer.js | 4 ---- scripts/system/hmd.js | 2 +- scripts/system/libraries/controllers.js | 3 +++ 3 files changed, 4 insertions(+), 5 deletions(-) diff --git a/scripts/system/controllers/handControllerPointer.js b/scripts/system/controllers/handControllerPointer.js index ce9b8e403f..5e8c0cb6ba 100644 --- a/scripts/system/controllers/handControllerPointer.js +++ b/scripts/system/controllers/handControllerPointer.js @@ -511,10 +511,6 @@ function update() { return off(); // Let them use mouse in peace. } - if (!Menu.isOptionChecked("First Person")) { - return off(); // What to do? menus can be behind hand! - } - if ((!Window.hasFocus() && !HMD.active) || !Reticle.allowMouseCapture) { // In desktop it's pretty clear when another app is on top. In that case we bail, because // hand controllers might be sputtering "valid" data and that will keep someone from deliberately diff --git a/scripts/system/hmd.js b/scripts/system/hmd.js index e1c846806f..bac8580222 100644 --- a/scripts/system/hmd.js +++ b/scripts/system/hmd.js @@ -26,7 +26,7 @@ var toolBar = Toolbars.getToolbar("com.highfidelity.interface.toolbar.system"); var button; // Independent and Entity mode make people sick. Third Person and Mirror have traps that we need to work through. // Disable them in hmd. -var desktopOnlyViews = ['Third Person', 'Mirror', 'Independent Mode', 'Entity Mode']; +var desktopOnlyViews = ['Mirror', 'Independent Mode', 'Entity Mode']; function onHmdChanged(isHmd) { button.writeProperty('buttonState', isHmd ? 0 : 1); button.writeProperty('defaultState', isHmd ? 0 : 1); diff --git a/scripts/system/libraries/controllers.js b/scripts/system/libraries/controllers.js index 38febf2de4..51b44ecc5c 100644 --- a/scripts/system/libraries/controllers.js +++ b/scripts/system/libraries/controllers.js @@ -42,6 +42,9 @@ getControllerWorldLocation = function (handController, doOffset) { if (doOffset) { position = Vec3.sum(position, Vec3.multiplyQbyV(orientation, getGrabPointSphereOffset(handController))); } + if (Menu.isOptionChecked("Third Person")) { + position = Vec3.sum(position, Vec3.subtract(Camera.position, MyAvatar.getEyePosition())); + } } else if (!HMD.isHandControllerAvailable()) { position = MyAvatar.getHeadPosition(); orientation = Quat.multiply(MyAvatar.headOrientation, Quat.angleAxis(-90, { x: 1, y: 0, z: 0 })); From 47bef18bd4c09526f7a9841d690c572ff76beb4a Mon Sep 17 00:00:00 2001 From: howard-stearns Date: Mon, 14 Nov 2016 13:05:35 -0800 Subject: [PATCH 2/5] show the hand controllers when in third-person+hmd, but currently, they're at avatar hands instead of physical hand, which is not what we want! --- scripts/system/hmd.js | 16 ++++++++++++++++ 1 file changed, 16 insertions(+) diff --git a/scripts/system/hmd.js b/scripts/system/hmd.js index bac8580222..5dd06de8eb 100644 --- a/scripts/system/hmd.js +++ b/scripts/system/hmd.js @@ -22,6 +22,19 @@ var desktopMenuItemName = "Desktop"; } }); +var controllerDisplay = false; +function updateControllerDisplay() { + if (HMD.active && Menu.isOptionChecked("Third Person")) { + if (!controllerDisplay) { + HMD.requestShowHandControllers(); + controllerDisplay = true; + } + } else if (controllerDisplay) { + HMD.requestHideHandControllers(); + controllerDisplay = false; + } +} + var toolBar = Toolbars.getToolbar("com.highfidelity.interface.toolbar.system"); var button; // Independent and Entity mode make people sick. Third Person and Mirror have traps that we need to work through. @@ -34,6 +47,7 @@ function onHmdChanged(isHmd) { desktopOnlyViews.forEach(function (view) { Menu.setMenuEnabled("View>" + view, !isHmd); }); + updateControllerDisplay(); } function onClicked(){ var isDesktop = Menu.isOptionChecked(desktopMenuItemName); @@ -52,11 +66,13 @@ if (headset) { button.clicked.connect(onClicked); HMD.displayModeChanged.connect(onHmdChanged); + Camera.modeUpdated.connect(updateControllerDisplay); Script.scriptEnding.connect(function () { toolBar.removeButton("hmdToggle"); button.clicked.disconnect(onClicked); HMD.displayModeChanged.disconnect(onHmdChanged); + Camera.modeUpdated.disconnect(updateControllerDisplay); }); } From 38ac6fff03950c96e28008de7b0a94e22898fb73 Mon Sep 17 00:00:00 2001 From: "Anthony J. Thibault" Date: Thu, 17 Nov 2016 11:33:12 -0800 Subject: [PATCH 3/5] Hand Controller rendering is camera relative Basically, when using the third person camera in HMD mode. If the controllers are shown. They should be shown in front of the users camera, not in front of the users avatar. To accomplish this, two new faux joint indices are introduced. CAMERA_RELATIVE_CONTROLLER_RIGHTHAND_INDEX and CAMERA_RELATIVE_CONTROLLER_LEFTHAND_INDEX. These joint indices can be used for Overlay parenting. (But not for entity parenting because they are not transmitted over the network). They can also be queried for by using the MyAvatar.getAbsoluteJointRotationInObjectFrame() call. These new indices are now used by the controllerDisplay.js for the hand controller rendering. They are also used by system/libraries/controllers.js as the origin for hand controller grabbing and interaction lasers. --- interface/src/avatar/MyAvatar.cpp | 51 ++++++++++++++++++- interface/src/avatar/MyAvatar.h | 1 + libraries/avatars/src/AvatarData.cpp | 6 +++ libraries/avatars/src/AvatarData.h | 3 +- .../viveControllerConfiguration.js | 5 +- scripts/system/libraries/controllers.js | 18 ++++--- 6 files changed, 72 insertions(+), 12 deletions(-) diff --git a/interface/src/avatar/MyAvatar.cpp b/interface/src/avatar/MyAvatar.cpp index b49ee94ff6..b8f02fd17e 100644 --- a/interface/src/avatar/MyAvatar.cpp +++ b/interface/src/avatar/MyAvatar.cpp @@ -2333,14 +2333,49 @@ bool MyAvatar::hasDriveInput() const { return fabsf(_driveKeys[TRANSLATE_X]) > 0.0f || fabsf(_driveKeys[TRANSLATE_Y]) > 0.0f || fabsf(_driveKeys[TRANSLATE_Z]) > 0.0f; } +// The resulting matrix is used to render the hand controllers, even if the camera is decoupled from the avatar. +// Specificly, if we are rendering using a third person camera. We would like to render the hand controllers in front of the camera, +// not in front of the avatar. +glm::mat4 MyAvatar::computeCameraRelativeHandControllerMatrix(const glm::mat4& controllerSensorMatrix) const { + + // Fetch the current camera transform. + glm::mat4 cameraWorldMatrix = qApp->getCamera()->getTransform(); + if (qApp->getCamera()->getMode() == CAMERA_MODE_MIRROR) { + cameraWorldMatrix *= createMatFromScaleQuatAndPos(vec3(-1.0f, 1.0f, 1.0f), glm::quat(), glm::vec3()); + } + + // compute a NEW sensorToWorldMatrix for the camera. The equation is cameraWorldMatrix = cameraSensorToWorldMatrix * _hmdSensorMatrix. + // here we solve for the unknown cameraSensorToWorldMatrix. + glm::mat4 cameraSensorToWorldMatrix = cameraWorldMatrix * glm::inverse(_hmdSensorMatrix); + + // Using the new cameraSensorToWorldMatrix, compute where the controller is in world space. + glm::mat4 controllerWorldMatrix = cameraSensorToWorldMatrix * controllerSensorMatrix; + + // move it into avatar space + glm::mat4 avatarMatrix = createMatFromQuatAndPos(getOrientation(), getPosition()); + return glm::inverse(avatarMatrix) * controllerWorldMatrix; +} + glm::quat MyAvatar::getAbsoluteJointRotationInObjectFrame(int index) const { - switch(index) { + switch (index) { case CONTROLLER_LEFTHAND_INDEX: { return getLeftHandControllerPoseInAvatarFrame().getRotation(); } case CONTROLLER_RIGHTHAND_INDEX: { return getRightHandControllerPoseInAvatarFrame().getRotation(); } + case CAMERA_RELATIVE_CONTROLLER_LEFTHAND_INDEX: { + auto pose = _leftHandControllerPoseInSensorFrameCache.get(); + glm::mat4 controllerSensorMatrix = createMatFromQuatAndPos(pose.rotation, pose.translation); + glm::mat4 result = computeCameraRelativeHandControllerMatrix(controllerSensorMatrix); + return glmExtractRotation(result); + } + case CAMERA_RELATIVE_CONTROLLER_RIGHTHAND_INDEX: { + auto pose = _rightHandControllerPoseInSensorFrameCache.get(); + glm::mat4 controllerSensorMatrix = createMatFromQuatAndPos(pose.rotation, pose.translation); + glm::mat4 result = computeCameraRelativeHandControllerMatrix(controllerSensorMatrix); + return glmExtractRotation(result); + } default: { return Avatar::getAbsoluteJointRotationInObjectFrame(index); } @@ -2348,13 +2383,25 @@ glm::quat MyAvatar::getAbsoluteJointRotationInObjectFrame(int index) const { } glm::vec3 MyAvatar::getAbsoluteJointTranslationInObjectFrame(int index) const { - switch(index) { + switch (index) { case CONTROLLER_LEFTHAND_INDEX: { return getLeftHandControllerPoseInAvatarFrame().getTranslation(); } case CONTROLLER_RIGHTHAND_INDEX: { return getRightHandControllerPoseInAvatarFrame().getTranslation(); } + case CAMERA_RELATIVE_CONTROLLER_LEFTHAND_INDEX: { + auto pose = _leftHandControllerPoseInSensorFrameCache.get(); + glm::mat4 controllerSensorMatrix = createMatFromQuatAndPos(pose.rotation, pose.translation); + glm::mat4 result = computeCameraRelativeHandControllerMatrix(controllerSensorMatrix); + return extractTranslation(result); + } + case CAMERA_RELATIVE_CONTROLLER_RIGHTHAND_INDEX: { + auto pose = _rightHandControllerPoseInSensorFrameCache.get(); + glm::mat4 controllerSensorMatrix = createMatFromQuatAndPos(pose.rotation, pose.translation); + glm::mat4 result = computeCameraRelativeHandControllerMatrix(controllerSensorMatrix); + return extractTranslation(result); + } default: { return Avatar::getAbsoluteJointTranslationInObjectFrame(index); } diff --git a/interface/src/avatar/MyAvatar.h b/interface/src/avatar/MyAvatar.h index 60049bea67..f081ec533b 100644 --- a/interface/src/avatar/MyAvatar.h +++ b/interface/src/avatar/MyAvatar.h @@ -374,6 +374,7 @@ private: void clampTargetScaleToDomainLimits(); void clampScaleChangeToDomainLimits(float desiredScale); + glm::mat4 computeCameraRelativeHandControllerMatrix(const glm::mat4& controllerSensorMatrix) const; float _driveKeys[MAX_DRIVE_KEYS]; bool _wasPushing; diff --git a/libraries/avatars/src/AvatarData.cpp b/libraries/avatars/src/AvatarData.cpp index 2469d0df04..cbda35966d 100644 --- a/libraries/avatars/src/AvatarData.cpp +++ b/libraries/avatars/src/AvatarData.cpp @@ -953,6 +953,12 @@ int AvatarData::getFauxJointIndex(const QString& name) const { if (name == "_CONTROLLER_RIGHTHAND") { return CONTROLLER_RIGHTHAND_INDEX; } + if (name == "_CAMERA_RELATIVE_CONTROLLER_LEFTHAND") { + return CAMERA_RELATIVE_CONTROLLER_LEFTHAND_INDEX; + } + if (name == "_CAMERA_RELATIVE_CONTROLLER_RIGHTHAND") { + return CAMERA_RELATIVE_CONTROLLER_RIGHTHAND_INDEX; + } return -1; } diff --git a/libraries/avatars/src/AvatarData.h b/libraries/avatars/src/AvatarData.h index 97879700ee..27d4e336d5 100644 --- a/libraries/avatars/src/AvatarData.h +++ b/libraries/avatars/src/AvatarData.h @@ -539,6 +539,7 @@ void RayToAvatarIntersectionResultFromScriptValue(const QScriptValue& object, Ra const int SENSOR_TO_WORLD_MATRIX_INDEX = 65534; // -2 const int CONTROLLER_RIGHTHAND_INDEX = 65533; // -3 const int CONTROLLER_LEFTHAND_INDEX = 65532; // -4 - +const int CAMERA_RELATIVE_CONTROLLER_RIGHTHAND_INDEX = 65531; // -5 +const int CAMERA_RELATIVE_CONTROLLER_LEFTHAND_INDEX = 65530; // -6 #endif // hifi_AvatarData_h diff --git a/scripts/system/controllers/viveControllerConfiguration.js b/scripts/system/controllers/viveControllerConfiguration.js index b49c3e1d04..4b8fc34ef7 100644 --- a/scripts/system/controllers/viveControllerConfiguration.js +++ b/scripts/system/controllers/viveControllerConfiguration.js @@ -69,7 +69,7 @@ VIVE_CONTROLLER_CONFIGURATION_LEFT = { controllers: [ { modelURL: viveModelURL, - jointIndex: MyAvatar.getJointIndex("_CONTROLLER_LEFTHAND"), + jointIndex: MyAvatar.getJointIndex("_CAMERA_RELATIVE_CONTROLLER_LEFTHAND"), naturalPosition: viveNaturalPosition, rotation: leftBaseRotation, position: Vec3.multiplyQbyV(Quat.fromPitchYawRollDegrees(0, 0, 45), leftBasePosition), @@ -195,8 +195,7 @@ VIVE_CONTROLLER_CONFIGURATION_RIGHT = { controllers: [ { modelURL: viveModelURL, - jointIndex: MyAvatar.getJointIndex("_CONTROLLER_RIGHTHAND"), - + jointIndex: MyAvatar.getJointIndex("_CAMERA_RELATIVE_CONTROLLER_RIGHTHAND"), rotation: rightBaseRotation, position: Vec3.multiplyQbyV(Quat.fromPitchYawRollDegrees(0, 0, -45), rightBasePosition), diff --git a/scripts/system/libraries/controllers.js b/scripts/system/libraries/controllers.js index 51b44ecc5c..2e3a4a1e57 100644 --- a/scripts/system/libraries/controllers.js +++ b/scripts/system/libraries/controllers.js @@ -35,16 +35,22 @@ getControllerWorldLocation = function (handController, doOffset) { var position; var pose = Controller.getPoseValue(handController); var valid = pose.valid; + var controllerJointIndex; if (pose.valid) { - orientation = Quat.multiply(MyAvatar.orientation, pose.rotation); - position = Vec3.sum(Vec3.multiplyQbyV(MyAvatar.orientation, pose.translation), MyAvatar.position); + if (handController === Controller.Standard.RightHand) { + controllerJointIndex = MyAvatar.getJointIndex("_CAMERA_RELATIVE_CONTROLLER_RIGHTHAND"); + } else { + controllerJointIndex = MyAvatar.getJointIndex("_CAMERA_RELATIVE_CONTROLLER_LEFTHAND"); + } + orientation = Quat.multiply(MyAvatar.orientation, MyAvatar.getAbsoluteJointRotationInObjectFrame(controllerJointIndex)); + position = Vec3.sum(MyAvatar.position, Vec3.multiplyQbyV(MyAvatar.orientation, MyAvatar.getAbsoluteJointTranslationInObjectFrame(controllerJointIndex))); + // add to the real position so the grab-point is out in front of the hand, a bit if (doOffset) { - position = Vec3.sum(position, Vec3.multiplyQbyV(orientation, getGrabPointSphereOffset(handController))); - } - if (Menu.isOptionChecked("Third Person")) { - position = Vec3.sum(position, Vec3.subtract(Camera.position, MyAvatar.getEyePosition())); + var offset = getGrabPointSphereOffset(handController); + position = Vec3.sum(position, Vec3.multiplyQbyV(orientation, offset)); } + } else if (!HMD.isHandControllerAvailable()) { position = MyAvatar.getHeadPosition(); orientation = Quat.multiply(MyAvatar.headOrientation, Quat.angleAxis(-90, { x: 1, y: 0, z: 0 })); From 270adfe692eea68efc0accba36bd902893420580 Mon Sep 17 00:00:00 2001 From: "Anthony J. Thibault" Date: Thu, 17 Nov 2016 11:40:05 -0800 Subject: [PATCH 4/5] removed debug prints --- scripts/system/controllers/controllerDisplay.js | 4 ---- 1 file changed, 4 deletions(-) diff --git a/scripts/system/controllers/controllerDisplay.js b/scripts/system/controllers/controllerDisplay.js index 6135f18426..48fea400af 100644 --- a/scripts/system/controllers/controllerDisplay.js +++ b/scripts/system/controllers/controllerDisplay.js @@ -48,7 +48,6 @@ createControllerDisplay = function(config) { mappingName: "mapping-display-" + Math.random(), setVisible: function(visible) { - debug("Setting visible", this.overlays.length); for (var i = 0; i < this.overlays.length; ++i) { Overlays.editOverlay(this.overlays[i], { visible: visible @@ -77,9 +76,6 @@ createControllerDisplay = function(config) { textures[part.textureName] = layer.defaultTextureURL; } for (var i = 0; i < this.partOverlays[partName].length; ++i) { - - // AJT: REMOVE - print("AJT: Overlays.editOverlays(" + partName + ", " + i + ", { textures: " + JSON.stringify(textures) + " })"); Overlays.editOverlay(this.partOverlays[partName][i], { textures: textures }); From 7f840d3a08bf874ad86f3fb3ebbdc9fb77ca1e29 Mon Sep 17 00:00:00 2001 From: "Anthony J. Thibault" Date: Wed, 30 Nov 2016 10:01:11 -0800 Subject: [PATCH 5/5] Can use 'xboxLaser' in third person camera --- scripts/system/controllers/handControllerPointer.js | 8 +++++--- scripts/system/libraries/controllers.js | 6 ++++-- 2 files changed, 9 insertions(+), 5 deletions(-) diff --git a/scripts/system/controllers/handControllerPointer.js b/scripts/system/controllers/handControllerPointer.js index 5e8c0cb6ba..0623ddf100 100644 --- a/scripts/system/controllers/handControllerPointer.js +++ b/scripts/system/controllers/handControllerPointer.js @@ -489,9 +489,11 @@ function setColoredLaser() { // answer trigger state if lasers supported, else f var color = (activeTrigger.state === 'full') ? LASER_TRIGGER_COLOR_XYZW : LASER_SEARCH_COLOR_XYZW; if (!HMD.isHandControllerAvailable()) { - var position = MyAvatar.getHeadPosition(); - var direction = Quat.getUp(Quat.multiply(MyAvatar.headOrientation, Quat.angleAxis(-90, { x: 1, y: 0, z: 0 }))); - return HMD.setExtraLaser(position, true, color, direction); + // NOTE: keep this offset in sync with scripts/system/librarires/controllers.js:57 + var VERTICAL_HEAD_LASER_OFFSET = 0.1; + var position = Vec3.sum(HMD.position, Vec3.multiplyQbyV(HMD.orientation, {x: 0, y: VERTICAL_HEAD_LASER_OFFSET, z: 0})); + var orientation = Quat.multiply(HMD.orientation, Quat.angleAxis(-90, { x: 1, y: 0, z: 0 })); + return HMD.setExtraLaser(position, true, color, Quat.getUp(orientation)); } return HMD.setHandLasers(activeHudLaser, true, color, SYSTEM_LASER_DIRECTION) && activeTrigger.state; diff --git a/scripts/system/libraries/controllers.js b/scripts/system/libraries/controllers.js index 2e3a4a1e57..75bca34ed6 100644 --- a/scripts/system/libraries/controllers.js +++ b/scripts/system/libraries/controllers.js @@ -52,8 +52,10 @@ getControllerWorldLocation = function (handController, doOffset) { } } else if (!HMD.isHandControllerAvailable()) { - position = MyAvatar.getHeadPosition(); - orientation = Quat.multiply(MyAvatar.headOrientation, Quat.angleAxis(-90, { x: 1, y: 0, z: 0 })); + // NOTE: keep this offset in sync with scripts/system/controllers/handControllerPointer.js:493 + var VERTICAL_HEAD_LASER_OFFSET = 0.1; + position = Vec3.sum(Camera.position, Vec3.multiplyQbyV(Camera.orientation, {x: 0, y: VERTICAL_HEAD_LASER_OFFSET, z: 0})); + orientation = Quat.multiply(Camera.orientation, Quat.angleAxis(-90, { x: 1, y: 0, z: 0 })); valid = true; }