From 38ac6fff03950c96e28008de7b0a94e22898fb73 Mon Sep 17 00:00:00 2001 From: "Anthony J. Thibault" Date: Thu, 17 Nov 2016 11:33:12 -0800 Subject: [PATCH] Hand Controller rendering is camera relative Basically, when using the third person camera in HMD mode. If the controllers are shown. They should be shown in front of the users camera, not in front of the users avatar. To accomplish this, two new faux joint indices are introduced. CAMERA_RELATIVE_CONTROLLER_RIGHTHAND_INDEX and CAMERA_RELATIVE_CONTROLLER_LEFTHAND_INDEX. These joint indices can be used for Overlay parenting. (But not for entity parenting because they are not transmitted over the network). They can also be queried for by using the MyAvatar.getAbsoluteJointRotationInObjectFrame() call. These new indices are now used by the controllerDisplay.js for the hand controller rendering. They are also used by system/libraries/controllers.js as the origin for hand controller grabbing and interaction lasers. --- interface/src/avatar/MyAvatar.cpp | 51 ++++++++++++++++++- interface/src/avatar/MyAvatar.h | 1 + libraries/avatars/src/AvatarData.cpp | 6 +++ libraries/avatars/src/AvatarData.h | 3 +- .../viveControllerConfiguration.js | 5 +- scripts/system/libraries/controllers.js | 18 ++++--- 6 files changed, 72 insertions(+), 12 deletions(-) diff --git a/interface/src/avatar/MyAvatar.cpp b/interface/src/avatar/MyAvatar.cpp index b49ee94ff6..b8f02fd17e 100644 --- a/interface/src/avatar/MyAvatar.cpp +++ b/interface/src/avatar/MyAvatar.cpp @@ -2333,14 +2333,49 @@ bool MyAvatar::hasDriveInput() const { return fabsf(_driveKeys[TRANSLATE_X]) > 0.0f || fabsf(_driveKeys[TRANSLATE_Y]) > 0.0f || fabsf(_driveKeys[TRANSLATE_Z]) > 0.0f; } +// The resulting matrix is used to render the hand controllers, even if the camera is decoupled from the avatar. +// Specificly, if we are rendering using a third person camera. We would like to render the hand controllers in front of the camera, +// not in front of the avatar. +glm::mat4 MyAvatar::computeCameraRelativeHandControllerMatrix(const glm::mat4& controllerSensorMatrix) const { + + // Fetch the current camera transform. + glm::mat4 cameraWorldMatrix = qApp->getCamera()->getTransform(); + if (qApp->getCamera()->getMode() == CAMERA_MODE_MIRROR) { + cameraWorldMatrix *= createMatFromScaleQuatAndPos(vec3(-1.0f, 1.0f, 1.0f), glm::quat(), glm::vec3()); + } + + // compute a NEW sensorToWorldMatrix for the camera. The equation is cameraWorldMatrix = cameraSensorToWorldMatrix * _hmdSensorMatrix. + // here we solve for the unknown cameraSensorToWorldMatrix. + glm::mat4 cameraSensorToWorldMatrix = cameraWorldMatrix * glm::inverse(_hmdSensorMatrix); + + // Using the new cameraSensorToWorldMatrix, compute where the controller is in world space. + glm::mat4 controllerWorldMatrix = cameraSensorToWorldMatrix * controllerSensorMatrix; + + // move it into avatar space + glm::mat4 avatarMatrix = createMatFromQuatAndPos(getOrientation(), getPosition()); + return glm::inverse(avatarMatrix) * controllerWorldMatrix; +} + glm::quat MyAvatar::getAbsoluteJointRotationInObjectFrame(int index) const { - switch(index) { + switch (index) { case CONTROLLER_LEFTHAND_INDEX: { return getLeftHandControllerPoseInAvatarFrame().getRotation(); } case CONTROLLER_RIGHTHAND_INDEX: { return getRightHandControllerPoseInAvatarFrame().getRotation(); } + case CAMERA_RELATIVE_CONTROLLER_LEFTHAND_INDEX: { + auto pose = _leftHandControllerPoseInSensorFrameCache.get(); + glm::mat4 controllerSensorMatrix = createMatFromQuatAndPos(pose.rotation, pose.translation); + glm::mat4 result = computeCameraRelativeHandControllerMatrix(controllerSensorMatrix); + return glmExtractRotation(result); + } + case CAMERA_RELATIVE_CONTROLLER_RIGHTHAND_INDEX: { + auto pose = _rightHandControllerPoseInSensorFrameCache.get(); + glm::mat4 controllerSensorMatrix = createMatFromQuatAndPos(pose.rotation, pose.translation); + glm::mat4 result = computeCameraRelativeHandControllerMatrix(controllerSensorMatrix); + return glmExtractRotation(result); + } default: { return Avatar::getAbsoluteJointRotationInObjectFrame(index); } @@ -2348,13 +2383,25 @@ glm::quat MyAvatar::getAbsoluteJointRotationInObjectFrame(int index) const { } glm::vec3 MyAvatar::getAbsoluteJointTranslationInObjectFrame(int index) const { - switch(index) { + switch (index) { case CONTROLLER_LEFTHAND_INDEX: { return getLeftHandControllerPoseInAvatarFrame().getTranslation(); } case CONTROLLER_RIGHTHAND_INDEX: { return getRightHandControllerPoseInAvatarFrame().getTranslation(); } + case CAMERA_RELATIVE_CONTROLLER_LEFTHAND_INDEX: { + auto pose = _leftHandControllerPoseInSensorFrameCache.get(); + glm::mat4 controllerSensorMatrix = createMatFromQuatAndPos(pose.rotation, pose.translation); + glm::mat4 result = computeCameraRelativeHandControllerMatrix(controllerSensorMatrix); + return extractTranslation(result); + } + case CAMERA_RELATIVE_CONTROLLER_RIGHTHAND_INDEX: { + auto pose = _rightHandControllerPoseInSensorFrameCache.get(); + glm::mat4 controllerSensorMatrix = createMatFromQuatAndPos(pose.rotation, pose.translation); + glm::mat4 result = computeCameraRelativeHandControllerMatrix(controllerSensorMatrix); + return extractTranslation(result); + } default: { return Avatar::getAbsoluteJointTranslationInObjectFrame(index); } diff --git a/interface/src/avatar/MyAvatar.h b/interface/src/avatar/MyAvatar.h index 60049bea67..f081ec533b 100644 --- a/interface/src/avatar/MyAvatar.h +++ b/interface/src/avatar/MyAvatar.h @@ -374,6 +374,7 @@ private: void clampTargetScaleToDomainLimits(); void clampScaleChangeToDomainLimits(float desiredScale); + glm::mat4 computeCameraRelativeHandControllerMatrix(const glm::mat4& controllerSensorMatrix) const; float _driveKeys[MAX_DRIVE_KEYS]; bool _wasPushing; diff --git a/libraries/avatars/src/AvatarData.cpp b/libraries/avatars/src/AvatarData.cpp index 2469d0df04..cbda35966d 100644 --- a/libraries/avatars/src/AvatarData.cpp +++ b/libraries/avatars/src/AvatarData.cpp @@ -953,6 +953,12 @@ int AvatarData::getFauxJointIndex(const QString& name) const { if (name == "_CONTROLLER_RIGHTHAND") { return CONTROLLER_RIGHTHAND_INDEX; } + if (name == "_CAMERA_RELATIVE_CONTROLLER_LEFTHAND") { + return CAMERA_RELATIVE_CONTROLLER_LEFTHAND_INDEX; + } + if (name == "_CAMERA_RELATIVE_CONTROLLER_RIGHTHAND") { + return CAMERA_RELATIVE_CONTROLLER_RIGHTHAND_INDEX; + } return -1; } diff --git a/libraries/avatars/src/AvatarData.h b/libraries/avatars/src/AvatarData.h index 97879700ee..27d4e336d5 100644 --- a/libraries/avatars/src/AvatarData.h +++ b/libraries/avatars/src/AvatarData.h @@ -539,6 +539,7 @@ void RayToAvatarIntersectionResultFromScriptValue(const QScriptValue& object, Ra const int SENSOR_TO_WORLD_MATRIX_INDEX = 65534; // -2 const int CONTROLLER_RIGHTHAND_INDEX = 65533; // -3 const int CONTROLLER_LEFTHAND_INDEX = 65532; // -4 - +const int CAMERA_RELATIVE_CONTROLLER_RIGHTHAND_INDEX = 65531; // -5 +const int CAMERA_RELATIVE_CONTROLLER_LEFTHAND_INDEX = 65530; // -6 #endif // hifi_AvatarData_h diff --git a/scripts/system/controllers/viveControllerConfiguration.js b/scripts/system/controllers/viveControllerConfiguration.js index b49c3e1d04..4b8fc34ef7 100644 --- a/scripts/system/controllers/viveControllerConfiguration.js +++ b/scripts/system/controllers/viveControllerConfiguration.js @@ -69,7 +69,7 @@ VIVE_CONTROLLER_CONFIGURATION_LEFT = { controllers: [ { modelURL: viveModelURL, - jointIndex: MyAvatar.getJointIndex("_CONTROLLER_LEFTHAND"), + jointIndex: MyAvatar.getJointIndex("_CAMERA_RELATIVE_CONTROLLER_LEFTHAND"), naturalPosition: viveNaturalPosition, rotation: leftBaseRotation, position: Vec3.multiplyQbyV(Quat.fromPitchYawRollDegrees(0, 0, 45), leftBasePosition), @@ -195,8 +195,7 @@ VIVE_CONTROLLER_CONFIGURATION_RIGHT = { controllers: [ { modelURL: viveModelURL, - jointIndex: MyAvatar.getJointIndex("_CONTROLLER_RIGHTHAND"), - + jointIndex: MyAvatar.getJointIndex("_CAMERA_RELATIVE_CONTROLLER_RIGHTHAND"), rotation: rightBaseRotation, position: Vec3.multiplyQbyV(Quat.fromPitchYawRollDegrees(0, 0, -45), rightBasePosition), diff --git a/scripts/system/libraries/controllers.js b/scripts/system/libraries/controllers.js index 51b44ecc5c..2e3a4a1e57 100644 --- a/scripts/system/libraries/controllers.js +++ b/scripts/system/libraries/controllers.js @@ -35,16 +35,22 @@ getControllerWorldLocation = function (handController, doOffset) { var position; var pose = Controller.getPoseValue(handController); var valid = pose.valid; + var controllerJointIndex; if (pose.valid) { - orientation = Quat.multiply(MyAvatar.orientation, pose.rotation); - position = Vec3.sum(Vec3.multiplyQbyV(MyAvatar.orientation, pose.translation), MyAvatar.position); + if (handController === Controller.Standard.RightHand) { + controllerJointIndex = MyAvatar.getJointIndex("_CAMERA_RELATIVE_CONTROLLER_RIGHTHAND"); + } else { + controllerJointIndex = MyAvatar.getJointIndex("_CAMERA_RELATIVE_CONTROLLER_LEFTHAND"); + } + orientation = Quat.multiply(MyAvatar.orientation, MyAvatar.getAbsoluteJointRotationInObjectFrame(controllerJointIndex)); + position = Vec3.sum(MyAvatar.position, Vec3.multiplyQbyV(MyAvatar.orientation, MyAvatar.getAbsoluteJointTranslationInObjectFrame(controllerJointIndex))); + // add to the real position so the grab-point is out in front of the hand, a bit if (doOffset) { - position = Vec3.sum(position, Vec3.multiplyQbyV(orientation, getGrabPointSphereOffset(handController))); - } - if (Menu.isOptionChecked("Third Person")) { - position = Vec3.sum(position, Vec3.subtract(Camera.position, MyAvatar.getEyePosition())); + var offset = getGrabPointSphereOffset(handController); + position = Vec3.sum(position, Vec3.multiplyQbyV(orientation, offset)); } + } else if (!HMD.isHandControllerAvailable()) { position = MyAvatar.getHeadPosition(); orientation = Quat.multiply(MyAvatar.headOrientation, Quat.angleAxis(-90, { x: 1, y: 0, z: 0 }));