From 1160ecb335cc53143af20ea2a16fb905d9ad5d29 Mon Sep 17 00:00:00 2001 From: Brad Hefta-Gaub Date: Sun, 31 Jan 2016 22:16:30 -0800 Subject: [PATCH] improvements in HMD UI behavior --- .../controllers/getHUDLookAtPositionTest.js | 16 +++-- examples/controllers/handControllerGrab.js | 7 ++ .../controllers/reticleHandRotationTest.js | 4 +- .../src/scripting/HMDScriptingInterface.cpp | 34 ++++++++-- .../src/scripting/HMDScriptingInterface.h | 10 +++ interface/src/ui/ApplicationCompositor.cpp | 65 ++++++++++++++----- interface/src/ui/ApplicationCompositor.h | 2 + .../src/controllers/ScriptingInterface.h | 17 +++-- 8 files changed, 119 insertions(+), 36 deletions(-) diff --git a/examples/controllers/getHUDLookAtPositionTest.js b/examples/controllers/getHUDLookAtPositionTest.js index 348b6757b8..cad9c944bf 100644 --- a/examples/controllers/getHUDLookAtPositionTest.js +++ b/examples/controllers/getHUDLookAtPositionTest.js @@ -19,18 +19,20 @@ var cubeSize = 0.03; var cube = Overlays.addOverlay("cube", { position: cubePosition, size: cubeSize, - color: { red: 255, green: 0, blue: 0}, + color: { red: 0, green: 255, blue: 0}, alpha: 1, solid: false }); -var square = Overlays.addOverlay("text", { +var SQUARE_SIZE = 20; + +var square = Overlays.addOverlay("rectangle", { x: 0, y: 0, - width: 20, - height: 20, - color: { red: 255, green: 255, blue: 0}, - backgroundColor: { red: 255, green: 255, blue: 0}, + width: SQUARE_SIZE, + height: SQUARE_SIZE, + color: { red: 0, green: 255, blue: 0}, + backgroundColor: { red: 0, green: 255, blue: 0}, alpha: 1 }); @@ -43,7 +45,7 @@ Script.update.connect(function(deltaTime) { Overlays.editOverlay(cube, { position: lookAt3D }); var lookAt2D = HMD.getHUDLookAtPosition2D(); - Overlays.editOverlay(square, { x: lookAt2D.x, y: lookAt2D.y }); + Overlays.editOverlay(square, { x: lookAt2D.x - (SQUARE_SIZE/2), y: lookAt2D.y - (SQUARE_SIZE/2)}); }); Script.scriptEnding.connect(function(){ diff --git a/examples/controllers/handControllerGrab.js b/examples/controllers/handControllerGrab.js index 908337f643..e213834385 100644 --- a/examples/controllers/handControllerGrab.js +++ b/examples/controllers/handControllerGrab.js @@ -14,6 +14,7 @@ Script.include("../libraries/utils.js"); + // // add lines where the hand ray picking is happening // @@ -722,6 +723,9 @@ function MyController(hand) { this.particleBeamOff(); } this.searchSphereOff(); + + Controller.setReticleVisible(true); + }; this.triggerPress = function(value) { @@ -1018,6 +1022,9 @@ function MyController(hand) { this.overlayLineOn(handPosition, searchSphereLocation, (this.triggerSmoothedGrab() || this.bumperSqueezed()) ? INTERSECT_COLOR : NO_INTERSECT_COLOR); } + + Controller.setReticleVisible(false); + }; this.distanceGrabTimescale = function(mass, distance) { diff --git a/examples/controllers/reticleHandRotationTest.js b/examples/controllers/reticleHandRotationTest.js index a303e5e7b4..a27e6f2c4e 100644 --- a/examples/controllers/reticleHandRotationTest.js +++ b/examples/controllers/reticleHandRotationTest.js @@ -31,8 +31,8 @@ function moveReticleAbsolute(x, y) { var MAPPING_NAME = "com.highfidelity.testing.reticleWithHandRotation"; var mapping = Controller.newMapping(MAPPING_NAME); -mapping.from(Controller.Standard.LT).peek().constrainToInteger().to(Controller.Actions.ReticleClick); -mapping.from(Controller.Standard.RT).peek().constrainToInteger().to(Controller.Actions.ReticleClick); +mapping.from(Controller.Hardware.Hydra.L3).peek().to(Controller.Actions.ReticleClick); +mapping.from(Controller.Hardware.Hydra.R4).peek().to(Controller.Actions.ReticleClick); mapping.enable(); diff --git a/interface/src/scripting/HMDScriptingInterface.cpp b/interface/src/scripting/HMDScriptingInterface.cpp index ab53e6912a..d1572a0aa9 100644 --- a/interface/src/scripting/HMDScriptingInterface.cpp +++ b/interface/src/scripting/HMDScriptingInterface.cpp @@ -20,15 +20,39 @@ HMDScriptingInterface::HMDScriptingInterface() { } +glm::vec3 HMDScriptingInterface::calculateRayUICollisionPoint(const glm::vec3& position, const glm::vec3& direction) const { + glm::vec3 result { position }; + qApp->getApplicationCompositor().calculateRayUICollisionPoint(position, direction, result); + return result; +} + +glm::vec2 HMDScriptingInterface::overlayFromWorldPoint(const glm::vec3& position) const { + return qApp->getApplicationCompositor().overlayFromSphereSurface(position); +} + +glm::vec2 HMDScriptingInterface::sphericalToOverlay(const glm::vec2 & position) const { + return qApp->getApplicationCompositor().sphericalToOverlay(position); +} + +glm::vec2 HMDScriptingInterface::overlayToSpherical(const glm::vec2 & position) const { + return qApp->getApplicationCompositor().overlayToSpherical(position); +} + +glm::vec2 HMDScriptingInterface::screenToOverlay(const glm::vec2 & position) const { + return qApp->getApplicationCompositor().screenToOverlay(position); +} + +glm::vec2 HMDScriptingInterface::overlayToScreen(const glm::vec2 & position) const { + return qApp->getApplicationCompositor().overlayToScreen(position); +} + + + QScriptValue HMDScriptingInterface::getHUDLookAtPosition2D(QScriptContext* context, QScriptEngine* engine) { glm::vec3 hudIntersection; auto instance = DependencyManager::get(); if (instance->getHUDLookAtPosition3D(hudIntersection)) { - MyAvatar* myAvatar = DependencyManager::get()->getMyAvatar(); - glm::vec3 sphereCenter = myAvatar->getDefaultEyePosition(); - glm::vec3 direction = glm::inverse(myAvatar->getOrientation()) * (hudIntersection - sphereCenter); - glm::vec2 polar = glm::vec2(glm::atan(direction.x, -direction.z), glm::asin(direction.y)) * -1.0f; - auto overlayPos = qApp->getApplicationCompositor().sphericalToOverlay(polar); + glm::vec2 overlayPos = qApp->getApplicationCompositor().overlayFromSphereSurface(hudIntersection); return qScriptValueFromValue(engine, overlayPos); } return QScriptValue::NullValue; diff --git a/interface/src/scripting/HMDScriptingInterface.h b/interface/src/scripting/HMDScriptingInterface.h index f07350605f..1245c06575 100644 --- a/interface/src/scripting/HMDScriptingInterface.h +++ b/interface/src/scripting/HMDScriptingInterface.h @@ -25,6 +25,16 @@ class HMDScriptingInterface : public AbstractHMDScriptingInterface, public Depen Q_OBJECT Q_PROPERTY(glm::vec3 position READ getPosition) Q_PROPERTY(glm::quat orientation READ getOrientation) + +public: + Q_INVOKABLE glm::vec3 calculateRayUICollisionPoint(const glm::vec3& position, const glm::vec3& direction) const; + Q_INVOKABLE glm::vec2 overlayFromWorldPoint(const glm::vec3& position) const; + + Q_INVOKABLE glm::vec2 sphericalToOverlay(const glm::vec2 & sphericalPos) const; + Q_INVOKABLE glm::vec2 overlayToSpherical(const glm::vec2 & overlayPos) const; + Q_INVOKABLE glm::vec2 screenToOverlay(const glm::vec2 & screenPos) const; + Q_INVOKABLE glm::vec2 overlayToScreen(const glm::vec2 & overlayPos) const; + public: HMDScriptingInterface(); static QScriptValue getHUDLookAtPosition2D(QScriptContext* context, QScriptEngine* engine); diff --git a/interface/src/ui/ApplicationCompositor.cpp b/interface/src/ui/ApplicationCompositor.cpp index 5f14d446ff..736b1fffce 100644 --- a/interface/src/ui/ApplicationCompositor.cpp +++ b/interface/src/ui/ApplicationCompositor.cpp @@ -262,7 +262,6 @@ void ApplicationCompositor::displayOverlayTextureHmd(RenderArgs* renderArgs, int camMat = (headPose * eyeToHead) * camMat; batch.setViewportTransform(renderArgs->_viewport); batch.setViewTransform(camMat); - batch.setProjectionTransform(qApp->getEyeProjection(eye)); #ifdef DEBUG_OVERLAY @@ -282,21 +281,28 @@ void ApplicationCompositor::displayOverlayTextureHmd(RenderArgs* renderArgs, int bindCursorTexture(batch); - //Controller Pointers - glm::mat4 overlayXfm; - _modelTransform.getMatrix(overlayXfm); - //Mouse Pointer - glm::vec2 projection = screenToSpherical(qApp->getTrueMouse()); - mat4 pointerXfm = glm::mat4_cast(quat(vec3(-projection.y, projection.x, 0.0f))) * glm::translate(mat4(), vec3(0, 0, -1)); - mat4 reticleXfm = overlayXfm * pointerXfm; - reticleXfm = glm::scale(reticleXfm, reticleScale); - batch.setModelTransform(reticleXfm); - geometryCache->renderUnitQuad(batch, glm::vec4(1), _reticleQuad); + auto controllerScriptingInterface = DependencyManager::get(); + bool reticleVisible = controllerScriptingInterface->getReticleVisible(); + if (reticleVisible) { + glm::mat4 overlayXfm; + _modelTransform.getMatrix(overlayXfm); + + glm::vec2 projection = screenToSpherical(qApp->getTrueMouse()); + + float cursorDepth = controllerScriptingInterface->getReticleDepth(); + mat4 pointerXfm = glm::scale(mat4(), vec3(cursorDepth)) * glm::mat4_cast(quat(vec3(-projection.y, projection.x, 0.0f))) * glm::translate(mat4(), vec3(0, 0, -1)); + mat4 reticleXfm = overlayXfm * pointerXfm; + reticleXfm = glm::scale(reticleXfm, reticleScale); + batch.setModelTransform(reticleXfm); + geometryCache->renderUnitQuad(batch, glm::vec4(1), _reticleQuad); + } }); } +// FIXME - this probably is hella buggy and probably doesn't work correctly +// we should kill it asap. void ApplicationCompositor::computeHmdPickRay(glm::vec2 cursorPos, glm::vec3& origin, glm::vec3& direction) const { const glm::vec2 projection = overlayToSpherical(cursorPos); // The overlay space orientation of the mouse coordinates @@ -326,16 +332,22 @@ void ApplicationCompositor::computeHmdPickRay(glm::vec2 cursorPos, glm::vec3& or //Finds the collision point of a world space ray bool ApplicationCompositor::calculateRayUICollisionPoint(const glm::vec3& position, const glm::vec3& direction, glm::vec3& result) const { - MyAvatar* myAvatar = DependencyManager::get()->getMyAvatar(); - glm::quat inverseOrientation = glm::inverse(myAvatar->getOrientation()); + auto displayPlugin = qApp->getActiveDisplayPlugin(); + auto headPose = displayPlugin->getHeadPose(qApp->getFrameCount()); - glm::vec3 relativePosition = inverseOrientation * (position - myAvatar->getDefaultEyePosition()); - glm::vec3 relativeDirection = glm::normalize(inverseOrientation * direction); + auto myCamera = qApp->getCamera(); + mat4 cameraMat = myCamera->getTransform(); + auto UITransform = cameraMat * glm::inverse(headPose); + auto relativePosition4 = glm::inverse(UITransform) * vec4(position, 1); + auto relativePosition = vec3(relativePosition4) / relativePosition4.w; + auto relativeDirection = glm::inverse(glm::quat_cast(UITransform)) * direction; - float t; - if (raySphereIntersect(relativeDirection, relativePosition, _oculusUIRadius * myAvatar->getUniformScale(), &t)){ - result = position + direction * t; + float uiRadius = _oculusUIRadius; // * myAvatar->getUniformScale(); // FIXME - how do we want to handle avatar scale + + float instersectionDistance; + if (raySphereIntersect(relativeDirection, relativePosition, uiRadius, &instersectionDistance)){ + result = position + glm::normalize(direction) * instersectionDistance; return true; } @@ -494,6 +506,23 @@ glm::vec2 ApplicationCompositor::overlayToScreen(const glm::vec2& overlayPos) co return sphericalToScreen(overlayToSpherical(overlayPos)); } +glm::vec2 ApplicationCompositor::overlayFromSphereSurface(const glm::vec3& sphereSurfacePoint) const { + + auto displayPlugin = qApp->getActiveDisplayPlugin(); + auto headPose = displayPlugin->getHeadPose(qApp->getFrameCount()); + auto myCamera = qApp->getCamera(); + mat4 cameraMat = myCamera->getTransform(); + auto UITransform = cameraMat * glm::inverse(headPose); + auto relativePosition4 = glm::inverse(UITransform) * vec4(sphereSurfacePoint, 1); + auto relativePosition = vec3(relativePosition4) / relativePosition4.w; + auto center = vec3(0); // center of HUD in HUD space + auto direction = relativePosition - center; // direction to relative position in HUD space + + glm::vec2 polar = glm::vec2(glm::atan(direction.x, -direction.z), glm::asin(direction.y)) * -1.0f; + auto overlayPos = sphericalToOverlay(polar); + return overlayPos; +} + void ApplicationCompositor::updateTooltips() { if (_hoverItemId != _noItemId) { quint64 hoverDuration = usecTimestampNow() - _hoverItemEnterUsecs; diff --git a/interface/src/ui/ApplicationCompositor.h b/interface/src/ui/ApplicationCompositor.h index 8476782550..ae45352e84 100644 --- a/interface/src/ui/ApplicationCompositor.h +++ b/interface/src/ui/ApplicationCompositor.h @@ -62,6 +62,8 @@ public: void computeHmdPickRay(glm::vec2 cursorPos, glm::vec3& origin, glm::vec3& direction) const; uint32_t getOverlayTexture() const; + glm::vec2 overlayFromSphereSurface(const glm::vec3& sphereSurfacePoint) const; + void setCameraBaseTransform(const Transform& transform) { _cameraBaseTransform = transform; } const Transform& getCameraBaseTransform() const { return _cameraBaseTransform; } diff --git a/libraries/controllers/src/controllers/ScriptingInterface.h b/libraries/controllers/src/controllers/ScriptingInterface.h index 52ec52e852..72b6ca78e8 100644 --- a/libraries/controllers/src/controllers/ScriptingInterface.h +++ b/libraries/controllers/src/controllers/ScriptingInterface.h @@ -89,6 +89,12 @@ namespace controller { Q_INVOKABLE QObject* parseMapping(const QString& json); Q_INVOKABLE QObject* loadMapping(const QString& jsonUrl); + Q_INVOKABLE bool getReticleVisible() { return _reticleVisible; } + Q_INVOKABLE void setReticleVisible(bool visible) { _reticleVisible = visible; } + + Q_INVOKABLE float getReticleDepth() { return _reticleDepth; } + Q_INVOKABLE void setReticleDepth(float depth) { _reticleDepth = depth; } + Q_INVOKABLE glm::vec2 getReticlePosition() { return toGlm(QCursor::pos()); } @@ -159,10 +165,13 @@ namespace controller { QVariantMap _actions; QVariantMap _standard; - bool _mouseCaptured{ false }; - bool _touchCaptured{ false }; - bool _wheelCaptured{ false }; - bool _actionsCaptured{ false }; + bool _mouseCaptured { false }; + bool _touchCaptured { false }; + bool _wheelCaptured { false }; + bool _actionsCaptured { false }; + bool _reticleVisible { true }; + + float _reticleDepth { 1.0f }; };