Merge pull request #7156 from ZappoMan/depthReticleWork

Depth reticle work

examples/depthReticle.js

@@ -0,0 +1,47 @@
+//  depthReticle.js
+//  examples
+//
+//  Created by Brad Hefta-Gaub on 2/23/16.
+//  Copyright 2016 High Fidelity, Inc.
+//
+//  When used in HMD, this script will make the reticle depth track to any clickable item in view.
+//
+//  Distributed under the Apache License, Version 2.0.
+//  See the accompanying file LICENSE or http://www.apache.org/licenses/LICENSE-2.0.html
+//
+
+var APPARENT_2D_OVERLAY_DEPTH = 1.0;
+var APPARENT_MAXIMUM_DEPTH = 100.0; // this is a depth at which things all seem sufficiently distant
+var lastDepthCheckTime = 0;
+
+Script.update.connect(function(deltaTime) {
+    var TIME_BETWEEN_DEPTH_CHECKS = 100;
+    var timeSinceLastDepthCheck = Date.now() - lastDepthCheckTime;
+    if (timeSinceLastDepthCheck > TIME_BETWEEN_DEPTH_CHECKS) {
+        var reticlePosition = Reticle.position;
+
+        // first check the 2D Overlays
+        if (Reticle.pointingAtSystemOverlay || Overlays.getOverlayAtPoint(reticlePosition)) {
+            Reticle.setDepth(APPARENT_2D_OVERLAY_DEPTH);
+        } else {
+            var pickRay = Camera.computePickRay(reticlePosition.x, reticlePosition.y);
+
+            // Then check the 3D overlays
+            var result = Overlays.findRayIntersection(pickRay);
+
+            if (!result.intersects) {
+                // finally check the entities
+                result = Entities.findRayIntersection(pickRay, true);
+            }
+
+            // If either the overlays or entities intersect, then set the reticle depth to 
+            // the distance of intersection
+            if (result.intersects) {
+                Reticle.setDepth(result.distance);
+            } else {
+                // if nothing intersects... set the depth to some sufficiently large depth
+                Reticle.setDepth(APPARENT_MAXIMUM_DEPTH);
+            }
+        }
+    }
+});

interface/resources/qml/Stats.qml

@@ -5,6 +5,8 @@ import QtQuick.Controls 1.2
 Item {
     anchors.fill: parent
     anchors.leftMargin: 300
+    objectName: "StatsItem"
+
     Hifi.Stats {
         id: root
         objectName: "Stats"
@@ -27,6 +29,7 @@ Item {
                 MouseArea {
                     anchors.fill: parent
                     onClicked: { root.expanded = !root.expanded; }
+                    hoverEnabled: true
                 }
 
                 Column {
@@ -83,6 +86,7 @@ Item {
                 MouseArea {
                     anchors.fill: parent
                     onClicked: { root.expanded = !root.expanded; }
+                    hoverEnabled: true
                 }
                 Column {
                     id: pingCol
@@ -123,6 +127,7 @@ Item {
                 MouseArea {
                     anchors.fill: parent
                     onClicked: { root.expanded = !root.expanded; }
+                    hoverEnabled: true
                 }
                 Column {
                     id: geoCol
@@ -172,6 +177,7 @@ Item {
                 MouseArea {
                     anchors.fill: parent
                     onClicked: { root.expanded = !root.expanded; }
+                    hoverEnabled: true
                 }
                 Column {
                     id: octreeCol

interface/resources/qml/desktop/Desktop.qml

@@ -313,6 +313,7 @@ FocusScope {
 
     Rectangle {
         id: focusDebugger;
+        objectName: "focusDebugger"
         z: 9999; visible: false; color: "red"
         ColorAnimation on color { from: "#7fffff00"; to: "#7f0000ff"; duration: 1000; loops: 9999 }
     }

interface/resources/qml/desktop/FocusHack.qml

@@ -2,6 +2,7 @@ import QtQuick 2.5
 
 FocusScope {
     id: root
+    objectName: "FocusHack"
 
     TextInput {
         id: textInput;

interface/resources/qml/hifi/Desktop.qml

@@ -8,6 +8,16 @@ import ".."
 Desktop {
     id: desktop
 
+    MouseArea {
+        anchors.fill: parent
+        hoverEnabled: true
+        propagateComposedEvents: true
+        scrollGestureEnabled: false // we don't need/want these
+        onEntered: ApplicationCompositor.reticleOverDesktop = true
+        onExited: ApplicationCompositor.reticleOverDesktop = false
+        acceptedButtons: Qt.NoButton
+    }
+
     Component.onCompleted: {
         WebEngine.settings.javascriptCanOpenWindows = true;
         WebEngine.settings.javascriptCanAccessClipboard = false;

interface/resources/qml/menus/MenuMouseHandler.qml

@@ -6,9 +6,11 @@ import "."
 Item {
     id: root
     anchors.fill: parent
+    objectName: "MouseMenuHandlerItem"
 
     MouseArea {
         id: menuRoot;
+        objectName: "MouseMenuHandlerMouseArea"
         anchors.fill: parent
         enabled: d.topMenu !== null
         onClicked: {

interface/src/Application.cpp

@@ -1260,6 +1260,8 @@ void Application::initializeUi() {
     rootContext->setContextProperty("Render", _renderEngine->getConfiguration().get());
     rootContext->setContextProperty("Reticle", _compositor.getReticleInterface());
 
+    rootContext->setContextProperty("ApplicationCompositor", &_compositor);
+
     _glWidget->installEventFilter(offscreenUi.data());
     offscreenUi->setMouseTranslator([=](const QPointF& pt) {
         QPointF result = pt;
@@ -5120,4 +5122,4 @@ void Application::showDesktop() {
     if (!_overlayConductor.getEnabled()) {
         _overlayConductor.setEnabled(true);
     }
-}
+}

interface/src/ui/ApplicationCompositor.cpp

@@ -283,16 +283,43 @@ void ApplicationCompositor::displayOverlayTextureHmd(RenderArgs* renderArgs, int
 
         //Mouse Pointer
         if (getReticleVisible()) {
-            glm::mat4 overlayXfm;
+            if (getReticleDepth() != 1.0f) {
-            _modelTransform.getMatrix(overlayXfm);
+                // calculate the "apparent location" based on the depth and the current ray
+                glm::vec3 origin, direction;
+                auto reticlePosition = getReticlePosition();
+                computeHmdPickRay(reticlePosition, origin, direction);
+                auto apparentPosition = origin + (direction * getReticleDepth());
 
-            auto reticlePosition = getReticlePosition();
+                // same code as used to render for apparent location
-            glm::vec2 projection = overlayToSpherical(reticlePosition);
+                auto myCamera = qApp->getCamera();
-            float cursorDepth = getReticleDepth();
+                mat4 cameraMat = myCamera->getTransform();
-            mat4 pointerXfm = glm::scale(mat4(), vec3(cursorDepth)) * glm::mat4_cast(quat(vec3(-projection.y, projection.x, 0.0f))) * glm::translate(mat4(), vec3(0, 0, -1));
+                auto UITransform = cameraMat * glm::inverse(headPose);
-            mat4 reticleXfm = overlayXfm * pointerXfm;
+                auto relativePosition4 = glm::inverse(UITransform) * vec4(apparentPosition, 1);
-            reticleXfm = glm::scale(reticleXfm, reticleScale);
+                auto relativePosition = vec3(relativePosition4) / relativePosition4.w;
-            batch.setModelTransform(reticleXfm);
+                auto relativeDistance = glm::length(relativePosition);
+
+                // look at borrowed from overlays
+                float elevation = -asinf(relativePosition.y / glm::length(relativePosition));
+                float azimuth = atan2f(relativePosition.x, relativePosition.z);
+                glm::quat faceCamera = glm::quat(glm::vec3(elevation, azimuth, 0)) * quat(vec3(0, 0, -1)); // this extra *quat(vec3(0,0,-1)) was required to get the quad to flip this seems like we could optimize
+
+                Transform transform;
+                transform.setTranslation(relativePosition);
+                transform.setScale(reticleScale);
+                transform.postScale(relativeDistance); // scale not quite working, distant things too large
+                transform.setRotation(faceCamera);
+                batch.setModelTransform(transform);
+            } else {
+                glm::mat4 overlayXfm;
+                _modelTransform.getMatrix(overlayXfm);
+
+                auto reticlePosition = getReticlePosition();
+                glm::vec2 projection = overlayToSpherical(reticlePosition);
+                mat4 pointerXfm = glm::mat4_cast(quat(vec3(-projection.y, projection.x, 0.0f))) * glm::translate(mat4(), vec3(0, 0, -1));
+                mat4 reticleXfm = overlayXfm * pointerXfm;
+                reticleXfm = glm::scale(reticleXfm, reticleScale);
+                batch.setModelTransform(reticleXfm);
+            }
             geometryCache->renderUnitQuad(batch, glm::vec4(1), _reticleQuad);
         }
     });
@@ -300,7 +327,7 @@ void ApplicationCompositor::displayOverlayTextureHmd(RenderArgs* renderArgs, int
 
 QPointF ApplicationCompositor::getMouseEventPosition(QMouseEvent* event) {
     if (qApp->isHMDMode()) {
-        QMutexLocker locker(&_reticlePositionInHMDLock);
+        QMutexLocker locker(&_reticleLock);
         return QPointF(_reticlePositionInHMD.x, _reticlePositionInHMD.y);
     }
     return event->localPos();
@@ -354,7 +381,7 @@ bool ApplicationCompositor::handleRealMouseMoveEvent(bool sendFakeEvent) {
 
     // If we're in HMD mode
     if (shouldCaptureMouse()) {
-        QMutexLocker locker(&_reticlePositionInHMDLock);
+        QMutexLocker locker(&_reticleLock);
         auto newPosition = QCursor::pos();
         auto changeInRealMouse = newPosition - _lastKnownRealMouse;
         auto newReticlePosition = _reticlePositionInHMD + toGlm(changeInRealMouse);
@@ -368,9 +395,9 @@ bool ApplicationCompositor::handleRealMouseMoveEvent(bool sendFakeEvent) {
     return false; // let the caller know to process the event
 }
 
-glm::vec2 ApplicationCompositor::getReticlePosition() {
+glm::vec2 ApplicationCompositor::getReticlePosition() const {
     if (qApp->isHMDMode()) {
-        QMutexLocker locker(&_reticlePositionInHMDLock);
+        QMutexLocker locker(&_reticleLock);
         return _reticlePositionInHMD;
     }
     return toGlm(QCursor::pos());
@@ -378,7 +405,7 @@ glm::vec2 ApplicationCompositor::getReticlePosition() {
 
 void ApplicationCompositor::setReticlePosition(glm::vec2 position, bool sendFakeEvent) {
     if (qApp->isHMDMode()) {
-        QMutexLocker locker(&_reticlePositionInHMDLock);
+        QMutexLocker locker(&_reticleLock);
         const float MOUSE_EXTENTS_VERT_ANGULAR_SIZE = 170.0f; // 5deg from poles
         const float MOUSE_EXTENTS_VERT_PIXELS = VIRTUAL_SCREEN_SIZE_Y * (MOUSE_EXTENTS_VERT_ANGULAR_SIZE / DEFAULT_HMD_UI_VERT_ANGULAR_SIZE);
         const float MOUSE_EXTENTS_HORZ_ANGULAR_SIZE = 360.0f; // full sphere

interface/src/ui/ApplicationCompositor.h

@@ -29,6 +29,7 @@ class RenderArgs;
 class ReticleInterface;
 
 
+const float DEFAULT_RETICLE_DEPTH = 1.0f; // FIXME - probably should be based on UI radius
 
 const float MAGNIFY_WIDTH = 220.0f;
 const float MAGNIFY_HEIGHT = 100.0f;
@@ -46,6 +47,7 @@ class ApplicationCompositor : public QObject {
     Q_OBJECT
 
     Q_PROPERTY(float alpha READ getAlpha WRITE setAlpha)
+    Q_PROPERTY(bool reticleOverDesktop READ getReticleOverDesktop WRITE setReticleOverDesktop)
 public:
     ApplicationCompositor();
     ~ApplicationCompositor();
@@ -83,13 +85,14 @@ public:
     float getAlpha() const { return _alpha; }
     void setAlpha(float alpha) { _alpha = alpha; }
 
-    bool getReticleVisible() { return _reticleVisible; }
+    bool getReticleVisible() const { return _reticleVisible; }
     void setReticleVisible(bool visible) { _reticleVisible = visible; }
 
-    float getReticleDepth() { return _reticleDepth; }
+    float getReticleDepth() const { return _reticleDepth; }
     void setReticleDepth(float depth) { _reticleDepth = depth; }
+    void resetReticleDepth() { _reticleDepth = DEFAULT_RETICLE_DEPTH; }
 
-    glm::vec2 getReticlePosition();
+    glm::vec2 getReticlePosition() const;
     void setReticlePosition(glm::vec2 position, bool sendFakeEvent = true);
 
     glm::vec2 getReticleMaximumPosition() const;
@@ -103,7 +106,12 @@ public:
 
     bool shouldCaptureMouse() const;
 
+    /// if the reticle is pointing to a system overlay (a dialog box for example) then the function returns true otherwise false
+    bool getReticleOverDesktop() const { return _isOverDesktop; }
+    void setReticleOverDesktop(bool value) { _isOverDesktop = value; }
+
 private:
+    bool _isOverDesktop { true };
 
     void displayOverlayTextureStereo(RenderArgs* renderArgs, float aspectRatio, float fov);
     void bindCursorTexture(gpu::Batch& batch, uint8_t cursorId = 0);
@@ -147,11 +155,13 @@ private:
     // application specific position, when it's in desktop mode, the reticle position will simply move
     // the system mouse.
     glm::vec2 _reticlePositionInHMD { 0.0f, 0.0f };
-    mutable QMutex _reticlePositionInHMDLock{ QMutex::Recursive };
+    mutable QMutex _reticleLock { QMutex::Recursive };
 
     QPointF _lastKnownRealMouse;
     bool _ignoreMouseMove { false };
 
+    bool _reticleOverQml { false };
+
     ReticleInterface* _reticleInterface;
 };
 
@@ -163,11 +173,13 @@ class ReticleInterface : public QObject {
     Q_PROPERTY(float depth READ getDepth WRITE setDepth)
     Q_PROPERTY(glm::vec2 maximumPosition READ getMaximumPosition)
     Q_PROPERTY(bool mouseCaptured READ isMouseCaptured)
+    Q_PROPERTY(bool pointingAtSystemOverlay READ isPointingAtSystemOverlay)
 
 public:
     ReticleInterface(ApplicationCompositor* outer) : QObject(outer), _compositor(outer) {}
 
     Q_INVOKABLE bool isMouseCaptured() { return _compositor->shouldCaptureMouse(); }
+    Q_INVOKABLE bool isPointingAtSystemOverlay() { return !_compositor->getReticleOverDesktop(); }
 
     Q_INVOKABLE bool getVisible() { return _compositor->getReticleVisible(); }
     Q_INVOKABLE void setVisible(bool visible) { _compositor->setReticleVisible(visible); }
@@ -179,6 +191,7 @@ public:
     Q_INVOKABLE void setPosition(glm::vec2 position) { _compositor->setReticlePosition(position); }
 
     Q_INVOKABLE glm::vec2 getMaximumPosition() { return _compositor->getReticleMaximumPosition(); }
+
 private:
     ApplicationCompositor* _compositor;
 };