From 3f7a2eb10aa8cae9f3b8f9a815d15c3a640e1d76 Mon Sep 17 00:00:00 2001
From: Brad Hefta-Gaub <brad@highfidelity.io>
Date: Mon, 22 Feb 2016 13:54:46 -0800
Subject: [PATCH] add support in depthReticle.js to set depth to overlay depth
 when you're over a 2D overlay

---
 examples/depthReticle.js                 | 35 +++++++++++++-----------
 interface/src/ui/ApplicationCompositor.h |  9 ++++++
 2 files changed, 28 insertions(+), 16 deletions(-)

diff --git a/examples/depthReticle.js b/examples/depthReticle.js
index 4d72836577..4c93f714aa 100644
--- a/examples/depthReticle.js
+++ b/examples/depthReticle.js
@@ -6,24 +6,27 @@ Script.update.connect(function(deltaTime) {
     var timeSinceLastDepthCheck = Date.now() - lastDepthCheckTime;
     if (timeSinceLastDepthCheck > TIME_BETWEEN_DEPTH_CHECKS) {
         var reticlePosition = Reticle.position;
-        var pickRay = Camera.computePickRay(reticlePosition.x, reticlePosition.y);
-        var result = Overlays.findRayIntersection(pickRay);
-
-        if (!result.intersects) {
-            result = Entities.findRayIntersection(pickRay, true);
-        }
-        if (result.intersects) {
-            // + JSON.stringify(result)
-            print("something hovered!! result.distance:" +result.distance);
-            Vec3.print("something hovered!! result.intersection:", result.intersection);
-            Reticle.setDepth(result.distance);
 
+        // first check the 2D Overlays
+        if (Reticle.pointingAtSystemOverlay || Overlays.getOverlayAtPoint(reticlePosition)) {
+            print("intersecting with 2D overlay...");
+            Reticle.setDepth(1.0);
         } else {
-            Reticle.setDepth(100.0);
-            print("NO INTERSECTION...");
-            var pointAt2D = Reticle.position;
-            var pointAt3D = HMD.worldPointFromOverlay(pointAt2D);
-        }
+            var pickRay = Camera.computePickRay(reticlePosition.x, reticlePosition.y);
+            var result = Overlays.findRayIntersection(pickRay);
 
+            if (!result.intersects) {
+                result = Entities.findRayIntersection(pickRay, true);
+            }
+            if (result.intersects) {
+                // + JSON.stringify(result)
+                print("something hovered!! result.distance:" +result.distance);
+                Vec3.print("something hovered!! result.intersection:", result.intersection);
+                Reticle.setDepth(result.distance);
+            } else {
+                Reticle.setDepth(100.0);
+                print("NO INTERSECTION...");
+            }
+        }
     }
 });
diff --git a/interface/src/ui/ApplicationCompositor.h b/interface/src/ui/ApplicationCompositor.h
index 7337598d8f..3ee7c53f1b 100644
--- a/interface/src/ui/ApplicationCompositor.h
+++ b/interface/src/ui/ApplicationCompositor.h
@@ -105,6 +105,9 @@ public:
 
     bool shouldCaptureMouse() const;
 
+    /// if the reticle is pointing to a system overlay (a dialog box for example) then the function returns true otherwise false
+    bool isReticlePointingAtSystemOverlay() const { return _reticleOverQml; }
+
 private:
 
     void displayOverlayTextureStereo(RenderArgs* renderArgs, float aspectRatio, float fov);
@@ -154,6 +157,8 @@ private:
     QPointF _lastKnownRealMouse;
     bool _ignoreMouseMove { false };
 
+    bool _reticleOverQml { false };
+
     ReticleInterface* _reticleInterface;
 };
 
@@ -165,6 +170,7 @@ class ReticleInterface : public QObject {
     Q_PROPERTY(float depth READ getDepth WRITE setDepth)
     Q_PROPERTY(glm::vec2 maximumPosition READ getMaximumPosition)
     Q_PROPERTY(bool mouseCaptured READ isMouseCaptured)
+    Q_PROPERTY(bool pointingAtSystemOverlay READ isPointingAtSystemOverlay)
 
 public:
     ReticleInterface(ApplicationCompositor* outer) : QObject(outer), _compositor(outer) {}
@@ -182,6 +188,9 @@ public:
 
     Q_INVOKABLE glm::vec2 getMaximumPosition() { return _compositor->getReticleMaximumPosition(); }
 
+    Q_INVOKABLE bool isPointingAtSystemOverlay() { return _compositor->isReticlePointingAtSystemOverlay(); }
+
+
 private:
     ApplicationCompositor* _compositor;
 };