First pass at controller interaction with 3D Web overlay

interface/src/Application.cpp

@@ -3981,6 +3981,10 @@ void Application::setKeyboardFocusEntity(EntityItemID entityItemID) {
     }
 }
 
+unsigned int Application::getKeyboardFocusOverlay() {
+    return _keyboardFocusedOverlay.get();
+}
+
 void Application::setKeyboardFocusOverlay(unsigned int overlayID) {
     if (overlayID != _keyboardFocusedOverlay.get()) {
         _keyboardFocusedOverlay.set(overlayID);

interface/src/Application.h

@@ -374,6 +374,7 @@ public slots:
     void setKeyboardFocusEntity(QUuid id);
     void setKeyboardFocusEntity(EntityItemID entityItemID);
 
+    unsigned int getKeyboardFocusOverlay();
     void setKeyboardFocusOverlay(unsigned int overlayID);
 
 private slots:

interface/src/ui/overlays/Overlays.cpp

@@ -604,6 +604,38 @@ bool Overlays::isAddedOverlay(unsigned int id) {
     return _overlaysHUD.contains(id) || _overlaysWorld.contains(id);
 }
 
+void Overlays::sendMousePressOnOverlay(unsigned int overlayID, const PointerEvent& event) {
+    emit mousePressOnOverlay(overlayID, event);
+}
+
+void Overlays::sendMouseReleaseOnOverlay(unsigned int overlayID, const PointerEvent& event) {
+    emit mouseReleaseOnOverlay(overlayID, event);
+}
+
+void Overlays::sendMouseMoveOnOverlay(unsigned int overlayID, const PointerEvent& event) {
+    emit mouseMoveOnOverlay(overlayID, event);
+}
+
+void Overlays::sendHoverEnterOverlay(unsigned int id, PointerEvent event) {
+    emit hoverEnterOverlay(id, event);
+}
+
+void Overlays::sendHoverOverOverlay(unsigned int  id, PointerEvent event) {
+    emit hoverOverOverlay(id, event);
+}
+
+void Overlays::sendHoverLeaveOverlay(unsigned int  id, PointerEvent event) {
+    emit hoverLeaveOverlay(id, event);
+}
+
+unsigned int Overlays::getKeyboardFocusOverlay() const {
+    return qApp->getKeyboardFocusOverlay();
+}
+
+void Overlays::setKeyboardFocusOverlay(unsigned int id) {
+    qApp->setKeyboardFocusOverlay(id);
+}
+
 float Overlays::width() const {
     auto offscreenUi = DependencyManager::get<OffscreenUi>();
     return offscreenUi->getWindow()->size().width();

interface/src/ui/overlays/Overlays.h

@@ -82,6 +82,8 @@ const unsigned int UNKNOWN_OVERLAY_ID = 0;
 class Overlays : public QObject {
     Q_OBJECT
 
+    Q_PROPERTY(unsigned int keyboardFocusOverlay READ getKeyboardFocusOverlay WRITE setKeyboardFocusOverlay)
+
 public:
     Overlays();
 
@@ -256,6 +258,17 @@ public slots:
     /// return true if there is a panel with that id else false
     bool isAddedPanel(unsigned int id) { return _panels.contains(id); }
 
+    void sendMousePressOnOverlay(unsigned int overlayID, const PointerEvent& event);
+    void sendMouseReleaseOnOverlay(unsigned int overlayID, const PointerEvent& event);
+    void sendMouseMoveOnOverlay(unsigned int overlayID, const PointerEvent& event);
+
+    void sendHoverEnterOverlay(unsigned int id, PointerEvent event);
+    void sendHoverOverOverlay(unsigned int id, PointerEvent event);
+    void sendHoverLeaveOverlay(unsigned int id, PointerEvent event);
+
+    unsigned int getKeyboardFocusOverlay() const;
+    void setKeyboardFocusOverlay(unsigned int id);
+
 signals:
     /**jsdoc
      * Emitted when an overlay is deleted

scripts/system/controllers/handControllerGrab.js

@@ -186,6 +186,7 @@ var STATE_NEAR_TRIGGER = 4;
 var STATE_FAR_TRIGGER = 5;
 var STATE_HOLD = 6;
 var STATE_ENTITY_TOUCHING = 7;
+var STATE_OVERLAY_TOUCHING = 8;
 
 var holdEnabled = true;
 var nearGrabEnabled = true;
@@ -208,6 +209,8 @@ var mostRecentSearchingHand = RIGHT_HAND;
 
 var DEFAULT_SPHERE_MODEL_URL = "http://hifi-content.s3.amazonaws.com/alan/dev/equip-Fresnel-3.fbx";
 
+var HARDWARE_MOUSE_ID = 0;  // Value reserved for hardware mouse.
+
 CONTROLLER_STATE_MACHINE[STATE_OFF] = {
     name: "off",
     enterMethod: "offEnter",
@@ -249,6 +252,12 @@ CONTROLLER_STATE_MACHINE[STATE_ENTITY_TOUCHING] = {
     exitMethod: "entityTouchingExit",
     updateMethod: "entityTouching"
 };
+CONTROLLER_STATE_MACHINE[STATE_OVERLAY_TOUCHING] = {
+    name: "overlayTouching",
+    enterMethod: "overlayTouchingEnter",
+    exitMethod: "overlayTouchingExit",
+    updateMethod: "overlayTouching"
+};
 
 function distanceBetweenPointAndEntityBoundingBox(point, entityProps) {
     var entityXform = new Xform(entityProps.rotation, entityProps.position);
@@ -273,27 +282,40 @@ function angleBetween(a, b) {
     return Math.acos(Vec3.dot(Vec3.normalize(a), Vec3.normalize(b)));
 }
 
-function projectOntoEntityXYPlane(entityID, worldPos) {
+function projectOntoXYPlane(worldPos, position, rotation, dimensions, registrationPoint) {
-    var props = entityPropertiesCache.getProps(entityID);
+    var invRot = Quat.inverse(rotation);
-    var invRot = Quat.inverse(props.rotation);
+    var localPos = Vec3.multiplyQbyV(invRot, Vec3.subtract(worldPos, position));
-    var localPos = Vec3.multiplyQbyV(invRot, Vec3.subtract(worldPos, props.position));
+    var invDimensions = { x: 1 / dimensions.x,
-    var invDimensions = { x: 1 / props.dimensions.x,
+                          y: 1 / dimensions.y,
-                          y: 1 / props.dimensions.y,
+                          z: 1 / dimensions.z };
-                          z: 1 / props.dimensions.z };
+    var normalizedPos = Vec3.sum(Vec3.multiplyVbyV(localPos, invDimensions), registrationPoint);
-    var normalizedPos = Vec3.sum(Vec3.multiplyVbyV(localPos, invDimensions), props.registrationPoint);
+    return { x: normalizedPos.x * dimensions.x,
-    return { x: normalizedPos.x * props.dimensions.x,
+             y: (1 - normalizedPos.y) * dimensions.y }; // flip y-axis
-             y: (1 - normalizedPos.y) * props.dimensions.y }; // flip y-axis
 }
 
-function handLaserIntersectEntity(entityID, start) {
+function projectOntoEntityXYPlane(entityID, worldPos) {
+    var props = entityPropertiesCache.getProps(entityID);
+    return projectOntoXYPlane(worldPos, props.position, props.rotation, props.dimensions, props.registrationPoint);
+}
+
+function projectOntoOverlayXYPlane(overlayID, worldPos) {
+    var position = Overlays.getProperty(overlayID, "position");
+    var rotation = Overlays.getProperty(overlayID, "rotation");
+    var dimensions = Overlays.getProperty(overlayID, "dimensions");
+    if (!dimensions.z) {
+        dimensions.z = 0;
+    }
+    var registrationPoint = { x: 0.5, y: 0.5, z: 0.5 };
+    return projectOntoXYPlane(worldPos, position, rotation, dimensions, registrationPoint);
+}
+
+function handLaserIntersectItem(position, rotation, start) {
     var worldHandPosition = start.position;
     var worldHandRotation = start.orientation;
 
-    var props = entityPropertiesCache.getProps(entityID);
+    if (position) {
-
+        var planePosition = position;
-    if (props.position) {
+        var planeNormal = Vec3.multiplyQbyV(rotation, {x: 0, y: 0, z: 1.0});
-        var planePosition = props.position;
-        var planeNormal = Vec3.multiplyQbyV(props.rotation, {x: 0, y: 0, z: 1.0});
         var rayStart = worldHandPosition;
         var rayDirection = Quat.getUp(worldHandRotation);
         var intersectionInfo = rayIntersectPlane(planePosition, planeNormal, rayStart, rayDirection);
@@ -318,6 +340,17 @@ function handLaserIntersectEntity(entityID, start) {
     }
 }
 
+function handLaserIntersectEntity(entityID, start) {
+    var props = entityPropertiesCache.getProps(entityID);
+    return handLaserIntersectItem(props.position, props.rotation, start);
+}
+
+function handLaserIntersectOverlay(overlayID, start) {
+    var position = Overlays.getProperty(overlayID, "position");
+    var rotation = Overlays.getProperty(overlayID, "rotation");
+    return handLaserIntersectItem(position, rotation, start);
+}
+
 function rayIntersectPlane(planePosition, planeNormal, rayStart, rayDirection) {
     var rayDirectionDotPlaneNormal = Vec3.dot(rayDirection, planeNormal);
     if (rayDirectionDotPlaneNormal > 0.00001 || rayDirectionDotPlaneNormal < -0.00001) {
@@ -727,6 +760,7 @@ function MyController(hand) {
 
     this.actionID = null; // action this script created...
     this.grabbedEntity = null; // on this entity.
+    this.grabbedOverlay = null;
     this.state = STATE_OFF;
     this.pointer = null; // entity-id of line object
     this.entityActivated = false;
@@ -1159,6 +1193,7 @@ function MyController(hand) {
 
         var result = {
             entityID: null,
+            overlayID: null,
             searchRay: pickRay,
             distance: PICK_MAX_DISTANCE
         };
@@ -1427,6 +1462,7 @@ function MyController(hand) {
         var name;
 
         this.grabbedEntity = null;
+        this.grabbedOverlay = null;
         this.isInitialGrab = false;
         this.shouldResetParentOnRelease = false;
 
@@ -1539,7 +1575,8 @@ function MyController(hand) {
             // send mouse events for button highlights and tooltips.
             if (this.hand == mostRecentSearchingHand || (this.hand !== mostRecentSearchingHand &&
                                                          this.getOtherHandController().state !== STATE_SEARCHING &&
-                                                         this.getOtherHandController().state !== STATE_ENTITY_TOUCHING)) {
+                                                         this.getOtherHandController().state !== STATE_ENTITY_TOUCHING &&
+                                                         this.getOtherHandController().state !== STATE_OVERLAY_TOUCHING)) {
 
                 // most recently searching hand has priority over other hand, for the purposes of button highlighting.
                 pointerEvent = {
@@ -1592,6 +1629,64 @@ function MyController(hand) {
             }
         }
 
+        var overlay;
+
+        if (rayPickInfo.overlayID) {
+            overlay = rayPickInfo.overlayID;
+
+            if (Overlays.keyboardFocusOverlay != overlay) {
+                Overlays.keyboardFocusOverlay = overlay;
+
+                pointerEvent = {
+                    type: "Move",
+                    id: HARDWARE_MOUSE_ID,
+                    pos2D: projectOntoOverlayXYPlane(overlay, rayPickInfo.intersection),
+                    pos3D: rayPickInfo.intersection,
+                    normal: rayPickInfo.normal,
+                    direction: rayPickInfo.searchRay.direction,
+                    button: "None"
+                };
+
+                this.hoverOverlay = overlay;
+                Overlays.sendHoverEnterOverlay(overlay, pointerEvent);
+            }
+
+            // Send mouse events for button highlights and tooltips.
+            if (this.hand == mostRecentSearchingHand || (this.hand !== mostRecentSearchingHand &&
+                                                         this.getOtherHandController().state !== STATE_SEARCHING &&
+                                                         this.getOtherHandController().state !== STATE_ENTITY_TOUCHING &&
+                                                         this.getOtherHandController().state !== STATE_OVERLAY_TOUCHING)) {
+
+                // most recently searching hand has priority over other hand, for the purposes of button highlighting.
+                pointerEvent = {
+                    type: "Move",
+                    id: HARDWARE_MOUSE_ID,
+                    pos2D: projectOntoOverlayXYPlane(overlay, rayPickInfo.intersection),
+                    pos3D: rayPickInfo.intersection,
+                    normal: rayPickInfo.normal,
+                    direction: rayPickInfo.searchRay.direction,
+                    button: "None"
+                };
+
+                Overlays.sendMouseMoveOnOverlay(overlay, pointerEvent);
+                Overlays.sendHoverOverOverlay(overlay, pointerEvent);
+            }
+
+            if (this.triggerSmoothedGrab() && !isEditing()) {
+                this.grabbedOverlay = overlay;
+                this.setState(STATE_OVERLAY_TOUCHING, "begin touching overlay '" + overlay + "'");
+                return;
+            }
+
+        } else if (this.hoverOverlay) {
+            pointerEvent = {
+                type: "Move",
+                id: HARDWARE_MOUSE_ID
+            };
+            Overlays.sendHoverLeaveOverlay(this.hoverOverlay, pointerEvent);
+            this.hoverOverlay = null;
+        }
+
         this.updateEquipHaptics(potentialEquipHotspot, handPosition);
 
         var nearEquipHotspots = this.chooseNearEquipHotspots(candidateEntities, EQUIP_HOTSPOT_RENDER_RADIUS);
@@ -2342,7 +2437,6 @@ function MyController(hand) {
             Entities.sendClickReleaseOnEntity(this.grabbedEntity, pointerEvent);
             Entities.sendHoverLeaveEntity(this.grabbedEntity, pointerEvent);
         }
-        this.focusedEntity = null;
     };
 
     this.entityTouching = function(dt) {
@@ -2396,6 +2490,110 @@ function MyController(hand) {
         }
     };
 
+    this.overlayTouchingEnter = function () {
+        // Test for intersection between controller laser and Web overlay plane.
+        var intersectInfo =
+            handLaserIntersectOverlay(this.grabbedOverlay, getControllerWorldLocation(this.handToController(), true));
+        if (intersectInfo) {
+            var pointerEvent = {
+                type: "Press",
+                id: HARDWARE_MOUSE_ID,
+                pos2D: projectOntoOverlayXYPlane(this.grabbedOverlay, intersectInfo.point),
+                pos3D: intersectInfo.point,
+                normal: intersectInfo.normal,
+                direction: intersectInfo.searchRay.direction,
+                button: "Primary",
+                isPrimaryHeld: true
+            };
+
+            // TODO: post2D isn't calculated correctly; use dummy values to prevent crash.
+            pointerEvent.pos2D.x = 50;
+            pointerEvent.pos2D.y = 50;
+
+            Overlays.sendMousePressOnOverlay(this.grabbedOverlay, pointerEvent);
+
+            this.touchingEnterTimer = 0;
+            this.touchingEnterPointerEvent = pointerEvent;
+            this.touchingEnterPointerEvent.button = "None";
+            this.deadspotExpired = false;
+        }
+    };
+
+    this.overlayTouchingExit = function () {
+        // Test for intersection between controller laser and Web overlay plane.
+        var intersectInfo =
+            handLaserIntersectOverlay(this.grabbedOverlay, getControllerWorldLocation(this.handToController(), true));
+        if (intersectInfo) {
+            var pointerEvent;
+            if (this.deadspotExpired) {
+                pointerEvent = {
+                    type: "Release",
+                    id: HARDWARE_MOUSE_ID,
+                    pos2D: projectOntoOverlayXYPlane(this.grabbedOverlay, intersectInfo.point),
+                    pos3D: intersectInfo.point,
+                    normal: intersectInfo.normal,
+                    direction: intersectInfo.searchRay.direction,
+                    button: "Primary"
+                };
+            } else {
+                pointerEvent = this.touchingEnterPointerEvent;
+                pointerEvent.type = "Release";
+                pointerEvent.button = "Primary";
+                pointerEvent.isPrimaryHeld = false;
+            }
+
+            Overlays.sendMouseReleaseOnOverlay(this.grabbedOverlay, pointerEvent);
+            Overlays.sendHoverLeaveOverlay(this.grabbedOverlay, pointerEvent);
+        }
+    };
+
+    this.overlayTouching = function (dt) {
+        this.touchingEnterTimer += dt;
+
+        if (!this.triggerSmoothedGrab()) {
+            this.setState(STATE_OFF, "released trigger");
+            return;
+        }
+
+        // Test for intersection between controller laser and Web overlay plane.
+        var intersectInfo =
+            handLaserIntersectOverlay(this.grabbedOverlay, getControllerWorldLocation(this.handToController(), true));
+        if (intersectInfo) {
+
+            if (Overlays.keyboardFocusOverlay != this.grabbedOverlay) {
+                Overlays.keyboardFocusOverlay = this.grabbedOverlay;
+            }
+
+            var pointerEvent = {
+                type: "Move",
+                id: HARDWARE_MOUSE_ID,
+                pos2D: projectOntoOverlayXYPlane(this.grabbedOverlay, intersectInfo.point),
+                pos3D: intersectInfo.point,
+                normal: intersectInfo.normal,
+                direction: intersectInfo.searchRay.direction,
+                button: "NoButtons",
+                isPrimaryHeld: true
+            };
+
+            var POINTER_PRESS_TO_MOVE_DELAY = 0.15; // seconds
+            var POINTER_PRESS_TO_MOVE_DEADSPOT_ANGLE = 0.05; // radians ~ 3 degrees
+            if (this.deadspotExpired || this.touchingEnterTimer > POINTER_PRESS_TO_MOVE_DELAY ||
+                angleBetween(pointerEvent.direction, this.touchingEnterPointerEvent.direction) > POINTER_PRESS_TO_MOVE_DEADSPOT_ANGLE) {
+                Overlays.sendMouseMoveOnOverlay(this.grabbedOverlay, pointerEvent);
+                this.deadspotExpired = true;
+            }
+
+            this.intersectionDistance = intersectInfo.distance;
+            if (farGrabEnabled) {
+                this.searchIndicatorOn(intersectInfo.searchRay);
+            }
+            Reticle.setVisible(false);
+        } else {
+            this.setState(STATE_OFF, "grabbed overlay was destroyed");
+            return;
+        }
+    };
+
     this.release = function() {
         this.turnOffVisualizations();
 
@@ -2437,6 +2635,7 @@ function MyController(hand) {
 
         this.actionID = null;
         this.grabbedEntity = null;
+        this.grabbedOverlay = null;
         this.grabbedHotspot = null;
 
         if (this.triggerSmoothedGrab()) {