HUD events WIP, fix entity/overlay hover, no hover for empty renderState

interface/src/Application.cpp

@@ -1811,6 +1811,12 @@ Application::Application(int& argc, char** argv, QElapsedTimer& startupTimer, bo
     connect(&_myCamera, &Camera::modeUpdated, this, &Application::cameraModeChanged);
 
     DependencyManager::get<PickManager>()->setShouldPickHUDOperator([&]() { return DependencyManager::get<HMDScriptingInterface>()->isHMDMode(); });
+    DependencyManager::get<PickManager>()->setCalculatePos2DFromHUDOperator([&](const glm::vec3& intersection) {
+        const glm::vec2 MARGIN(25.0f);
+        glm::vec2 maxPos = _controllerScriptingInterface->getViewportDimensions() - MARGIN;
+        glm::vec2 pos2D = DependencyManager::get<HMDScriptingInterface>()->overlayFromWorldPoint(intersection);
+        return glm::max(MARGIN, glm::min(pos2D, maxPos));
+    });
 
     // Setup the mouse ray pick and related operators
     DependencyManager::get<EntityTreeRenderer>()->setMouseRayPickID(DependencyManager::get<PickManager>()->addPick(PickQuery::Ray, std::make_shared<MouseRayPick>(

interface/src/raypick/LaserPointer.cpp

@@ -301,6 +301,8 @@ PointerEvent LaserPointer::buildPointerEvent(const PickedObject& target, const Q
         pos2D = projectOntoEntityXYPlane(target.objectID, intersection);
     } else if (target.type == OVERLAY) {
         pos2D = projectOntoOverlayXYPlane(target.objectID, intersection);
+    } else if (target.type == HUD) {
+        pos2D = DependencyManager::get<PickManager>()->calculatePos2DFromHUD(intersection);
     }
     return PointerEvent(PointerEvent::Move, id, pos2D, intersection, surfaceNormal, direction, PointerEvent::NoButtons);
 }

interface/src/raypick/LaserPointer.h

@@ -75,6 +75,9 @@ public:
 protected:
     PointerEvent buildPointerEvent(const PickedObject& target, const QVariantMap& pickResult) const override;
 
+    bool shouldHover() override { return _currentRenderState != ""; }
+    bool shouldTrigger() override { return _currentRenderState != ""; }
+
 private:
     PointerTriggers _triggers;
     float _laserLength { 0.0f };

interface/src/ui/overlays/Overlays.cpp

@@ -46,7 +46,7 @@ extern void initOverlay3DPipelines(render::ShapePlumber& plumber, bool depthTest
 Overlays::Overlays() {
     auto pointerManager = DependencyManager::get<PointerManager>();
     connect(pointerManager.data(), &PointerManager::hoverBeginOverlay, this, &Overlays::hoverEnterOverlay);
-    connect(pointerManager.data(), &PointerManager::hoverContinueOverlay, this, &Overlays::hoverOverOverlay);
+    connect(pointerManager.data(), &PointerManager::hoverContinueOverlay, this, &Overlays::hoverOverPointerEvent);
     connect(pointerManager.data(), &PointerManager::hoverEndOverlay, this, &Overlays::hoverLeavePointerEvent);
     connect(pointerManager.data(), &PointerManager::triggerBeginOverlay, this, &Overlays::mousePressPointerEvent);
     connect(pointerManager.data(), &PointerManager::triggerContinueOverlay, this, &Overlays::mouseMovePointerEvent);
@@ -745,7 +745,7 @@ void Overlays::sendHoverEnterOverlay(const OverlayID& overlayID, const PointerEv
 }
 
 void Overlays::sendHoverOverOverlay(const OverlayID& overlayID, const PointerEvent& event) {
-    emit hoverOverOverlay(overlayID, event);
+    hoverOverPointerEvent(overlayID, event);
 }
 
 void Overlays::sendHoverLeaveOverlay(const OverlayID& overlayID, const PointerEvent& event) {
@@ -940,6 +940,21 @@ bool Overlays::mouseDoublePressEvent(QMouseEvent* event) {
     return false;
 }
 
+void Overlays::hoverOverPointerEvent(const OverlayID& overlayID, const PointerEvent& event) {
+    // TODO: generalize this to allow any overlay to recieve events
+    std::shared_ptr<Web3DOverlay> thisOverlay;
+    if (getOverlayType(overlayID) == "web3d") {
+        thisOverlay = std::static_pointer_cast<Web3DOverlay>(getOverlay(overlayID));
+    }
+    if (thisOverlay) {
+        // Send to web overlay
+        QMetaObject::invokeMethod(thisOverlay.get(), "handlePointerEvent", Q_ARG(PointerEvent, event));
+    }
+
+    // emit to scripts
+    emit hoverOverOverlay(overlayID, event);
+}
+
 void Overlays::hoverLeavePointerEvent(const OverlayID& overlayID, const PointerEvent& event) {
     // TODO: generalize this to allow any overlay to recieve events
     std::shared_ptr<Web3DOverlay> thisOverlay;
@@ -1005,7 +1020,7 @@ bool Overlays::mouseMoveEvent(QMouseEvent* event) {
         }
 
         // Hover over current overlay.
-        emit hoverOverOverlay(rayPickResult.overlayID, pointerEvent);
+        hoverOverPointerEvent(rayPickResult.overlayID, pointerEvent);
 
         _currentHoverOverOverlayID = rayPickResult.overlayID;
     } else {

interface/src/ui/overlays/Overlays.h

@@ -308,6 +308,7 @@ public slots:
     void mousePressPointerEvent(const OverlayID& overlayID, const PointerEvent& event);
     void mouseMovePointerEvent(const OverlayID& overlayID, const PointerEvent& event);
     void mouseReleasePointerEvent(const OverlayID& overlayID, const PointerEvent& event);
+    void hoverOverPointerEvent(const OverlayID& overlayID, const PointerEvent& event);
     void hoverLeavePointerEvent(const OverlayID& overlayID, const PointerEvent& event);
 
 signals:

libraries/entities-renderer/src/EntityTreeRenderer.cpp

@@ -83,6 +83,7 @@ EntityTreeRenderer::EntityTreeRenderer(bool wantScripts, AbstractViewStateInterf
     connect(entityScriptingInterface.data(), &EntityScriptingInterface::mousePressOnEntity, this, handlePointerEvent);
     connect(entityScriptingInterface.data(), &EntityScriptingInterface::mouseReleaseOnEntity, this, handlePointerEvent);
     connect(entityScriptingInterface.data(), &EntityScriptingInterface::mouseMoveOnEntity, this, handlePointerEvent);
+    connect(entityScriptingInterface.data(), &EntityScriptingInterface::hoverOverEntity, this, handlePointerEvent);
     connect(entityScriptingInterface.data(), &EntityScriptingInterface::hoverLeaveEntity, this, [&](const EntityItemID& entityID, const PointerEvent& event) {
         std::shared_ptr<render::entities::WebEntityRenderer> thisEntity;
         auto entity = getEntity(entityID);

libraries/pointers/src/pointers/PickManager.cpp

@@ -9,6 +9,7 @@
 
 PickManager::PickManager() {
     setShouldPickHUDOperator([]() { return false; });
+    setCalculatePos2DFromHUDOperator([](const glm::vec3& intersection) { return glm::vec2(NAN); });
 }
 
 QUuid PickManager::addPick(PickQuery::PickType type, const std::shared_ptr<PickQuery> pick) {

libraries/pointers/src/pointers/PickManager.h

@@ -34,9 +34,12 @@ public:
     void setIncludeItems(const QUuid& uid, const QVector<QUuid>& include) const;
 
     void setShouldPickHUDOperator(std::function<bool()> shouldPickHUDOperator) { _shouldPickHUDOperator = shouldPickHUDOperator; }
+    void setCalculatePos2DFromHUDOperator(std::function<glm::vec2(const glm::vec3&)> calculatePos2DFromHUDOperator) { _calculatePos2DFromHUDOperator = calculatePos2DFromHUDOperator; }
+    glm::vec2 calculatePos2DFromHUD(const glm::vec3& intersection) { return _calculatePos2DFromHUDOperator(intersection); }
 
 protected:
     std::function<bool()> _shouldPickHUDOperator;
+    std::function<glm::vec2(const glm::vec3&)> _calculatePos2DFromHUDOperator;
 
     std::shared_ptr<PickQuery> findPick(const QUuid& uid) const;
     QHash<PickQuery::PickType, QHash<QUuid, std::shared_ptr<PickQuery>>> _picks;

libraries/pointers/src/pointers/Pointer.cpp

@@ -56,7 +56,7 @@ void Pointer::update() {
 }
 
 void Pointer::generatePointerEvents(const QVariantMap& pickResult) {
-    // TODO: avatars/HUD?
+    // TODO: avatars?
     auto pointerManager = DependencyManager::get<PointerManager>();
 
     // Hover events
@@ -65,7 +65,7 @@ void Pointer::generatePointerEvents(const QVariantMap& pickResult) {
     hoveredEvent.setType(PointerEvent::Move);
     // TODO: set buttons on hover events
     hoveredEvent.setButton(PointerEvent::NoButtons);
-    if (_enabled && _hover) {
+    if (_enabled && _hover && shouldHover()) {
         if (hoveredObject.type == OVERLAY) {
             if (_prevHoveredObject.type == OVERLAY) {
                 if (hoveredObject.objectID == _prevHoveredObject.objectID) {
@@ -79,6 +79,8 @@ void Pointer::generatePointerEvents(const QVariantMap& pickResult) {
                 emit pointerManager->hoverBeginOverlay(hoveredObject.objectID, hoveredEvent);
                 if (_prevHoveredObject.type == ENTITY) {
                     emit pointerManager->hoverEndEntity(_prevHoveredObject.objectID, hoveredEvent);
+                } else if (_prevHoveredObject.type == HUD) {
+                    emit pointerManager->hoverEndHUD(_prevHoveredObject.objectID, hoveredEvent);
                 }
             }
         }
@@ -97,6 +99,22 @@ void Pointer::generatePointerEvents(const QVariantMap& pickResult) {
                 emit pointerManager->hoverBeginEntity(hoveredObject.objectID, hoveredEvent);
                 if (_prevHoveredObject.type == OVERLAY) {
                     emit pointerManager->hoverEndOverlay(_prevHoveredObject.objectID, hoveredEvent);
+                } else if (_prevHoveredObject.type == HUD) {
+                    emit pointerManager->hoverEndHUD(_prevHoveredObject.objectID, hoveredEvent);
+                }
+            }
+        }
+
+        if (hoveredObject.type == HUD) {
+            if (_prevHoveredObject.type == HUD) {
+                // There's only one HUD
+                emit pointerManager->hoverContinueHUD(hoveredObject.objectID, hoveredEvent);
+            } else {
+                emit pointerManager->hoverBeginHUD(hoveredObject.objectID, hoveredEvent);
+                if (_prevHoveredObject.type == ENTITY) {
+                    emit pointerManager->hoverEndEntity(_prevHoveredObject.objectID, hoveredEvent);
+                } else if (_prevHoveredObject.type == OVERLAY) {
+                    emit pointerManager->hoverEndOverlay(_prevHoveredObject.objectID, hoveredEvent);
                 }
             }
         }
@@ -107,8 +125,8 @@ void Pointer::generatePointerEvents(const QVariantMap& pickResult) {
     Buttons newButtons;
     Buttons sameButtons;
     // NOTE: After this loop: _prevButtons = buttons that were removed
-    // If !_enabled, release all buttons
+    // If switching to disabled or should stop triggering, release all buttons
-    if (_enabled) {
+    if (_enabled && shouldTrigger()) {
         buttons = getPressedButtons();
         for (const std::string& button : buttons) {
             if (_prevButtons.find(button) == _prevButtons.end()) {
@@ -130,6 +148,8 @@ void Pointer::generatePointerEvents(const QVariantMap& pickResult) {
             emit pointerManager->triggerBeginEntity(hoveredObject.objectID, hoveredEvent);
         } else if (hoveredObject.type == OVERLAY) {
             emit pointerManager->triggerBeginOverlay(hoveredObject.objectID, hoveredEvent);
+        } else if (hoveredObject.type == HUD) {
+            emit pointerManager->triggerBeginHUD(hoveredObject.objectID, hoveredEvent);
         }
         _triggeredObjects[button] = hoveredObject;
     }
@@ -143,6 +163,8 @@ void Pointer::generatePointerEvents(const QVariantMap& pickResult) {
             emit pointerManager->triggerContinueEntity(_triggeredObjects[button].objectID, triggeredEvent);
         } else if (_triggeredObjects[button].type == OVERLAY) {
             emit pointerManager->triggerContinueOverlay(_triggeredObjects[button].objectID, triggeredEvent);
+        } else if (_triggeredObjects[button].type == HUD) {
+            emit pointerManager->triggerContinueHUD(_triggeredObjects[button].objectID, triggeredEvent);
         }
     }
 
@@ -155,6 +177,8 @@ void Pointer::generatePointerEvents(const QVariantMap& pickResult) {
             emit pointerManager->triggerEndEntity(_triggeredObjects[button].objectID, triggeredEvent);
         } else if (_triggeredObjects[button].type == OVERLAY) {
             emit pointerManager->triggerEndOverlay(_triggeredObjects[button].objectID, triggeredEvent);
+        } else if (_triggeredObjects[button].type == HUD) {
+            emit pointerManager->triggerEndHUD(_triggeredObjects[button].objectID, triggeredEvent);
         }
         _triggeredObjects.erase(button);
     }

libraries/pointers/src/pointers/Pointer.h

@@ -71,9 +71,6 @@ public:
 
     using Buttons = std::unordered_set<std::string>;
 
-    virtual PickedObject getHoveredObject(const QVariantMap& pickResult) = 0;
-    virtual Buttons getPressedButtons() = 0;
-
     QUuid getRayUID() { return _pickUID; }
 
 protected:
@@ -83,6 +80,12 @@ protected:
 
     virtual PointerEvent buildPointerEvent(const PickedObject& target, const QVariantMap& pickResult) const = 0;
 
+    virtual PickedObject getHoveredObject(const QVariantMap& pickResult) = 0;
+    virtual Buttons getPressedButtons() = 0;
+
+    virtual bool shouldHover() = 0;
+    virtual bool shouldTrigger() = 0;
+
 private:
     PickedObject _prevHoveredObject;
     Buttons _prevButtons;

libraries/pointers/src/pointers/PointerManager.h

@@ -58,6 +58,13 @@ signals:
     void hoverBeginEntity(const QUuid& id, const PointerEvent& pointerEvent);
     void hoverContinueEntity(const QUuid& id, const PointerEvent& pointerEvent);
     void hoverEndEntity(const QUuid& id, const PointerEvent& pointerEvent);
+
+    void triggerBeginHUD(const QUuid& id, const PointerEvent& pointerEvent);
+    void triggerContinueHUD(const QUuid& id, const PointerEvent& pointerEvent);
+    void triggerEndHUD(const QUuid& id, const PointerEvent& pointerEvent);
+    void hoverBeginHUD(const QUuid& id, const PointerEvent& pointerEvent);
+    void hoverContinueHUD(const QUuid& id, const PointerEvent& pointerEvent);
+    void hoverEndHUD(const QUuid& id, const PointerEvent& pointerEvent);
 };
 
 #endif // hifi_pointers_PointerManager_h

libraries/shared/src/PointerEvent.cpp

@@ -41,6 +41,11 @@ PointerEvent::PointerEvent(EventType type, uint32_t id,
     ;
 }
 
+void PointerEvent::setButton(Button button) {
+    _button = button;
+    _buttons |= button;
+}
+
 QScriptValue PointerEvent::toScriptValue(QScriptEngine* engine, const PointerEvent& event) {
     QScriptValue obj = engine->newObject();
 

libraries/shared/src/PointerEvent.h

@@ -58,7 +58,7 @@ public:
     bool shouldFocus() const { return _shouldFocus; }
 
     void setType(EventType type) { _type = type; }
-    void setButton(Button button) { _button = button; }
+    void setButton(Button button);
     void setShouldFocus(bool focus) { _shouldFocus = focus; }
 
 private:

libraries/ui/CMakeLists.txt

@@ -1,6 +1,7 @@
 set(TARGET_NAME ui)
 setup_hifi_library(OpenGL Network Qml Quick Script WebChannel WebEngine WebSockets XmlPatterns)
-link_hifi_libraries(shared networking gl audio)
+link_hifi_libraries(shared networking gl audio pointers)
+include_hifi_library_headers(controllers)
 
 # Required for some low level GL interaction in the OffscreenQMLSurface
 target_opengl()

libraries/ui/src/ui/OffscreenQmlSurface.cpp

@@ -49,6 +49,8 @@
 
 #include "Logging.h"
 
+#include <pointers/PointerManager.h>
+
 Q_LOGGING_CATEGORY(trace_render_qml, "trace.render.qml")
 Q_LOGGING_CATEGORY(trace_render_qml_gl, "trace.render.qml.gl")
 Q_LOGGING_CATEGORY(offscreenFocus, "hifi.offscreen.focus")
@@ -529,6 +531,13 @@ bool OffscreenQmlSurface::allowNewFrame(uint8_t fps) {
 }
 
 OffscreenQmlSurface::OffscreenQmlSurface() {
+    auto pointerManager = DependencyManager::get<PointerManager>();
+    connect(pointerManager.data(), &PointerManager::hoverBeginHUD, this, &OffscreenQmlSurface::handlePointerEvent);
+    connect(pointerManager.data(), &PointerManager::hoverContinueHUD, this, &OffscreenQmlSurface::handlePointerEvent);
+    connect(pointerManager.data(), &PointerManager::hoverEndHUD, this, &OffscreenQmlSurface::handlePointerEvent);
+    connect(pointerManager.data(), &PointerManager::triggerBeginHUD, this, &OffscreenQmlSurface::handlePointerEvent);
+    connect(pointerManager.data(), &PointerManager::triggerContinueHUD, this, &OffscreenQmlSurface::handlePointerEvent);
+    connect(pointerManager.data(), &PointerManager::triggerEndHUD, this, &OffscreenQmlSurface::handlePointerEvent);
 }
 
 OffscreenQmlSurface::~OffscreenQmlSurface() {
@@ -932,14 +941,8 @@ bool OffscreenQmlSurface::eventFilter(QObject* originalDestination, QEvent* even
                     transformedPos,
                     mouseEvent->screenPos(), mouseEvent->button(),
                     mouseEvent->buttons(), mouseEvent->modifiers());
-            if (event->type() == QEvent::MouseMove) {
+            if (sendMouseEvent(mappedEvent)) {
-                // TODO - this line necessary for the QML Tooltop to work (which is not currently being used), but it causes interface to crash on launch on a fresh install
+                return true;
-                // need to investigate into why this crash is happening.
-                //_qmlContext->setContextProperty("lastMousePosition", transformedPos);
-            }
-            mappedEvent.ignore();
-            if (QCoreApplication::sendEvent(_quickWindow, &mappedEvent)) {
-                return mappedEvent.isAccepted();
             }
             break;
         }
@@ -951,6 +954,44 @@ bool OffscreenQmlSurface::eventFilter(QObject* originalDestination, QEvent* even
     return false;
 }
 
+void OffscreenQmlSurface::handlePointerEvent(const QUuid& id, const PointerEvent& event) {
+    if (_paused) {
+        return;
+    }
+
+    QEvent::Type type;
+    switch (event.getType()) {
+        case PointerEvent::Press:
+            type = QEvent::Type::MouseButtonPress;
+            break;
+        case PointerEvent::DoublePress:
+            type = QEvent::Type::MouseButtonDblClick;
+            break;
+        case PointerEvent::Release:
+            type = QEvent::Type::MouseButtonRelease;
+            break;
+        case PointerEvent::Move:
+            type = QEvent::Type::MouseMove;
+            break;
+    }
+    QPointF screenPos(event.getPos2D().x, event.getPos2D().y);
+    QMouseEvent mouseEvent(type, screenPos, Qt::MouseButton(event.getButton()), Qt::MouseButtons(event.getButtons()), event.getKeyboardModifiers());
+    sendMouseEvent(mouseEvent);
+}
+
+bool OffscreenQmlSurface::sendMouseEvent(QMouseEvent& mouseEvent) {
+    if (mouseEvent.type() == QEvent::MouseMove) {
+        // TODO - this line necessary for the QML Tooltop to work (which is not currently being used), but it causes interface to crash on launch on a fresh install
+        // need to investigate into why this crash is happening.
+        //_qmlContext->setContextProperty("lastMousePosition", mouseEvent.localPos());
+    }
+    mouseEvent.ignore();
+    if (QCoreApplication::sendEvent(_quickWindow, &mouseEvent)) {
+        return mouseEvent.isAccepted();
+    }
+    return false;
+}
+
 void OffscreenQmlSurface::pause() {
     _paused = true;
 }

libraries/ui/src/ui/OffscreenQmlSurface.h

@@ -30,6 +30,8 @@ class QQmlContext;
 class QQmlComponent;
 class QQuickWindow;
 class QQuickItem;
+class QMouseEvent;
+class PointerEvent;
 
 // GPU resources are typically buffered for one copy being used by the renderer, 
 // one copy in flight, and one copy being used by the receiver
@@ -135,6 +137,7 @@ private:
 private slots:
     void updateQuick();
     void onFocusObjectChanged(QObject* newFocus);
+    void handlePointerEvent(const QUuid& id, const PointerEvent& event);
 
 private:
     QQuickWindow* _quickWindow { nullptr };
@@ -161,6 +164,8 @@ private:
     QWindow* _proxyWindow { nullptr };
 
     QQuickItem* _currentFocusItem { nullptr };
+
+    bool sendMouseEvent(QMouseEvent& mouseEvent);
 };
 
 #endif