Add mouse press, move, and release handling to Web overlay

interface/src/Application.cpp

@@ -2867,6 +2867,7 @@ void Application::mouseMoveEvent(QMouseEvent* event) {
 
     if (compositor.getReticleVisible() || !isHMDMode() || !compositor.getReticleOverDesktop() ||
         getOverlays().getOverlayAtPoint(glm::vec2(transformedPos.x(), transformedPos.y()))) {
+        getOverlays().mouseMoveEvent(&mappedEvent);
         getEntities()->mouseMoveEvent(&mappedEvent);
     }
     _controllerScriptingInterface->emitMouseMoveEvent(&mappedEvent); // send events to any registered scripts
@@ -2901,6 +2902,7 @@ void Application::mousePressEvent(QMouseEvent* event) {
         event->buttons(), event->modifiers());
 
     if (!_aboutToQuit) {
+        getOverlays().mousePressEvent(&mappedEvent);
         getEntities()->mousePressEvent(&mappedEvent);
     }
 
@@ -2947,6 +2949,7 @@ void Application::mouseReleaseEvent(QMouseEvent* event) {
         event->buttons(), event->modifiers());
 
     if (!_aboutToQuit) {
+        getOverlays().mouseReleaseEvent(&mappedEvent);
         getEntities()->mouseReleaseEvent(&mappedEvent);
     }
 

interface/src/ui/overlays/Overlays.cpp

@@ -605,3 +605,134 @@ float Overlays::height() const {
     auto offscreenUi = DependencyManager::get<OffscreenUi>();
     return offscreenUi->getWindow()->size().height();
 }
+
+static const uint32_t MOUSE_POINTER_ID = 0;
+
+static glm::vec2 projectOntoOverlayXYPlane(glm::vec3 position, glm::quat rotation, glm::vec2 dimensions, const PickRay& pickRay, 
+    const RayToOverlayIntersectionResult& rayPickResult) {
+
+    // Project the intersection point onto the local xy plane of the overlay.
+    float distance;
+    glm::vec3 planePosition = position;
+    glm::vec3 planeNormal = rotation * Vectors::UNIT_Z;
+    glm::vec3 overlayDimensions = glm::vec3(dimensions.x, dimensions.y, 0.0f);
+    glm::vec3 rayDirection = pickRay.direction;
+    glm::vec3 rayStart = pickRay.origin;
+    glm::vec3 p;
+    if (rayPlaneIntersection(planePosition, planeNormal, rayStart, rayDirection, distance)) {
+        p = rayStart + rayDirection * distance;
+    } else {
+        p = rayPickResult.intersection;
+    }
+    glm::vec3 localP = glm::inverse(rotation) * (p - position);
+    glm::vec3 normalizedP = (localP / overlayDimensions) + glm::vec3(0.5f);
+    return glm::vec2(normalizedP.x * overlayDimensions.x,
+        (1.0f - normalizedP.y) * overlayDimensions.y);  // flip y-axis
+}
+
+static uint32_t toPointerButtons(const QMouseEvent& event) {
+    uint32_t buttons = 0;
+    buttons |= event.buttons().testFlag(Qt::LeftButton) ? PointerEvent::PrimaryButton : 0;
+    buttons |= event.buttons().testFlag(Qt::RightButton) ? PointerEvent::SecondaryButton : 0;
+    buttons |= event.buttons().testFlag(Qt::MiddleButton) ? PointerEvent::TertiaryButton : 0;
+    return buttons;
+}
+
+static PointerEvent::Button toPointerButton(const QMouseEvent& event) {
+    switch (event.button()) {
+        case Qt::LeftButton:
+            return PointerEvent::PrimaryButton;
+        case Qt::RightButton:
+            return PointerEvent::SecondaryButton;
+        case Qt::MiddleButton:
+            return PointerEvent::TertiaryButton;
+        default:
+            return PointerEvent::NoButtons;
+    }
+}
+
+void Overlays::mousePressEvent(QMouseEvent* event) {
+    PerformanceTimer perfTimer("Overlays::mousePressEvent");
+
+    PickRay ray = qApp->computePickRay(event->x(), event->y());
+    RayToOverlayIntersectionResult rayPickResult = findRayIntersection(ray);
+    if (rayPickResult.intersects) {
+        _currentClickingOnOverlayID = rayPickResult.overlayID;
+
+        // Only Web overlays can have focus.
+        auto thisOverlay = std::dynamic_pointer_cast<Web3DOverlay>(getOverlay(_currentClickingOnOverlayID));
+        if (thisOverlay) {
+            QReadLocker lock(&_lock);
+            auto position = thisOverlay->getPosition();
+            auto rotation = thisOverlay->getRotation();
+            auto dimensions = thisOverlay->getSize();
+
+            glm::vec2 pos2D = projectOntoOverlayXYPlane(position, rotation, dimensions, ray, rayPickResult);
+            PointerEvent pointerEvent(PointerEvent::Press, MOUSE_POINTER_ID,
+                pos2D, rayPickResult.intersection,
+                rayPickResult.surfaceNormal, ray.direction,
+                toPointerButton(*event), toPointerButtons(*event));
+
+            emit mousePressOnOverlay(_currentClickingOnOverlayID, pointerEvent);
+
+        } else {
+            emit mousePressOffOverlay();
+        }
+    } else {
+        emit mousePressOffOverlay();
+    }
+}
+
+void Overlays::mouseReleaseEvent(QMouseEvent* event) {
+    PerformanceTimer perfTimer("Overlays::mouseReleaseEvent");
+
+    PickRay ray = qApp->computePickRay(event->x(), event->y());
+    RayToOverlayIntersectionResult rayPickResult = findRayIntersection(ray);
+    if (rayPickResult.intersects) {
+
+        // Only Web overlays can have focus.
+        auto thisOverlay = std::dynamic_pointer_cast<Web3DOverlay>(getOverlay(rayPickResult.overlayID));
+        if (thisOverlay) {
+            QReadLocker lock(&_lock);
+            auto position = thisOverlay->getPosition();
+            auto rotation = thisOverlay->getRotation();
+            auto dimensions = thisOverlay->getSize();
+
+            glm::vec2 pos2D = projectOntoOverlayXYPlane(position, rotation, dimensions, ray, rayPickResult);
+            PointerEvent pointerEvent(PointerEvent::Release, MOUSE_POINTER_ID,
+                pos2D, rayPickResult.intersection,
+                rayPickResult.surfaceNormal, ray.direction,
+                toPointerButton(*event), toPointerButtons(*event));
+
+            emit mouseReleaseOnOverlay(rayPickResult.overlayID, pointerEvent);
+        }
+    }
+
+    _currentClickingOnOverlayID = UNKNOWN_OVERLAY_ID;
+}
+
+void Overlays::mouseMoveEvent(QMouseEvent* event) {
+    PerformanceTimer perfTimer("Overlays::mouseMoveEvent");
+
+    PickRay ray = qApp->computePickRay(event->x(), event->y());
+    RayToOverlayIntersectionResult rayPickResult = findRayIntersection(ray);
+    if (rayPickResult.intersects) {
+
+        // Only Web overlays can have focus.
+        auto thisOverlay = std::dynamic_pointer_cast<Web3DOverlay>(getOverlay(_currentClickingOnOverlayID));
+        if (thisOverlay) {
+            QReadLocker lock(&_lock);
+            auto position = thisOverlay->getPosition();
+            auto rotation = thisOverlay->getRotation();
+            auto dimensions = thisOverlay->getSize();
+
+            glm::vec2 pos2D = projectOntoOverlayXYPlane(position, rotation, dimensions, ray, rayPickResult);
+            PointerEvent pointerEvent(PointerEvent::Move, MOUSE_POINTER_ID,
+                pos2D, rayPickResult.intersection,
+                rayPickResult.surfaceNormal, ray.direction,
+                toPointerButton(*event), toPointerButtons(*event));
+
+            emit mouseMoveOnOverlay(rayPickResult.overlayID, pointerEvent);
+        }
+    }
+}

interface/src/ui/overlays/Overlays.h

@@ -18,11 +18,13 @@
 #ifndef hifi_Overlays_h
 #define hifi_Overlays_h
 
+#include <QMouseEvent>
 #include <QReadWriteLock>
 #include <QScriptValue>
 
-#include "Overlay.h"
+#include <PointerEvent.h>
 
+#include "Overlay.h"
 #include "OverlayPanel.h"
 #include "PanelAttachable.h"
 
@@ -75,6 +77,8 @@ void RayToOverlayIntersectionResultFromScriptValue(const QScriptValue& object, R
  * @namespace Overlays
  */
 
+const int UNKNOWN_OVERLAY_ID = -1;
+
 class Overlays : public QObject {
     Q_OBJECT
 
@@ -94,6 +98,10 @@ public:
     unsigned int addOverlay(Overlay* overlay) { return addOverlay(Overlay::Pointer(overlay)); }
     unsigned int addOverlay(Overlay::Pointer overlay);
 
+    void mousePressEvent(QMouseEvent* event);
+    void mouseReleaseEvent(QMouseEvent* event);
+    void mouseMoveEvent(QMouseEvent* event);
+
     void cleanupAllOverlays();
 
 public slots:
@@ -249,6 +257,11 @@ signals:
     void overlayDeleted(unsigned int id);
     void panelDeleted(unsigned int id);
 
+    void mousePressOnOverlay(const int overlayID, const PointerEvent& event);
+    void mouseReleaseOnOverlay(const int overlayID, const PointerEvent& event);
+    void mouseMoveOnOverlay(const int overlayID, const PointerEvent& event);
+    void mousePressOffOverlay();
+
 private:
     void cleanupOverlaysToDelete();
 
@@ -262,8 +275,8 @@ private:
     QReadWriteLock _deleteLock;
     QScriptEngine* _scriptEngine;
     bool _enabled = true;
+
+    int _currentClickingOnOverlayID = UNKNOWN_OVERLAY_ID;
 };
 
-
-
 #endif // hifi_Overlays_h

interface/src/ui/overlays/Web3DOverlay.cpp

@@ -1,7 +1,6 @@
 //
 //  Web3DOverlay.cpp
 //
-//
 //  Created by Clement on 7/1/14.
 //  Modified and renamed by Zander Otavka on 8/4/15
 //  Copyright 2014 High Fidelity, Inc.
@@ -12,6 +11,9 @@
 
 #include "Web3DOverlay.h"
 
+#include <Application.h>
+
+#include <QQuickWindow>
 #include <QtGui/QOpenGLContext>
 #include <QtQuick/QQuickItem>
 
@@ -33,6 +35,11 @@ static float OPAQUE_ALPHA_THRESHOLD = 0.99f;
 QString const Web3DOverlay::TYPE = "web3d";
 
 Web3DOverlay::Web3DOverlay() : _dpi(DPI) { 
+    _touchDevice.setCapabilities(QTouchDevice::Position);
+    _touchDevice.setType(QTouchDevice::TouchScreen);
+    _touchDevice.setName("RenderableWebEntityItemTouchDevice");
+    _touchDevice.setMaximumTouchPoints(4);
+
     _geometryId = DependencyManager::get<GeometryCache>()->allocateID();
 }
 
@@ -50,6 +57,12 @@ Web3DOverlay::~Web3DOverlay() {
         _webSurface->pause();
         _webSurface->disconnect(_connection);
 
+        QObject::disconnect(_mousePressConnection);
+        _mousePressConnection = QMetaObject::Connection();
+        QObject::disconnect(_mouseReleaseConnection);
+        _mouseReleaseConnection = QMetaObject::Connection();
+        QObject::disconnect(_mouseMoveConnection);
+        _mouseMoveConnection = QMetaObject::Connection();
 
         // The lifetime of the QML surface MUST be managed by the main thread
         // Additionally, we MUST use local variables copied by value, rather than
@@ -98,10 +111,19 @@ void Web3DOverlay::render(RenderArgs* args) {
         _webSurface->getRootItem()->setProperty("url", _url);
         _webSurface->resize(QSize(_resolution.x, _resolution.y));
         currentContext->makeCurrent(currentSurface);
+
+        auto forwardPointerEvent = [=](const int overlayID, const PointerEvent& event) {
+            if (overlayID == getOverlayID()) {
+                handlePointerEvent(event);
+            }
+        };
+
+        _mousePressConnection = connect(&(qApp->getOverlays()), &Overlays::mousePressOnOverlay, forwardPointerEvent);
+        _mouseReleaseConnection = connect(&(qApp->getOverlays()), &Overlays::mouseReleaseOnOverlay, forwardPointerEvent);
+        _mouseMoveConnection = connect(&(qApp->getOverlays()), &Overlays::mouseMoveOnOverlay, forwardPointerEvent);
     }
 
-    vec2 size = _resolution / _dpi * INCHES_TO_METERS;
-    vec2 halfSize = size / 2.0f;
+    vec2 halfSize = getSize() / 2.0f;
     vec4 color(toGlm(getColor()), getAlpha());
 
     Transform transform = getTransform();
@@ -144,6 +166,59 @@ const render::ShapeKey Web3DOverlay::getShapeKey() {
     return builder.build();
 }
 
+void Web3DOverlay::handlePointerEvent(const PointerEvent& event) {
+    if (!_webSurface) {
+        return;
+    }
+
+    const float METERS_TO_INCHES = 39.3701f;
+    glm::vec2 windowPos = event.getPos2D() * (METERS_TO_INCHES * _dpi);
+    QPointF windowPoint(windowPos.x, windowPos.y);
+
+    if (event.getType() == PointerEvent::Move) {
+        // Forward a mouse move event to the Web surface.
+        QMouseEvent* mouseEvent = new QMouseEvent(QEvent::MouseMove, windowPoint, windowPoint, windowPoint, Qt::NoButton, 
+            Qt::NoButton, Qt::NoModifier);
+        QCoreApplication::postEvent(_webSurface->getWindow(), mouseEvent);
+    }
+
+    QEvent::Type type;
+    Qt::TouchPointState touchPointState;
+    switch (event.getType()) {
+        case PointerEvent::Press:
+            type = QEvent::TouchBegin;
+            touchPointState = Qt::TouchPointPressed;
+            break;
+        case PointerEvent::Release:
+            type = QEvent::TouchEnd;
+            touchPointState = Qt::TouchPointReleased;
+            break;
+        case PointerEvent::Move:
+        default:
+            type = QEvent::TouchUpdate;
+            touchPointState = Qt::TouchPointMoved;
+            break;
+    }
+
+    QTouchEvent::TouchPoint point;
+    point.setId(event.getID());
+    point.setState(touchPointState);
+    point.setPos(windowPoint);
+    point.setScreenPos(windowPoint);
+    QList<QTouchEvent::TouchPoint> touchPoints;
+    touchPoints.push_back(point);
+
+    QTouchEvent* touchEvent = new QTouchEvent(type);
+    touchEvent->setWindow(_webSurface->getWindow());
+    touchEvent->setDevice(&_touchDevice);
+    touchEvent->setTarget(_webSurface->getRootItem());
+    touchEvent->setTouchPoints(touchPoints);
+    touchEvent->setTouchPointStates(touchPointState);
+
+
+    QCoreApplication::postEvent(_webSurface->getWindow(), touchEvent);
+}
+
 void Web3DOverlay::setProperties(const QVariantMap& properties) {
     Billboard3DOverlay::setProperties(properties);
 
@@ -188,9 +263,12 @@ void Web3DOverlay::setURL(const QString& url) {
             _webSurface->getRootItem()->setProperty("url", url);
         });
     }
-
 }
 
+glm::vec2 Web3DOverlay::getSize() {
+    return _resolution / _dpi * INCHES_TO_METERS * getDimensions();
+};
+
 bool Web3DOverlay::findRayIntersection(const glm::vec3& origin, const glm::vec3& direction, float& distance, BoxFace& face, glm::vec3& surfaceNormal) {
     // FIXME - face and surfaceNormal not being returned
 
@@ -200,9 +278,8 @@ bool Web3DOverlay::findRayIntersection(const glm::vec3& origin, const glm::vec3&
     //applyTransformTo(transform, true);
     //setTransform(transform);
 
-    vec2 size = _resolution / _dpi * INCHES_TO_METERS * vec2(getDimensions());
     // Produce the dimensions of the overlay based on the image's aspect ratio and the overlay's scale.
-    return findRayRectangleIntersection(origin, direction, getRotation(), getPosition(), size, distance);
+    return findRayRectangleIntersection(origin, direction, getRotation(), getPosition(), getSize(), distance);
 }
 
 Web3DOverlay* Web3DOverlay::createClone() const {

interface/src/ui/overlays/Web3DOverlay.h

@@ -11,6 +11,10 @@
 
 #include "Billboard3DOverlay.h"
 
+#include <QTouchEvent>
+
+#include <PointerEvent.h>
+
 class OffscreenQmlSurface;
 
 class Web3DOverlay : public Billboard3DOverlay {
@@ -29,12 +33,16 @@ public:
 
     virtual void update(float deltatime) override;
 
+    void handlePointerEvent(const PointerEvent& event);
+
     // setters
     void setURL(const QString& url);
 
     void setProperties(const QVariantMap& properties) override;
     QVariant getProperty(const QString& property) override;
 
+    glm::vec2 getSize();
+
     virtual bool findRayIntersection(const glm::vec3& origin, const glm::vec3& direction, float& distance, 
         BoxFace& face, glm::vec3& surfaceNormal) override;
 
@@ -48,6 +56,12 @@ private:
     float _dpi;
     vec2 _resolution{ 640, 480 };
     int _geometryId { 0 };
+
+    QTouchDevice _touchDevice;
+
+    QMetaObject::Connection _mousePressConnection;
+    QMetaObject::Connection _mouseReleaseConnection;
+    QMetaObject::Connection _mouseMoveConnection;
 };
 
 #endif // hifi_Web3DOverlay_h