Working on 2D overlay mouse interaction

examples/tests/overlayMouseTrackingTest.js

@@ -0,0 +1,46 @@
+MouseTracker = function() {
+    this.WIDTH = 60;
+    this.HEIGHT = 60;
+
+    this.overlay = Overlays.addOverlay("image", {
+        imageURL: Script.resolvePath("dot.png"),
+        width: this.HEIGHT,
+        height: this.WIDTH,
+        x: 100,
+        y: 100,
+        visible: true
+    });
+
+    var that = this;
+    Script.scriptEnding.connect(function() {
+        that.onCleanup();
+    });
+
+    Controller.mousePressEvent.connect(function(event) {
+        that.onMousePress(event);
+    });
+
+    Controller.mouseMoveEvent.connect(function(event) {
+        that.onMouseMove(event);
+    });
+}
+
+MouseTracker.prototype.onCleanup = function() {
+    Overlays.deleteOverlay(this.overlay);
+}
+
+MouseTracker.prototype.onMousePress = function(event) {
+}
+
+MouseTracker.prototype.onMouseMove = function(event) {
+    var width = Overlays.width();
+    var height = Overlays.height();
+    var x = Math.max(event.x, 0);
+    x = Math.min(x, width);
+    var y = Math.max(event.y, 0);
+    y = Math.min(y, height);
+    Overlays.editOverlay(this.overlay, {x: x - this.WIDTH / 2.0, y: y - this.HEIGHT / 2.0});
+} 
+
+
+new MouseTracker();

interface/src/Application.cpp

@@ -1849,8 +1849,16 @@ void Application::mouseMoveEvent(QMouseEvent* event, unsigned int deviceID) {
 
 
     _entities.mouseMoveEvent(event, deviceID);
+    {
+        auto offscreenUi = DependencyManager::get<OffscreenUi>();
+        QPointF transformedPos = offscreenUi->mapToVirtualScreen(event->localPos(), _glWidget);
+        QMouseEvent mappedEvent(event->type(),
+            transformedPos,
+            event->screenPos(), event->button(),
+            event->buttons(), event->modifiers());
+        _controllerScriptingInterface.emitMouseMoveEvent(&mappedEvent, deviceID); // send events to any registered scripts
+    }
 
-    _controllerScriptingInterface.emitMouseMoveEvent(event, deviceID); // send events to any registered scripts
     // if one of our scripts have asked to capture this event, then stop processing it
     if (_controllerScriptingInterface.isMouseCaptured()) {
         return;
@@ -1865,12 +1873,19 @@ void Application::mouseMoveEvent(QMouseEvent* event, unsigned int deviceID) {
 void Application::mousePressEvent(QMouseEvent* event, unsigned int deviceID) {
     // Inhibit the menu if the user is using alt-mouse dragging
     _altPressed = false;
-
     if (!_aboutToQuit) {
         _entities.mousePressEvent(event, deviceID);
     }
 
-    _controllerScriptingInterface.emitMousePressEvent(event); // send events to any registered scripts
+    {
+        auto offscreenUi = DependencyManager::get<OffscreenUi>();
+        QPointF transformedPos = offscreenUi->mapToVirtualScreen(event->localPos(), _glWidget);
+        QMouseEvent mappedEvent(event->type(),
+            transformedPos,
+            event->screenPos(), event->button(),
+            event->buttons(), event->modifiers());
+        _controllerScriptingInterface.emitMousePressEvent(&mappedEvent); // send events to any registered scripts
+    }
 
     // if one of our scripts have asked to capture this event, then stop processing it
     if (_controllerScriptingInterface.isMouseCaptured()) {
@@ -1921,7 +1936,15 @@ void Application::mouseReleaseEvent(QMouseEvent* event, unsigned int deviceID) {
         _entities.mouseReleaseEvent(event, deviceID);
     }
 
-    _controllerScriptingInterface.emitMouseReleaseEvent(event); // send events to any registered scripts
+    {
+        auto offscreenUi = DependencyManager::get<OffscreenUi>();
+        QPointF transformedPos = offscreenUi->mapToVirtualScreen(event->localPos(), _glWidget);
+        QMouseEvent mappedEvent(event->type(),
+            transformedPos,
+            event->screenPos(), event->button(),
+            event->buttons(), event->modifiers());
+        _controllerScriptingInterface.emitMouseReleaseEvent(&mappedEvent); // send events to any registered scripts
+    }
 
     // if one of our scripts have asked to capture this event, then stop processing it
     if (_controllerScriptingInterface.isMouseCaptured()) {

interface/src/ui/overlays/Overlays.cpp

@@ -10,7 +10,7 @@
 
 #include "Overlays.h"
 
-#include <QScriptValueIterator>
+#include <QtScript/QScriptValueIterator>
 
 #include <limits>
 
@@ -31,6 +31,7 @@
 #include "TextOverlay.h"
 #include "Text3DOverlay.h"
 #include "Web3DOverlay.h"
+#include <QtQuick/QQuickWindow>
 
 
 Overlays::Overlays() : _nextOverlayID(1) {
@@ -331,10 +332,6 @@ void Overlays::setParentPanel(unsigned int childId, unsigned int panelId) {
 
 unsigned int Overlays::getOverlayAtPoint(const glm::vec2& point) {
     glm::vec2 pointCopy = point;
-    if (qApp->isHMDMode()) {
-        pointCopy = qApp->getApplicationCompositor().screenToOverlay(point);
-    }
-    
     QReadLocker lock(&_lock);
     if (!_enabled) {
         return 0;
@@ -607,3 +604,13 @@ void Overlays::deletePanel(unsigned int panelId) {
 bool Overlays::isAddedOverlay(unsigned int id) {
     return _overlaysHUD.contains(id) || _overlaysWorld.contains(id);
 }
+
+float Overlays::width() const {
+    auto offscreenUi = DependencyManager::get<OffscreenUi>();
+    return offscreenUi->getWindow()->size().width();
+}
+
+float Overlays::height() const {
+    auto offscreenUi = DependencyManager::get<OffscreenUi>();
+    return offscreenUi->getWindow()->size().height();
+}

interface/src/ui/overlays/Overlays.h

@@ -113,6 +113,10 @@ public slots:
     /// overlay; in meters if it is a 3D text overlay
     QSizeF textSize(unsigned int id, const QString& text) const;
 
+    // Return the size of the virtual screen
+    float width() const;
+    float height() const;
+
 
     /// adds a panel that has already been created
     unsigned int addPanel(OverlayPanel::Pointer panel);

libraries/render-utils/src/OffscreenQmlSurface.cpp

@@ -496,6 +496,13 @@ QPointF OffscreenQmlSurface::mapWindowToUi(const QPointF& sourcePosition, QObjec
     return QPointF(offscreenPosition.x, offscreenPosition.y);
 }
 
+QPointF OffscreenQmlSurface::mapToVirtualScreen(const QPointF& originalPoint, QObject* originalWidget) {
+    QPointF transformedPos = _mouseTranslator(originalPoint);
+    transformedPos = mapWindowToUi(transformedPos, originalWidget);
+    return transformedPos;
+}
+
+
 ///////////////////////////////////////////////////////
 //
 // Event handling customization
@@ -541,8 +548,9 @@ bool OffscreenQmlSurface::eventFilter(QObject* originalDestination, QEvent* even
 
         case QEvent::Wheel: {
             QWheelEvent* wheelEvent = static_cast<QWheelEvent*>(event);
+            QPointF transformedPos = mapToVirtualScreen(wheelEvent->pos(), originalDestination);
             QWheelEvent mappedEvent(
-                    mapWindowToUi(wheelEvent->pos(), originalDestination),
+                    transformedPos,
                     wheelEvent->delta(),  wheelEvent->buttons(),
                     wheelEvent->modifiers(), wheelEvent->orientation());
             mappedEvent.ignore();
@@ -558,9 +566,7 @@ bool OffscreenQmlSurface::eventFilter(QObject* originalDestination, QEvent* even
         case QEvent::MouseButtonRelease:
         case QEvent::MouseMove: {
             QMouseEvent* mouseEvent = static_cast<QMouseEvent*>(event);
-            QPointF originalPos = mouseEvent->localPos();
-            QPointF transformedPos = _mouseTranslator(originalPos);
-            transformedPos = mapWindowToUi(transformedPos, originalDestination);
+            QPointF transformedPos = mapToVirtualScreen(mouseEvent->localPos(), originalDestination);
             QMouseEvent mappedEvent(mouseEvent->type(),
                     transformedPos,
                     mouseEvent->screenPos(), mouseEvent->button(),

libraries/render-utils/src/OffscreenQmlSurface.h

@@ -61,6 +61,7 @@ public:
     QQuickWindow* getWindow();
     QObject* getEventHandler();
 
+    QPointF mapToVirtualScreen(const QPointF& originalPoint, QObject* originalWidget);
     virtual bool eventFilter(QObject* originalDestination, QEvent* event);
 
 signals: