fix multi-touch and keyboard on web entities

interface/src/raypick/LaserPointer.cpp

@@ -285,13 +285,12 @@ PointerEvent LaserPointer::buildPointerEvent(const PickedObject& target, const Q
     uint32_t id = 0;
     glm::vec3 intersection = vec3FromVariant(pickResult["intersection"]);
     glm::vec3 surfaceNormal = vec3FromVariant(pickResult["surfaceNormal"]);
-    glm::vec3 direction = -surfaceNormal;
+    QVariantMap searchRay = pickResult["searchRay"].toMap();
+    glm::vec3 direction = vec3FromVariant(searchRay["direction"]);
     QUuid pickedID = pickResult["objectID"].toUuid();
     glm::vec2 pos2D;
     if (pickedID != target.objectID) {
-        QVariantMap searchRay = pickResult["searchRay"].toMap();
         glm::vec3 origin = vec3FromVariant(searchRay["origin"]);
-        glm::vec3 direction = vec3FromVariant(searchRay["direction"]);
         if (target.type == ENTITY) {
             intersection = intersectRayWithEntityXYPlane(target.objectID, origin, direction);
         } else if (target.type == OVERLAY) {

interface/src/ui/overlays/Web3DOverlay.cpp

@@ -66,7 +66,7 @@ const QString Web3DOverlay::QML = "Web3DOverlay.qml";
 Web3DOverlay::Web3DOverlay() : _dpi(DPI) {
     _touchDevice.setCapabilities(QTouchDevice::Position);
     _touchDevice.setType(QTouchDevice::TouchScreen);
-    _touchDevice.setName("RenderableWebEntityItemTouchDevice");
+    _touchDevice.setName("Web3DOverlayTouchDevice");
     _touchDevice.setMaximumTouchPoints(4);
 
     _geometryId = DependencyManager::get<GeometryCache>()->allocateID();
@@ -332,6 +332,12 @@ void Web3DOverlay::setProxyWindow(QWindow* proxyWindow) {
 }
 
 void Web3DOverlay::handlePointerEvent(const PointerEvent& event) {
+    if (event.getType() == PointerEvent::Press) {
+        _pressed = true;
+    } else if (event.getType() == PointerEvent::Release) {
+        _pressed = false;
+    }
+
     if (_inputMode == Touch) {
         handlePointerEventAsTouch(event);
     } else {
@@ -344,19 +350,8 @@ void Web3DOverlay::handlePointerEventAsTouch(const PointerEvent& event) {
         return;
     }
 
-    //do not send secondary button events to tablet
-    if (event.getButton() == PointerEvent::SecondaryButton ||
-        //do not block composed events
-        event.getButtons() == PointerEvent::SecondaryButton) {
-        return;
-    }
-
-
-    QPointF windowPoint;
-    {
-        glm::vec2 windowPos = event.getPos2D() * (METERS_TO_INCHES * _dpi);
-        windowPoint = QPointF(windowPos.x, windowPos.y);
-    }
+    glm::vec2 windowPos = event.getPos2D() * (METERS_TO_INCHES * _dpi);
+    QPointF windowPoint(windowPos.x, windowPos.y);
 
     Qt::TouchPointState state = Qt::TouchPointStationary;
     if (event.getType() == PointerEvent::Press && event.getButton() == PointerEvent::PrimaryButton) {
@@ -395,14 +390,13 @@ void Web3DOverlay::handlePointerEventAsTouch(const PointerEvent& event) {
         }
 
         touchEvent.setWindow(_webSurface->getWindow());
+        touchEvent.setDevice(&_touchDevice);
         touchEvent.setTarget(_webSurface->getRootItem());
         touchEvent.setTouchPoints(touchPoints);
         touchEvent.setTouchPointStates(touchPointStates);
     }
 
     // Send mouse events to the Web surface so that HTML dialog elements work with mouse press and hover.
-    // FIXME: Scroll bar dragging is a bit unstable in the tablet (content can jump up and down at times).
-    // This may be improved in Qt 5.8. Release notes: "Cleaned up touch and mouse event delivery".
     //
     // In Qt 5.9 mouse events must be sent before touch events to make sure some QtQuick components will
     // receive mouse events
@@ -449,12 +443,6 @@ void Web3DOverlay::handlePointerEventAsMouse(const PointerEvent& event) {
     glm::vec2 windowPos = event.getPos2D() * (METERS_TO_INCHES * _dpi);
     QPointF windowPoint(windowPos.x, windowPos.y);
 
-    if (event.getType() == PointerEvent::Press) {
-        this->_pressed = true;
-    } else if (event.getType() == PointerEvent::Release) {
-        this->_pressed = false;
-    }
-
     Qt::MouseButtons buttons = Qt::NoButton;
     if (event.getButtons() & PointerEvent::PrimaryButton) {
         buttons |= Qt::LeftButton;

libraries/entities-renderer/src/RenderableWebEntityItem.cpp

@@ -227,6 +227,8 @@ bool WebEntityRenderer::buildWebSurface(const TypedEntityPointer& entity) {
     _webSurface->setMaxFps(DEFAULT_MAX_FPS);
     // FIXME - Keyboard HMD only: Possibly add "HMDinfo" object to context for WebView.qml.
     _webSurface->getSurfaceContext()->setContextProperty("desktop", QVariant());
+    // Let us interact with the keyboard
+    _webSurface->getSurfaceContext()->setContextProperty("tabletInterface", DependencyManager::get<TabletScriptingInterface>().data());
     _fadeStartTime = usecTimestampNow();
     loadSourceURL();
     _webSurface->resume();
@@ -315,21 +317,9 @@ void WebEntityRenderer::loadSourceURL() {
 void WebEntityRenderer::hoverLeaveEntity(const PointerEvent& event) {
     if (!_lastLocked && _webSurface && _pressed) {
         // If the user mouses off the entity while the button is down, simulate a touch end.
-        QTouchEvent::TouchPoint point;
-        point.setId(event.getID());
-        point.setState(Qt::TouchPointReleased);
-        glm::vec2 windowPos = event.getPos2D() * (METERS_TO_INCHES * _lastDPI);
-        QPointF windowPoint(windowPos.x, windowPos.y);
-        point.setScenePos(windowPoint);
-        point.setPos(windowPoint);
-        QList<QTouchEvent::TouchPoint> touchPoints;
-        touchPoints.push_back(point);
-        QTouchEvent* touchEvent = new QTouchEvent(QEvent::TouchEnd, nullptr,
-            Qt::NoModifier, Qt::TouchPointReleased, touchPoints);
-        touchEvent->setWindow(_webSurface->getWindow());
-        touchEvent->setDevice(&_touchDevice);
-        touchEvent->setTarget(_webSurface->getRootItem());
-        QCoreApplication::postEvent(_webSurface->getWindow(), touchEvent);
+        PointerEvent endEvent(PointerEvent::Release, event.getID(), event.getPos2D(), event.getPos3D(), event.getNormal(), event.getDirection(),
+            event.getButton(), event.getButtons(), event.getKeyboardModifiers());
+        handlePointerEvent(endEvent);
     }
 }
 
@@ -339,57 +329,95 @@ void WebEntityRenderer::handlePointerEvent(const PointerEvent& event) {
         return;
     }
 
+    if (event.getType() == PointerEvent::Press) {
+        _pressed = true;
+    } else if (event.getType() == PointerEvent::Release) {
+        _pressed = false;
+    }
+
     glm::vec2 windowPos = event.getPos2D() * (METERS_TO_INCHES * _lastDPI);
     QPointF windowPoint(windowPos.x, windowPos.y);
-    if (event.getType() == PointerEvent::Move) {
-        // Forward a mouse move event to webSurface
-        QMouseEvent* mouseEvent = new QMouseEvent(QEvent::MouseMove, windowPoint, windowPoint, windowPoint, Qt::NoButton, Qt::NoButton, Qt::NoModifier);
-        QCoreApplication::postEvent(_webSurface->getWindow(), mouseEvent);
+
+    Qt::TouchPointState state = Qt::TouchPointStationary;
+    if (event.getType() == PointerEvent::Press && event.getButton() == PointerEvent::PrimaryButton) {
+        state = Qt::TouchPointPressed;
+    } else if (event.getType() == PointerEvent::Release) {
+        state = Qt::TouchPointReleased;
+    } else if (_activeTouchPoints.count(event.getID()) && windowPoint != _activeTouchPoints[event.getID()].pos()) {
+        state = Qt::TouchPointMoved;
+    }
+
+    QEvent::Type touchType = QEvent::TouchUpdate;
+    if (_activeTouchPoints.empty()) {
+        // If the first active touch point is being created, send a begin
+        touchType = QEvent::TouchBegin;
+    } if (state == Qt::TouchPointReleased && _activeTouchPoints.size() == 1 && _activeTouchPoints.count(event.getID())) {
+        // If the last active touch point is being released, send an end
+        touchType = QEvent::TouchEnd;
     }
 
     {
-        // Forward a touch update event to webSurface
-        if (event.getType() == PointerEvent::Press) {
-            this->_pressed = true;
-        } else if (event.getType() == PointerEvent::Release) {
-            this->_pressed = false;
-        }
-
-        QEvent::Type type;
-        Qt::TouchPointState touchPointState;
-        switch (event.getType()) {
-            case PointerEvent::Press:
-                type = QEvent::TouchBegin;
-                touchPointState = Qt::TouchPointPressed;
-                break;
-            case PointerEvent::Release:
-                type = QEvent::TouchEnd;
-                touchPointState = Qt::TouchPointReleased;
-                break;
-            case PointerEvent::Move:
-            default:
-                type = QEvent::TouchUpdate;
-                touchPointState = Qt::TouchPointMoved;
-                break;
-        }
-
         QTouchEvent::TouchPoint point;
         point.setId(event.getID());
-        point.setState(touchPointState);
+        point.setState(state);
         point.setPos(windowPoint);
         point.setScreenPos(windowPoint);
-        QList<QTouchEvent::TouchPoint> touchPoints;
-        touchPoints.push_back(point);
-
-        QTouchEvent* touchEvent = new QTouchEvent(type);
-        touchEvent->setWindow(_webSurface->getWindow());
-        touchEvent->setDevice(&_touchDevice);
-        touchEvent->setTarget(_webSurface->getRootItem());
-        touchEvent->setTouchPoints(touchPoints);
-        touchEvent->setTouchPointStates(touchPointState);
-
-        QCoreApplication::postEvent(_webSurface->getWindow(), touchEvent);
+        _activeTouchPoints[event.getID()] = point;
     }
+
+    QTouchEvent touchEvent(touchType, &_touchDevice, event.getKeyboardModifiers());
+    {
+        QList<QTouchEvent::TouchPoint> touchPoints;
+        Qt::TouchPointStates touchPointStates;
+        for (const auto& entry : _activeTouchPoints) {
+            touchPointStates |= entry.second.state();
+            touchPoints.push_back(entry.second);
+        }
+
+        touchEvent.setWindow(_webSurface->getWindow());
+        touchEvent.setDevice(&_touchDevice);
+        touchEvent.setTarget(_webSurface->getRootItem());
+        touchEvent.setTouchPoints(touchPoints);
+        touchEvent.setTouchPointStates(touchPointStates);
+    }
+
+    // Send mouse events to the Web surface so that HTML dialog elements work with mouse press and hover.
+    //
+    // In Qt 5.9 mouse events must be sent before touch events to make sure some QtQuick components will
+    // receive mouse events
+    Qt::MouseButton button = Qt::NoButton;
+    Qt::MouseButtons buttons = Qt::NoButton;
+    if (event.getButton() == PointerEvent::PrimaryButton) {
+        button = Qt::LeftButton;
+    }
+    if (event.getButtons() & PointerEvent::PrimaryButton) {
+        buttons |= Qt::LeftButton;
+    }
+
+#if QT_VERSION >= QT_VERSION_CHECK(5, 9, 0)
+    if (event.getType() == PointerEvent::Move) {
+        QMouseEvent mouseEvent(QEvent::MouseMove, windowPoint, windowPoint, windowPoint, button, buttons, Qt::NoModifier);
+        QCoreApplication::sendEvent(_webSurface->getWindow(), &mouseEvent);
+    }
+#endif
+
+    if (touchType == QEvent::TouchBegin) {
+        _touchBeginAccepted = QCoreApplication::sendEvent(_webSurface->getWindow(), &touchEvent);
+    } else if (_touchBeginAccepted) {
+        QCoreApplication::sendEvent(_webSurface->getWindow(), &touchEvent);
+    }
+
+    // If this was a release event, remove the point from the active touch points
+    if (state == Qt::TouchPointReleased) {
+        _activeTouchPoints.erase(event.getID());
+    }
+
+#if QT_VERSION < QT_VERSION_CHECK(5, 9, 0)
+    if (event.getType() == PointerEvent::Move) {
+        QMouseEvent mouseEvent(QEvent::MouseMove, windowPoint, windowPoint, windowPoint, button, buttons, Qt::NoModifier);
+        QCoreApplication::sendEvent(_webSurface->getWindow(), &mouseEvent);
+    }
+#endif
 }
 
 void WebEntityRenderer::setProxyWindow(QWindow* proxyWindow) {

libraries/entities-renderer/src/RenderableWebEntityItem.h

@@ -12,6 +12,8 @@
 #include <WebEntityItem.h>
 #include "RenderableEntityItem.h"
 
+#include <QTouchEvent>
+
 class OffscreenQmlSurface;
 class PointerEvent;
 
@@ -58,13 +60,16 @@ private:
     QSharedPointer<OffscreenQmlSurface> _webSurface;
     glm::vec3 _contextPosition;
     gpu::TexturePointer _texture;
-    bool _pressed{ false };
     QString _lastSourceUrl;
     uint16_t _lastDPI;
     bool _lastLocked;
     QTimer _timer;
     uint64_t _lastRenderTime { 0 };
     Transform _renderTransform;
+
+    bool _pressed{ false };
+    bool _touchBeginAccepted{ false };
+    std::map<uint32_t, QTouchEvent::TouchPoint> _activeTouchPoints;
 };
 
 } } // namespace 

libraries/pointers/src/pointers/Pointer.cpp

@@ -23,10 +23,11 @@ void Pointer::enable() {
 }
 
 void Pointer::disable() {
-    DependencyManager::get<PickManager>()->disablePick(_pickUID);
+    // Disable the pointer first, then the pick, so someone can't try to use it while it's in a bad state
     withWriteLock([&] {
         _enabled = false;
     });
+    DependencyManager::get<PickManager>()->disablePick(_pickUID);
 }
 
 const QVariantMap Pointer::getPrevPickResult() {