From 6d768d832723f5894d10c85b978e7796f9946d2d Mon Sep 17 00:00:00 2001 From: "Anthony J. Thibault" Date: Tue, 2 Aug 2016 15:23:46 -0700 Subject: [PATCH] hand controllers send touch events instead of mouse events to web entities This gives a much better experience when scrolling web content. --- interface/src/Application.cpp | 41 +++++++++++ interface/src/Application.h | 5 ++ .../src/display-plugins/CompositorHelper.cpp | 12 ++++ .../src/display-plugins/CompositorHelper.h | 5 +- .../src/RenderableWebEntityItem.cpp | 32 +++++++++ .../src/RenderableWebEntityItem.h | 1 + .../system/controllers/handControllerGrab.js | 68 +++++++++++++++---- 7 files changed, 150 insertions(+), 14 deletions(-) diff --git a/interface/src/Application.cpp b/interface/src/Application.cpp index aefbeff1c3..3d1b28ada8 100644 --- a/interface/src/Application.cpp +++ b/interface/src/Application.cpp @@ -3553,6 +3553,47 @@ void Application::sendEntityMouseEvent(const QUuid& id, const QMouseEvent& mouse } } +void Application::sendEntityTouchUpdateEvent(QUuid entityID, int fingerID, glm::vec3 intersectionPoint) { + QTouchEvent::TouchPoint point; + point.setId(fingerID); + point.setState(Qt::TouchPointMoved); + QList touchPoints; + touchPoints.push_back(point); + QTouchEvent touchEvent(QEvent::TouchUpdate, nullptr, Qt::NoModifier, Qt::TouchPointMoved, touchPoints); + sendEntityTouchEvent(entityID, touchEvent, intersectionPoint); +} + +void Application::sendEntityTouchBeginEvent(QUuid entityID, int fingerID, glm::vec3 intersectionPoint) { + QTouchEvent::TouchPoint point; + point.setId(fingerID); + point.setState(Qt::TouchPointPressed); + QList touchPoints; + touchPoints.push_back(point); + QTouchEvent touchEvent(QEvent::TouchBegin, nullptr, Qt::NoModifier, Qt::TouchPointPressed, touchPoints); + sendEntityTouchEvent(entityID, touchEvent, intersectionPoint); +} + +void Application::sendEntityTouchEndEvent(QUuid entityID, int fingerID, glm::vec3 intersectionPoint) { + QTouchEvent::TouchPoint point; + point.setId(fingerID); + point.setState(Qt::TouchPointReleased); + QList touchPoints; + touchPoints.push_back(point); + QTouchEvent touchEvent(QEvent::TouchEnd, nullptr, Qt::NoModifier, Qt::TouchPointReleased, touchPoints); + sendEntityTouchEvent(entityID, touchEvent, intersectionPoint); +} + +void Application::sendEntityTouchEvent(const QUuid& id, const QTouchEvent& touchEvent, const glm::vec3& intersectionPoint) { + auto entityScriptingInterface = DependencyManager::get(); + EntityItemID entityItemID(id); + auto properties = entityScriptingInterface->getEntityProperties(entityItemID); + if (EntityTypes::Web == properties.getType() && !properties.getLocked() && properties.getVisible()) { + auto entity = entityScriptingInterface->getEntityTree()->findEntityByID(entityItemID); + RenderableWebEntityItem* webEntity = dynamic_cast(entity.get()); + webEntity->handleTouchEvent(touchEvent, intersectionPoint); + } +} + void Application::updateDialogs(float deltaTime) const { PerformanceTimer perfTimer("updateDialogs"); bool showWarnings = Menu::getInstance()->isOptionChecked(MenuOption::PipelineWarnings); diff --git a/interface/src/Application.h b/interface/src/Application.h index 139efb4ba7..bfa0856103 100644 --- a/interface/src/Application.h +++ b/interface/src/Application.h @@ -321,7 +321,12 @@ public slots: void sendEntityLeftMouseDownEvent(QUuid id, glm::vec3 intersectionPoint); void sendEntityLeftMouseUpEvent(QUuid id, glm::vec3 intersectionPoint); + void sendEntityTouchUpdateEvent(QUuid entityID, int fingerID, glm::vec3 intersectionPoint); + void sendEntityTouchBeginEvent(QUuid entityID, int fingerID, glm::vec3 intersectionPoint); + void sendEntityTouchEndEvent(QUuid entityID, int fingerID, glm::vec3 intersectionPoint); + private: + void sendEntityTouchEvent(const QUuid& id, const QTouchEvent& touchEvent, const glm::vec3& intersectionPoint); void sendEntityMouseEvent(const QUuid& id, const QMouseEvent& mouseEvent, const glm::vec3& intersectionPoint); private slots: diff --git a/libraries/display-plugins/src/display-plugins/CompositorHelper.cpp b/libraries/display-plugins/src/display-plugins/CompositorHelper.cpp index e460fa6648..3a154af462 100644 --- a/libraries/display-plugins/src/display-plugins/CompositorHelper.cpp +++ b/libraries/display-plugins/src/display-plugins/CompositorHelper.cpp @@ -473,3 +473,15 @@ void ReticleInterface::sendEntityLeftMouseDownEvent(QUuid id, glm::vec3 intersec void ReticleInterface::sendEntityLeftMouseUpEvent(QUuid id, glm::vec3 intersectionPoint) { QMetaObject::invokeMethod(qApp, "sendEntityLeftMouseUpEvent", Qt::QueuedConnection, Q_ARG(QUuid, id), Q_ARG(glm::vec3, intersectionPoint)); } + +void ReticleInterface::sendEntityTouchUpdateEvent(QUuid entityID, int fingerID, glm::vec3 intersectionPoint) { + QMetaObject::invokeMethod(qApp, "sendEntityTouchUpdateEvent", Qt::QueuedConnection, Q_ARG(QUuid, entityID), Q_ARG(int, fingerID), Q_ARG(glm::vec3, intersectionPoint)); +} + +void ReticleInterface::sendEntityTouchBeginEvent(QUuid entityID, int fingerID, glm::vec3 intersectionPoint) { + QMetaObject::invokeMethod(qApp, "sendEntityTouchBeginEvent", Qt::QueuedConnection, Q_ARG(QUuid, entityID), Q_ARG(int, fingerID), Q_ARG(glm::vec3, intersectionPoint)); +} + +void ReticleInterface::sendEntityTouchEndEvent(QUuid entityID, int fingerID, glm::vec3 intersectionPoint) { + QMetaObject::invokeMethod(qApp, "sendEntityTouchEndEvent", Qt::QueuedConnection, Q_ARG(QUuid, entityID), Q_ARG(int, fingerID), Q_ARG(glm::vec3, intersectionPoint)); +} diff --git a/libraries/display-plugins/src/display-plugins/CompositorHelper.h b/libraries/display-plugins/src/display-plugins/CompositorHelper.h index b93ffeb67b..78d4a10bf0 100644 --- a/libraries/display-plugins/src/display-plugins/CompositorHelper.h +++ b/libraries/display-plugins/src/display-plugins/CompositorHelper.h @@ -209,7 +209,10 @@ public: Q_INVOKABLE void sendEntityMouseMoveEvent(QUuid id, glm::vec3 intersectionPoint); Q_INVOKABLE void sendEntityLeftMouseDownEvent(QUuid id, glm::vec3 intersectionPoint); Q_INVOKABLE void sendEntityLeftMouseUpEvent(QUuid id, glm::vec3 intersectionPoint); - // TODO: right mouse + double click + + Q_INVOKABLE void sendEntityTouchUpdateEvent(QUuid entityID, int fingerID, glm::vec3 intersectionPoint); + Q_INVOKABLE void sendEntityTouchBeginEvent(QUuid entityID, int fingerID, glm::vec3 intersectionPoint); + Q_INVOKABLE void sendEntityTouchEndEvent(QUuid entityID, int fingerID, glm::vec3 intersectionPoint); private: CompositorHelper* _compositor; diff --git a/libraries/entities-renderer/src/RenderableWebEntityItem.cpp b/libraries/entities-renderer/src/RenderableWebEntityItem.cpp index 2b01de9beb..20e921cd16 100644 --- a/libraries/entities-renderer/src/RenderableWebEntityItem.cpp +++ b/libraries/entities-renderer/src/RenderableWebEntityItem.cpp @@ -238,6 +238,38 @@ void RenderableWebEntityItem::handleMouseEvent(QMouseEvent event, glm::vec3 inte QCoreApplication::sendEvent(_webSurface->getWindow(), &mappedEvent); } +void RenderableWebEntityItem::handleTouchEvent(QTouchEvent event, glm::vec3 intersectionPoint) { + // Ignore mouse interaction if we're locked + if (getLocked()) { + return; + } + + // Map the intersection point to an actual offscreen pixel + glm::vec3 point = intersectionPoint; + glm::vec3 dimensions = getDimensions(); + point -= getPosition(); + point = glm::inverse(getRotation()) * point; + point /= dimensions; + point += 0.5f; + point.y = 1.0f - point.y; + point *= dimensions * (METERS_TO_INCHES * DPI); + + QList touchPoints = event.touchPoints(); + for (auto& touchPoint : touchPoints) { + touchPoint.setPos(QPointF(point.x, point.y)); + touchPoint.setScreenPos(QPointF(point.x, point.y)); + } + + // Forward the touch event. + QTouchEvent mappedEvent(event.type(), + event.device(), + event.modifiers(), + event.touchPointStates(), + touchPoints); + QCoreApplication::sendEvent(_webSurface->getWindow(), &mappedEvent); +} + + void RenderableWebEntityItem::destroyWebSurface() { if (_webSurface) { --_currentWebCount; diff --git a/libraries/entities-renderer/src/RenderableWebEntityItem.h b/libraries/entities-renderer/src/RenderableWebEntityItem.h index b20a9af928..c6afacdf5f 100644 --- a/libraries/entities-renderer/src/RenderableWebEntityItem.h +++ b/libraries/entities-renderer/src/RenderableWebEntityItem.h @@ -34,6 +34,7 @@ public: QObject* getEventHandler(); void handleMouseEvent(QMouseEvent event, glm::vec3 intersectionPoint); + void handleTouchEvent(QTouchEvent event, glm::vec3 intersectionPoint); void update(const quint64& now) override; bool needsToCallUpdate() const override { return _webSurface != nullptr; } diff --git a/scripts/system/controllers/handControllerGrab.js b/scripts/system/controllers/handControllerGrab.js index 3d4c9ac8b0..6a3f0ca949 100644 --- a/scripts/system/controllers/handControllerGrab.js +++ b/scripts/system/controllers/handControllerGrab.js @@ -192,6 +192,7 @@ CONTROLLER_STATE_MACHINE[STATE_OFF] = { }; CONTROLLER_STATE_MACHINE[STATE_SEARCHING] = { name: "searching", + enterMethod: "searchEnter", updateMethod: "search" }; CONTROLLER_STATE_MACHINE[STATE_DISTANCE_HOLDING] = { @@ -220,6 +221,18 @@ CONTROLLER_STATE_MACHINE[STATE_FAR_TRIGGER] = { updateMethod: "farTrigger" }; +function rayIntersectPlane(planePosition, planeNormal, rayStart, rayDirection) { + var rayDirectionDotPlaneNormal = Vec3.dot(rayDirection, planeNormal); + if (rayDirectionDotPlaneNormal > 0.00001 || rayDirectionDotPlaneNormal < -0.00001) { + var rayStartDotPlaneNormal = Vec3.dot(Vec3.subtract(planePosition, rayStart), planeNormal); + var distance = rayStartDotPlaneNormal / rayDirectionDotPlaneNormal; + return {hit: true, distance: distance}; + } else { + // ray is parallel to the plane + return {hit: false, distance: 0}; + } +} + function stateToName(state) { return CONTROLLER_STATE_MACHINE[state] ? CONTROLLER_STATE_MACHINE[state].name : "???"; } @@ -965,7 +978,7 @@ function MyController(hand) { } else if (potentialEquipHotspot && Vec3.distance(this.lastHapticPulseLocation, currentLocation) > HAPTIC_TEXTURE_DISTANCE) { Controller.triggerHapticPulse(HAPTIC_TEXTURE_STRENGTH, HAPTIC_TEXTURE_DURATION, this.hand); this.lastHapticPulseLocation = currentLocation; - } + } this.prevPotentialEquipHotspot = potentialEquipHotspot; }; @@ -1243,6 +1256,10 @@ function MyController(hand) { } }; + this.searchEnter = function() { + this.capturedWebEntity = null; + }; + this.search = function(deltaTime, timestamp) { var _this = this; var name; @@ -1266,26 +1283,51 @@ function MyController(hand) { entityPropertiesCache.addEntity(rayPickInfo.entityID); } - // if the line probe hits a non-grabbable web entity or a web entity that is grabbed by the other hand. - // route simulated mouse events to that entity. - if (rayPickInfo.entityID && entityPropertiesCache.getProps(rayPickInfo.entityID).type === "Web" && - (!this.entityIsGrabbable(rayPickInfo.entityID) || this.getOtherHandController().grabbedEntity == rayPickInfo.entityID)) { + // route simulated touch events to a webEntity. + if (this.capturedWebEntity || + (rayPickInfo.entityID && entityPropertiesCache.getProps(rayPickInfo.entityID).type === "Web" && + (!this.entityIsGrabbable(rayPickInfo.entityID) || this.getOtherHandController().grabbedEntity == rayPickInfo.entityID))) { - if (Reticle.keyboardFocusEntity != rayPickInfo.entityID) { - Reticle.keyboardFocusEntity = rayPickInfo.entityID; + var standardControllerValue = (hand === RIGHT_HAND) ? Controller.Standard.RightHand : Controller.Standard.LeftHand; + var pose = Controller.getPoseValue(standardControllerValue); + var worldHandPosition = Vec3.sum(Vec3.multiplyQbyV(MyAvatar.orientation, pose.translation), MyAvatar.position); + var worldHandRotation = Quat.multiply(MyAvatar.orientation, pose.rotation); + + var focusedEntity = this.capturedWebEntity || rayPickInfo.entityID; + entityPropertiesCache.addEntity(focusedEntity); + + var props = entityPropertiesCache.getProps(focusedEntity); + var planePosition = props.position + var planeNormal = Vec3.multiplyQbyV(props.rotation, {x: 0, y: 0, z: 1.0}); + var rayStart = worldHandPosition; + var rayDirection = Quat.getUp(worldHandRotation); + var intersectionInfo = rayIntersectPlane(planePosition, planeNormal, rayStart, rayDirection); + + var intersectionPoint = planePosition; + if (intersectionInfo.hit && intersectionInfo.distance > 0) { + intersectionPoint = Vec3.sum(rayStart, Vec3.multiply(intersectionInfo.distance, rayDirection)); + } else { + intersectionPoint = planePosition; } - Reticle.sendEntityMouseMoveEvent(rayPickInfo.entityID, rayPickInfo.intersection); + + if (Reticle.keyboardFocusEntity != focusedEntity) { + Reticle.keyboardFocusEntity = focusedEntity; + } + + Reticle.sendEntityTouchUpdateEvent(focusedEntity, this.hand, intersectionPoint); + if (this.triggerSmoothedGrab() && !this.lastTriggerSmoothedGrab) { - print("AJT: mouse down"); - Reticle.sendEntityLeftMouseDownEvent(rayPickInfo.entityID, rayPickInfo.intersection); + Reticle.sendEntityTouchBeginEvent(focusedEntity, this.hand, intersectionPoint); + this.capturedWebEntity = focusedEntity; } if (!this.triggerSmoothedGrab() && this.lastTriggerSmoothedGrab) { - print("AJT: mouse up"); - Reticle.sendEntityLeftMouseUpEvent(rayPickInfo.entityID, rayPickInfo.intersection); + Reticle.sendEntityTouchEndEvent(focusedEntity, this.hand, intersectionPoint); + this.capturedWebEntity = null; } this.lastTriggerSmoothedGrab = this.triggerSmoothedGrab(); + equipHotspotBuddy.updateHotspots([], timestamp); - this.intersectionDistance = rayPickInfo.distance; + this.intersectionDistance = intersectionInfo.distance; } else { var candidateEntities = Entities.findEntities(handPosition, NEAR_GRAB_RADIUS);