Merge pull request #8894 from ZappoMan/xboxLasers

Adds support for "lasers" to the Oculus+xbox controller

interface/resources/controllers/xbox.json

@@ -3,6 +3,11 @@
     "channels": [
         { "from": "GamePad.LY", "filters": { "type": "deadZone", "min": 0.05 }, "to": "Actions.TranslateZ" },
         { "from": "GamePad.LX", "filters": { "type": "deadZone", "min": 0.05 }, "to": "Actions.TranslateX" },
+
+        { "from": "GamePad.LT", "to": "Standard.LTClick",  
+            "peek": true,
+            "filters": [ { "type": "hysteresis", "min": 0.85, "max": 0.9 } ]
+        },
         { "from": "GamePad.LT", "to": "Standard.LT" }, 
         { "from": "GamePad.LB", "to": "Standard.LB" }, 
         { "from": "GamePad.LS", "to": "Standard.LS" },
@@ -31,6 +36,10 @@
             ]
         }, 
 
+        { "from": "GamePad.RT", "to": "Standard.RTClick",  
+            "peek": true,
+            "filters": [ { "type": "hysteresis", "min": 0.85, "max": 0.9 } ]
+        },
         { "from": "GamePad.RT", "to": "Standard.RT" }, 
         { "from": "GamePad.RB", "to": "Standard.RB" }, 
         { "from": "GamePad.RS", "to": "Standard.RS" },

interface/resources/shaders/hmd_ui_glow.frag

@@ -13,6 +13,9 @@ struct OverlayData {
     vec4 glowPoints;
     vec4 glowColors[2];
     vec4 resolutionRadiusAlpha;
+
+    vec4 extraGlowColor;
+    vec2 extraGlowPoint;
 };
 
 layout(std140) uniform overlayBuffer {
@@ -25,6 +28,9 @@ float alpha = overlay.resolutionRadiusAlpha.w;
 vec4 glowPoints = overlay.glowPoints;
 vec4 glowColors[2] = overlay.glowColors;
 
+vec2 extraGlowPoint = overlay.extraGlowPoint;
+vec4 extraGlowColor = overlay.extraGlowColor;
+
 in vec3 vPosition;
 in vec2 vTexCoord;
 
@@ -48,11 +54,16 @@ void main() {
     float glowIntensity = 0.0;
     float dist1 = distance(vTexCoord * aspect, glowPoints.xy * aspect);
     float dist2 = distance(vTexCoord * aspect, glowPoints.zw * aspect);
-    float dist = min(dist1, dist2);
+    float dist3 = distance(vTexCoord * aspect, extraGlowPoint * aspect);
+    float distX = min(dist1, dist2);
+    float dist = min(distX, dist3);
     vec3 glowColor = glowColors[0].rgb;
     if (dist2 < dist1) {
         glowColor = glowColors[1].rgb;
     }
+    if (dist3 < dist2) {
+        glowColor = extraGlowColor.rgb;
+    }
 
     if (dist <= radius) {
        glowIntensity = 1.0 - (dist / radius);

interface/resources/shaders/hmd_ui_glow.vert

@@ -11,6 +11,9 @@ struct OverlayData {
     vec4 glowPoints;
     vec4 glowColors[2];
     vec4 resolutionRadiusAlpha;
+
+    vec4 extraGlowColor;
+    vec2 extraGlowPoint;
 };
 
 layout(std140) uniform overlayBuffer {

interface/src/scripting/HMDScriptingInterface.cpp

@@ -144,6 +144,27 @@ bool HMDScriptingInterface::setHandLasers(int hands, bool enabled, const glm::ve
         color, direction);
 }
 
+bool HMDScriptingInterface::setExtraLaser(const glm::vec3& worldStart, bool enabled, const glm::vec4& color, const glm::vec3& direction) const {
+    auto offscreenUi = DependencyManager::get<OffscreenUi>();
+    offscreenUi->executeOnUiThread([offscreenUi, enabled] {
+        offscreenUi->getDesktop()->setProperty("hmdHandMouseActive", enabled);
+    });
+
+
+    auto myAvatar = DependencyManager::get<AvatarManager>()->getMyAvatar();
+    auto sensorToWorld = myAvatar->getSensorToWorldMatrix();
+    auto worldToSensor = glm::inverse(sensorToWorld);
+    auto sensorStart = ::transformPoint(worldToSensor, worldStart);
+    auto sensorDirection = ::transformVectorFast(worldToSensor, direction);
+
+    return qApp->getActiveDisplayPlugin()->setExtraLaser(enabled ? DisplayPlugin::HandLaserMode::Overlay : DisplayPlugin::HandLaserMode::None,
+        color, sensorStart, sensorDirection);
+}
+
+void HMDScriptingInterface::disableExtraLaser() const {
+    setExtraLaser(vec3(0), false, vec4(0), vec3(0));
+}
+
 void HMDScriptingInterface::disableHandLasers(int hands) const {
     setHandLasers(hands, false, vec4(0), vec3(0));
 }

interface/src/scripting/HMDScriptingInterface.h

@@ -46,8 +46,12 @@ public:
     Q_INVOKABLE bool shouldShowHandControllers() const;
 
     Q_INVOKABLE bool setHandLasers(int hands, bool enabled, const glm::vec4& color, const glm::vec3& direction) const;
-
     Q_INVOKABLE void disableHandLasers(int hands) const;
+
+    Q_INVOKABLE bool setExtraLaser(const glm::vec3& worldStart, bool enabled, const glm::vec4& color, const glm::vec3& direction) const;
+    Q_INVOKABLE void disableExtraLaser() const;
+
+
     /// Suppress the activation of any on-screen keyboard so that a script operation will 
     /// not be interrupted by a keyboard popup
     /// Returns false if there is already an active keyboard displayed.

libraries/display-plugins/src/display-plugins/CompositorHelper.h

@@ -111,6 +111,8 @@ public:
     void setDisplayPlugin(const DisplayPluginPointer& displayPlugin) { _currentDisplayPlugin = displayPlugin; }
     void setFrameInfo(uint32_t frame, const glm::mat4& camera) { _currentCamera = camera; }
 
+    float getHmdUiRadius() const { return _hmdUIRadius; }
+
 signals:
     void allowMouseCaptureChanged();
     void alphaChanged();

libraries/display-plugins/src/display-plugins/hmd/HmdDisplayPlugin.cpp

@@ -116,6 +116,7 @@ void HmdDisplayPlugin::customizeContext() {
     for (size_t i = 0; i < _geometryIds.size(); ++i) {
         _geometryIds[i] = geometryCache->allocateID();
     }
+    _extraLaserID = geometryCache->allocateID();
 }
 
 void HmdDisplayPlugin::uncustomizeContext() {
@@ -135,6 +136,7 @@ void HmdDisplayPlugin::uncustomizeContext() {
     for (size_t i = 0; i < _geometryIds.size(); ++i) {
         geometryCache->releaseID(_geometryIds[i]);
     }
+    geometryCache->releaseID(_extraLaserID);
     Parent::uncustomizeContext();
 }
 
@@ -359,11 +361,18 @@ void HmdDisplayPlugin::updateFrameData() {
         _presentHandLasers = _handLasers;
         _presentHandPoses = _handPoses;
         _presentUiModelTransform = _uiModelTransform;
+
+        _presentExtraLaser = _extraLaser;
+        _presentExtraLaserStart = _extraLaserStart;
     });
 
     auto compositorHelper = DependencyManager::get<CompositorHelper>();
     glm::mat4 modelMat = compositorHelper->getModelTransform().getMatrix();
-    std::array<vec2, NUMBER_OF_HANDS> handGlowPoints{ { vec2(-1), vec2(-1) } };
+    static const float OUT_OF_BOUNDS = -1;
+    std::array<vec2, NUMBER_OF_HANDS> handGlowPoints { { vec2(OUT_OF_BOUNDS), vec2(OUT_OF_BOUNDS) } };
+    vec2 extraGlowPoint(OUT_OF_BOUNDS);
+
+    float uiRadius = compositorHelper->getHmdUiRadius();
 
     // compute the glow point interesections
     for (size_t i = 0; i < NUMBER_OF_HANDS; ++i) {
@@ -390,9 +399,6 @@ void HmdDisplayPlugin::updateFrameData() {
         }
         castStart += glm::quat_cast(model) * grabPointOffset;
 
-        // FIXME fetch the actual UI radius from... somewhere?
-        float uiRadius = 1.0f;
-
         // Find the intersection of the laser with he UI and use it to scale the model matrix
         float distance;
         if (!glm::intersectRaySphere(castStart, castDirection,
@@ -425,6 +431,42 @@ void HmdDisplayPlugin::updateFrameData() {
         handGlowPoints[i] = yawPitch;
     }
 
+    // compute the glow point interesections
+    if (_presentExtraLaser.valid()) {
+        const vec3& laserDirection = _presentExtraLaser.direction;
+        vec3 castStart = _presentExtraLaserStart;
+        vec3 castDirection = laserDirection;
+
+        // Find the intersection of the laser with he UI and use it to scale the model matrix
+        float distance;
+        if (glm::intersectRaySphere(castStart, castDirection,
+            _presentUiModelTransform.getTranslation(), uiRadius * uiRadius, distance)) {
+
+
+            _presentExtraLaserPoints.first = castStart;
+            _presentExtraLaserPoints.second = _presentExtraLaserPoints.first + (castDirection * distance);
+
+            vec3 intersectionPosition = castStart + (castDirection * distance) - _presentUiModelTransform.getTranslation();
+            intersectionPosition = glm::inverse(_presentUiModelTransform.getRotation()) * intersectionPosition;
+
+            // Take the interesection normal and convert it to a texture coordinate
+            vec2 yawPitch;
+            {
+                vec2 xdir = glm::normalize(vec2(intersectionPosition.x, -intersectionPosition.z));
+                yawPitch.x = glm::atan(xdir.x, xdir.y);
+                yawPitch.y = (acosf(intersectionPosition.y) * -1.0f) + (float)M_PI_2;
+            }
+            vec2 halfFov = CompositorHelper::VIRTUAL_UI_TARGET_FOV / 2.0f;
+
+            // Are we out of range
+            if (!glm::any(glm::greaterThan(glm::abs(yawPitch), halfFov))) {
+                yawPitch /= CompositorHelper::VIRTUAL_UI_TARGET_FOV;
+                yawPitch += 0.5f;
+                extraGlowPoint = yawPitch;
+            }
+        }
+    }
+
 
     for_each_eye([&](Eye eye) {
         auto modelView = glm::inverse(_currentPresentFrameInfo.presentPose * getEyeToHeadTransform(eye)) * modelMat;
@@ -438,6 +480,8 @@ void HmdDisplayPlugin::updateFrameData() {
         uniforms.glowPoints = vec4(handGlowPoints[0], handGlowPoints[1]);
         uniforms.glowColors[0] = _presentHandLasers[0].color;
         uniforms.glowColors[1] = _presentHandLasers[1].color;
+        uniforms.extraGlowPoint = extraGlowPoint;
+        uniforms.extraGlowColor = _presentExtraLaser.color;
     }
 }
 
@@ -613,13 +657,29 @@ bool HmdDisplayPlugin::setHandLaser(uint32_t hands, HandLaserMode mode, const ve
     return true;
 }
 
+bool HmdDisplayPlugin::setExtraLaser(HandLaserMode mode, const vec4& color, const glm::vec3& sensorSpaceStart, const vec3& sensorSpaceDirection) {
+    HandLaserInfo info;
+    info.mode = mode;
+    info.color = color;
+    info.direction = sensorSpaceDirection;
+    withNonPresentThreadLock([&] {
+        _extraLaser = info;
+        _extraLaserStart = sensorSpaceStart;
+    });
+
+    // FIXME defer to a child class plugin to determine if hand lasers are actually 
+    // available based on the presence or absence of hand controllers
+    return true;
+}
+
+
 void HmdDisplayPlugin::compositeExtra() {
     // If neither hand laser is activated, exit
-    if (!_presentHandLasers[0].valid() && !_presentHandLasers[1].valid()) {
+    if (!_presentHandLasers[0].valid() && !_presentHandLasers[1].valid() && !_presentExtraLaser.valid()) {
         return;
     }
 
-    if (_presentHandPoses[0] == IDENTITY_MATRIX && _presentHandPoses[1] == IDENTITY_MATRIX) {
+    if (_presentHandPoses[0] == IDENTITY_MATRIX && _presentHandPoses[1] == IDENTITY_MATRIX && !_presentExtraLaser.valid()) {
         return;
     }
 
@@ -639,6 +699,11 @@ void HmdDisplayPlugin::compositeExtra() {
                 geometryCache->renderGlowLine(batch, points.first, points.second, laser.color, _geometryIds[index]);
             }
         });
+
+        if (_presentExtraLaser.valid()) {
+            const auto& points = _presentExtraLaserPoints;
+            geometryCache->renderGlowLine(batch, points.first, points.second, _presentExtraLaser.color, _extraLaserID);
+        }
     });
 }
 

libraries/display-plugins/src/display-plugins/hmd/HmdDisplayPlugin.h

@@ -38,6 +38,7 @@ public:
     virtual glm::mat4 getHeadPose() const override;
 
     bool setHandLaser(uint32_t hands, HandLaserMode mode, const vec4& color, const vec3& direction) override;
+    bool setExtraLaser(HandLaserMode mode, const vec4& color, const glm::vec3& sensorSpaceStart, const vec3& sensorSpaceDirection) override;
 
     bool wantVsync() const override {
         return false;
@@ -78,8 +79,16 @@ protected:
     Transform _presentUiModelTransform;
     std::array<HandLaserInfo, 2> _presentHandLasers;
     std::array<int, 2> _geometryIds;
+    int _extraLaserID;
     std::array<mat4, 2> _presentHandPoses;
     std::array<std::pair<vec3, vec3>, 2> _presentHandLaserPoints;
+
+    HandLaserInfo _extraLaser;
+    HandLaserInfo _presentExtraLaser;
+    vec3 _extraLaserStart;
+    vec3 _presentExtraLaserStart;
+    std::pair<vec3, vec3> _presentExtraLaserPoints;
+
     std::array<mat4, 2> _eyeOffsets;
     std::array<mat4, 2> _eyeProjections;
     std::array<mat4, 2> _eyeInverseProjections;
@@ -130,6 +139,9 @@ private:
             vec2 resolution { CompositorHelper::VIRTUAL_SCREEN_SIZE };
             float radius { 0.005f };
             float alpha { 1.0f };
+
+            vec4 extraGlowColor;
+            vec2 extraGlowPoint { -1 };
         } uniforms;
         
         struct Vertex {

libraries/display-plugins/src/hmd_ui_glow.slf

@@ -1,75 +0,0 @@
-//
-//  Created by Bradley Austin Davis on 2016/07/11
-//  Copyright 2013-2016 High Fidelity, Inc.
-//
-//  Distributed under the Apache License, Version 2.0.
-//  See the accompanying file LICENSE or http://www.apache.org/licenses/LICENSE-2.0.html
-//
-
-uniform sampler2D sampler;
-
-struct OverlayData {
-    mat4 mvp;
-    vec4 glowPoints;
-    vec4 glowColors[2];
-    vec4 resolutionRadiusAlpha;
-};
-
-layout(std140) uniform overlayBuffer {
-    OverlayData overlay;
-};
-
-vec2 resolution = overlay.resolutionRadiusAlpha.xy;
-float radius = overlay.resolutionRadiusAlpha.z;
-float alpha = overlay.resolutionRadiusAlpha.w;
-vec4 glowPoints = overlay.glowPoints;
-vec4 glowColors[2] = overlay.glowColors;
-
-in vec3 vPosition;
-in vec2 vTexCoord;
-
-out vec4 FragColor;
-
-float easeInOutCubic(float f) {
-    const float d = 1.0;
-    const float b = 0.0;
-    const float c = 1.0;
-    float t = f;
-    if ((t /= d / 2.0) < 1.0) return c / 2.0 * t * t * t + b;
-    return c / 2.0 * ((t -= 2.0) * t * t + 2.0) + b;
-}
-
-void main() {
-    FragColor = texture(sampler, vTexCoord);
-
-    vec2 aspect = resolution;
-    aspect /= resolution.x;
-
-    float glowIntensity = 0.0;
-    float dist1 = distance(vTexCoord * aspect, glowPoints.xy * aspect);
-    float dist2 = distance(vTexCoord * aspect, glowPoints.zw * aspect);
-    float dist = min(dist1, dist2);
-    vec3 glowColor = glowColors[0].rgb;
-    if (dist2 < dist1) {
-        glowColor = glowColors[1].rgb;
-    }
-
-    if (dist <= radius) {
-       glowIntensity = 1.0 - (dist / radius);
-       glowColor.rgb = pow(glowColor, vec3(1.0 - glowIntensity));
-       glowIntensity = easeInOutCubic(glowIntensity);
-       glowIntensity = pow(glowIntensity, 0.5);
-    }
-
-    if (alpha <= 0.0) {
-        if (glowIntensity <= 0.0) {
-            discard;
-        }
-
-        FragColor = vec4(glowColor, glowIntensity);
-        return;
-    }
-
-    FragColor.rgb = mix(FragColor.rgb, glowColor.rgb, glowIntensity);
-    FragColor.a *= alpha;
-}

libraries/display-plugins/src/hmd_ui_glow.slv

@@ -1,32 +0,0 @@
-//
-//  Created by Bradley Austin Davis on 2016/07/11
-//  Copyright 2013-2016 High Fidelity, Inc.
-//
-//  Distributed under the Apache License, Version 2.0.
-//  See the accompanying file LICENSE or http://www.apache.org/licenses/LICENSE-2.0.html
-//
-
-struct OverlayData {
-    mat4 mvp;
-    vec4 glowPoints;
-    vec4 glowColors[2];
-    vec4 resolutionRadiusAlpha;
-};
-
-layout(std140) uniform overlayBuffer {
-    OverlayData overlay;
-};
-
-mat4 mvp = overlay.mvp;
-
-layout(location = 0) in vec3 Position;
-layout(location = 3) in vec2 TexCoord;
-
-out vec3 vPosition;
-out vec2 vTexCoord;
-
-void main() {
-  gl_Position = mvp * vec4(Position, 1);
-  vTexCoord = TexCoord;
-  vPosition = Position;
-}

libraries/plugins/src/plugins/DisplayPlugin.h

@@ -115,6 +115,10 @@ public:
         return false;
     }
 
+    virtual bool setExtraLaser(HandLaserMode mode, const vec4& color, const glm::vec3& sensorSpaceStart, const vec3& sensorSpaceDirection) {
+        return false;
+    }
+
     virtual bool suppressKeyboard() { return false;  }
     virtual void unsuppressKeyboard() {};
     virtual bool isKeyboardVisible() { return false; }

scripts/system/controllers/handControllerPointer.js

@@ -204,7 +204,7 @@ function overlayFromWorldPoint(point) {
 }
 
 function activeHudPoint2d(activeHand) { // if controller is valid, update reticle position and answer 2d point. Otherwise falsey.
-    var controllerPose = getControllerWorldLocation(activeHand, true);
+    var controllerPose = getControllerWorldLocation(activeHand, true); // note: this will return head pose if hand pose is invalid (third eye)
     if (!controllerPose.valid) {
         return; // Controller is cradled.
     }
@@ -447,12 +447,20 @@ function clearSystemLaser() {
         return;
     }
     HMD.disableHandLasers(BOTH_HUD_LASERS);
+    HMD.disableExtraLaser();
     systemLaserOn = false;
     weMovedReticle = true;
     Reticle.position = { x: -1, y: -1 };
 }
 function setColoredLaser() { // answer trigger state if lasers supported, else falsey.
     var color = (activeTrigger.state === 'full') ? LASER_TRIGGER_COLOR_XYZW : LASER_SEARCH_COLOR_XYZW;
+
+    if (!HMD.isHandControllerAvailable()) {
+        var position = MyAvatar.getHeadPosition();
+        var direction = Quat.getUp(Quat.multiply(MyAvatar.headOrientation, Quat.angleAxis(-90, { x: 1, y: 0, z: 0 })));
+        return HMD.setExtraLaser(position, true, color, direction);
+    }
+
     return HMD.setHandLasers(activeHudLaser, true, color, SYSTEM_LASER_DIRECTION) && activeTrigger.state;
 }
 
@@ -491,11 +499,21 @@ function update() {
     if (!hudPoint2d) {
         return off();
     }
+
+
     // If there's a HUD element at the (newly moved) reticle, just make it visible and bail.
     if (isPointingAtOverlay(hudPoint2d)) {
         if (HMD.active) {
             Reticle.depth = hudReticleDistance();
+
+            if (!HMD.isHandControllerAvailable()) {
+                var color = (activeTrigger.state === 'full') ? LASER_TRIGGER_COLOR_XYZW : LASER_SEARCH_COLOR_XYZW;
+                var position = MyAvatar.getHeadPosition();
+                var direction = Quat.getUp(Quat.multiply(MyAvatar.headOrientation, Quat.angleAxis(-90, { x: 1, y: 0, z: 0 })));
+                HMD.setExtraLaser(position, true, color, direction);
+            }
         }
+
         if (activeTrigger.state && (!systemLaserOn || (systemLaserOn !== activeTrigger.state))) { // last=>wrong color
             // If the active plugin doesn't implement hand lasers, show the mouse reticle instead.
             systemLaserOn = setColoredLaser();

scripts/system/libraries/controllers.js

@@ -34,6 +34,7 @@ getControllerWorldLocation = function (handController, doOffset) {
     var orientation;
     var position;
     var pose = Controller.getPoseValue(handController);
+    var valid = pose.valid;
     if (pose.valid) {
         orientation = Quat.multiply(MyAvatar.orientation, pose.rotation);
         position = Vec3.sum(Vec3.multiplyQbyV(MyAvatar.orientation, pose.translation), MyAvatar.position);
@@ -41,10 +42,15 @@ getControllerWorldLocation = function (handController, doOffset) {
         if (doOffset) {
             position = Vec3.sum(position, Vec3.multiplyQbyV(orientation, getGrabPointSphereOffset(handController)));
         }
+    } else if (!HMD.isHandControllerAvailable()) {
+        position = MyAvatar.getHeadPosition();
+        orientation = Quat.multiply(MyAvatar.headOrientation, Quat.angleAxis(-90, { x: 1, y: 0, z: 0 }));
+        valid = true;
     }
+
     return {position: position,
             translation: position,
             orientation: orientation,
             rotation: orientation,
-            valid: pose.valid};
+            valid: valid};
 };