Hand laser rendering support in HMD plugins.

interface/src/scripting/HMDScriptingInterface.cpp

@@ -105,3 +105,13 @@ QString HMDScriptingInterface::preferredAudioInput() const {
 QString HMDScriptingInterface::preferredAudioOutput() const {
     return qApp->getActiveDisplayPlugin()->getPreferredAudioOutDevice();
 }
+
+bool HMDScriptingInterface::setHandLasers(int hands, bool enabled, const glm::vec4& color, const glm::vec3& direction) const {
+    return qApp->getActiveDisplayPlugin()->setHandLaser(hands,
+        enabled ? DisplayPlugin::HandLaserMode::Overlay : DisplayPlugin::HandLaserMode::None,
+        color, direction);
+}
+
+void HMDScriptingInterface::disableHandLasers(int hands) const {
+    qApp->getActiveDisplayPlugin()->setHandLaser(hands, DisplayPlugin::HandLaserMode::None);
+}

interface/src/scripting/HMDScriptingInterface.h

@@ -36,6 +36,8 @@ public:
     Q_INVOKABLE glm::vec2 overlayToSpherical(const glm::vec2 & overlayPos) const;
     Q_INVOKABLE QString preferredAudioInput() const;
     Q_INVOKABLE QString preferredAudioOutput() const;
+    Q_INVOKABLE bool setHandLasers(int hands, bool enabled, const glm::vec4& color, const glm::vec3& direction) const;
+    Q_INVOKABLE void disableHandLasers(int hands) const;
 
 public:
     HMDScriptingInterface();

libraries/display-plugins/src/display-plugins/OpenGLDisplayPlugin.cpp

@@ -213,9 +213,10 @@ OpenGLDisplayPlugin::OpenGLDisplayPlugin() {
 }
 
 void OpenGLDisplayPlugin::cleanupForSceneTexture(const gpu::TexturePointer& sceneTexture) {
-    Lock lock(_mutex);
-    Q_ASSERT(_sceneTextureToFrameIndexMap.contains(sceneTexture));
-    _sceneTextureToFrameIndexMap.remove(sceneTexture);
+    withRenderThreadLock([&] {
+        Q_ASSERT(_sceneTextureToFrameIndexMap.contains(sceneTexture));
+        _sceneTextureToFrameIndexMap.remove(sceneTexture);
+    });
 }
 
 
@@ -394,10 +395,9 @@ void OpenGLDisplayPlugin::submitSceneTexture(uint32_t frameIndex, const gpu::Tex
         return;
     }
 
-    {
-        Lock lock(_mutex);
+    withRenderThreadLock([&] {
         _sceneTextureToFrameIndexMap[sceneTexture] = frameIndex;
-    }
+    });
 
     // Submit it to the presentation thread via escrow
     _sceneTextureEscrow.submit(sceneTexture);
@@ -431,11 +431,12 @@ void OpenGLDisplayPlugin::updateTextures() {
 }
 
 void OpenGLDisplayPlugin::updateFrameData() {
-    Lock lock(_mutex);
-    auto previousFrameIndex = _currentPresentFrameIndex;
-    _currentPresentFrameIndex = _sceneTextureToFrameIndexMap[_currentSceneTexture];
-    auto skippedCount = (_currentPresentFrameIndex - previousFrameIndex) - 1;
-    _droppedFrameRate.increment(skippedCount);
+    withPresentThreadLock([&] {
+        auto previousFrameIndex = _currentPresentFrameIndex;
+        _currentPresentFrameIndex = _sceneTextureToFrameIndexMap[_currentSceneTexture];
+        auto skippedCount = (_currentPresentFrameIndex - previousFrameIndex) - 1;
+        _droppedFrameRate.increment(skippedCount);
+    });
 }
 
 void OpenGLDisplayPlugin::compositeOverlay() {
@@ -492,14 +493,14 @@ void OpenGLDisplayPlugin::compositeLayers() {
     }
     _compositeFramebuffer->Bound(Framebuffer::Target::Draw, [&] {
         Context::Viewport(targetRenderSize.x, targetRenderSize.y);
-        Context::Clear().DepthBuffer();
-        glBindTexture(GL_TEXTURE_2D, getSceneTextureId());
-        compositeScene();
+        auto sceneTextureId = getSceneTextureId();
         auto overlayTextureId = getOverlayTextureId();
+        glBindTexture(GL_TEXTURE_2D, sceneTextureId);
+        compositeScene();
         if (overlayTextureId) {
-            glEnable(GL_BLEND);
-            glBlendFunc(GL_SRC_ALPHA, GL_ONE_MINUS_SRC_ALPHA);
             glBindTexture(GL_TEXTURE_2D, overlayTextureId);
+            Context::Enable(Capability::Blend);
+            Context::BlendFunc(BlendFunction::SrcAlpha, BlendFunction::OneMinusSrcAlpha);
             compositeOverlay();
 
             auto compositorHelper = DependencyManager::get<CompositorHelper>();
@@ -507,10 +508,14 @@ void OpenGLDisplayPlugin::compositeLayers() {
                 auto& cursorManager = Cursor::Manager::instance();
                 const auto& cursorData = _cursorsData[cursorManager.getCursor()->getIcon()];
                 glBindTexture(GL_TEXTURE_2D, cursorData.texture);
+                glActiveTexture(GL_TEXTURE1);
+                glBindTexture(GL_TEXTURE_2D, overlayTextureId);
                 compositePointer();
+                glBindTexture(GL_TEXTURE_2D, 0);
+                glActiveTexture(GL_TEXTURE0);
             }
             glBindTexture(GL_TEXTURE_2D, 0);
-            glDisable(GL_BLEND);
+            Context::Disable(Capability::Blend);
         }
     });
 }
@@ -549,7 +554,11 @@ float OpenGLDisplayPlugin::newFramePresentRate() const {
 }
 
 float OpenGLDisplayPlugin::droppedFrameRate() const {
-    return _droppedFrameRate.rate();
+    float result;
+    withRenderThreadLock([&] {
+        result = _droppedFrameRate.rate();
+    });
+    return result;
 }
 
 float OpenGLDisplayPlugin::presentRate() const {
@@ -664,3 +673,11 @@ void OpenGLDisplayPlugin::useProgram(const ProgramPtr& program) {
         _activeProgram = program;
     }
 }
+
+void OpenGLDisplayPlugin::assertIsRenderThread() const {
+    Q_ASSERT(QThread::currentThread() != _presentThread);
+}
+
+void OpenGLDisplayPlugin::assertIsPresentThread() const {
+    Q_ASSERT(QThread::currentThread() == _presentThread);
+}

libraries/display-plugins/src/display-plugins/OpenGLDisplayPlugin.h

@@ -109,7 +109,6 @@ protected:
     int32_t _alphaUniform { -1 };
     ShapeWrapperPtr _plane;
 
-    mutable Mutex _mutex;
     RateCounter<> _droppedFrameRate;
     RateCounter<> _newFrameRate;
     RateCounter<> _presentRate;
@@ -135,7 +134,27 @@ protected:
     BasicFramebufferWrapperPtr _compositeFramebuffer;
     bool _lockCurrentTexture { false };
 
+    void assertIsRenderThread() const;
+    void assertIsPresentThread() const;
+
+    template<typename F>
+    void withPresentThreadLock(F f) const {
+        assertIsPresentThread();
+        Lock lock(_presentMutex);
+        f();
+    }
+
+    template<typename F>
+    void withRenderThreadLock(F f) const {
+        assertIsRenderThread();
+        Lock lock(_presentMutex);
+        f();
+    }
+
 private:
+    // Any resource shared by the main thread and the presntaion thread must 
+    // be serialized through this mutex
+    mutable Mutex _presentMutex;
     ProgramPtr _activeProgram;
 };
 

libraries/display-plugins/src/display-plugins/hmd/HmdDisplayPlugin.cpp

@@ -9,6 +9,7 @@
 
 #include <memory>
 #include <glm/gtc/matrix_transform.hpp>
+#include <glm/gtx/intersect.hpp>
 
 #include <QtCore/QLoggingCategory>
 #include <QtWidgets/QApplication>
@@ -37,7 +38,6 @@ QRect HmdDisplayPlugin::getRecommendedOverlayRect() const {
     return CompositorHelper::VIRTUAL_SCREEN_RECOMMENDED_OVERLAY_RECT;
 }
 
-
 bool HmdDisplayPlugin::internalActivate() {
     _monoPreview = _container->getBoolSetting("monoPreview", DEFAULT_MONO_VIEW);
 
@@ -197,14 +197,43 @@ static ProgramPtr getReprojectionProgram() {
 #endif
 
 
+static const char * LASER_VS = R"VS(#version 410 core
+uniform mat4 mvp = mat4(1);
+
+in vec3 Position;
+
+out vec3 vPosition;
+
+void main() {
+  gl_Position = mvp * vec4(Position, 1);
+  vPosition = Position;
+}
+
+)VS";
+
+static const char * LASER_FS = R"FS(#version 410 core
+
+uniform vec4 color = vec4(1.0, 1.0, 1.0, 1.0);
+in vec3 vPosition;
+
+out vec4 FragColor;
+
+void main() {
+    FragColor = color;
+}
+
+)FS";
+
 void HmdDisplayPlugin::customizeContext() {
     Parent::customizeContext();
     // Only enable mirroring if we know vsync is disabled
     enableVsync(false);
     _enablePreview = !isVsyncEnabled();
     _sphereSection = loadSphereSection(_program, CompositorHelper::VIRTUAL_UI_TARGET_FOV.y, CompositorHelper::VIRTUAL_UI_ASPECT_RATIO);
+    compileProgram(_laserProgram, LASER_VS, LASER_FS);
+    _laserGeometry = loadLaser(_laserProgram);
     compileProgram(_reprojectionProgram, REPROJECTION_VS, REPROJECTION_FS);
-    
+
     using namespace oglplus;
     REPROJECTION_MATRIX_LOCATION = Uniform<glm::mat3>(*_reprojectionProgram, "reprojection").Location();
     INVERSE_PROJECTION_MATRIX_LOCATION = Uniform<glm::mat4>(*_reprojectionProgram, "inverseProjections").Location();
@@ -215,6 +244,8 @@ void HmdDisplayPlugin::uncustomizeContext() {
     _sphereSection.reset();
     _compositeFramebuffer.reset();
     _reprojectionProgram.reset();
+    _laserProgram.reset();
+    _laserGeometry.reset();
     Parent::uncustomizeContext();
 }
 
@@ -285,6 +316,8 @@ void HmdDisplayPlugin::compositePointer() {
         Uniform<glm::mat4>(*_program, _mvpUniform).Set(mvp);
         _plane->Draw();
     });
+
+    compositeLasers();
 }
 
 void HmdDisplayPlugin::internalPresent() {
@@ -343,22 +376,122 @@ void HmdDisplayPlugin::setEyeRenderPose(uint32_t frameIndex, Eye eye, const glm:
 
 void HmdDisplayPlugin::updateFrameData() {
     // Check if we have old frame data to discard
-    {
-        Lock lock(_mutex);
+    withPresentThreadLock([&] {
         auto itr = _frameInfos.find(_currentPresentFrameIndex);
         if (itr != _frameInfos.end()) {
             _frameInfos.erase(itr);
         }
-    }
+    });
 
     Parent::updateFrameData();
 
-    {
-        Lock lock(_mutex);
+    withPresentThreadLock([&] {
         _currentPresentFrameInfo = _frameInfos[_currentPresentFrameIndex];
-    }
+    });
 }
 
 glm::mat4 HmdDisplayPlugin::getHeadPose() const {
     return _currentRenderFrameInfo.renderPose;
 }
+
+bool HmdDisplayPlugin::setHandLaser(uint32_t hands, HandLaserMode mode, const vec4& color, const vec3& direction) {
+    HandLaserInfo info;
+    info.mode = mode;
+    info.color = color;
+    info.direction = direction;
+    withRenderThreadLock([&] {
+        if (hands & Hand::LeftHand) {
+            _handLasers[0] = info;
+        }
+        if (hands & Hand::RightHand) {
+            _handLasers[1] = info;
+        }
+    });
+    // FIXME defer to a child class plugin to determine if hand lasers are actually 
+    return true;
+}
+
+static float calculateRayUiCollisionDistance(const glm::mat4& headPose, const glm::vec3& position, const glm::vec3& direction) {
+    auto relativePosition4 = glm::inverse(headPose) * vec4(position, 1);
+    auto relativePosition = vec3(relativePosition4) / relativePosition4.w;
+    auto relativeDirection = glm::inverse(glm::quat_cast(headPose)) * direction;
+    if (glm::abs(glm::length2(relativeDirection) - 1.0f) > EPSILON) {
+        relativeDirection = glm::normalize(relativeDirection);
+    }
+    float uiRadius = 1.0f;
+    float instersectionDistance;
+    if (!glm::intersectRaySphere(relativePosition, relativeDirection, vec3(0), uiRadius * uiRadius, instersectionDistance)) {
+        return -1;
+    }
+    return instersectionDistance;
+}
+
+void HmdDisplayPlugin::compositeLasers() {
+    std::array<HandLaserInfo, 2> handLasers;
+    std::array<mat4, 2> renderHandPoses;
+    withPresentThreadLock([&] {
+        handLasers = _handLasers;
+        renderHandPoses = _handPoses;
+    });
+
+    // If neither hand laser is activated, exit
+    if (!handLasers[0].valid() && !handLasers[1].valid()) {
+        return;
+    }
+
+    static const glm::mat4 identity;
+    if (renderHandPoses[0] == identity && renderHandPoses[1] == identity) {
+        return;
+    }
+
+    // Render hand lasers
+    using namespace oglplus;
+    useProgram(_laserProgram);
+    _laserGeometry->Use();
+    std::array<mat4, 2> handLaserModelMatrices;
+
+    for (int i = 0; i < 2; ++i) {
+        if (renderHandPoses[i] == identity) {
+            continue;
+        }
+        const auto& handLaser = handLasers[i];
+        if (!handLaser.valid()) {
+            continue;
+        }
+
+        const auto& laserDirection = handLaser.direction;
+        auto model = renderHandPoses[i];
+        auto castDirection = glm::quat_cast(model) * laserDirection;
+
+        // Find the intersection of the laser with he UI and use it to scale the model matrix
+        float distance = calculateRayUiCollisionDistance(_currentPresentFrameInfo.presentPose, vec3(renderHandPoses[i][3]), castDirection);
+        if (distance < 0) {
+            continue;
+        }
+
+        // Make sure we rotate to match the desired laser direction
+        if (laserDirection != Vectors::UNIT_NEG_Z) {
+            auto rotation = glm::rotation(Vectors::UNIT_NEG_Z, laserDirection);
+            model = model * glm::mat4_cast(rotation);
+        }
+
+        model = glm::scale(model, vec3(distance));
+        handLaserModelMatrices[i] = model;
+    }
+
+    for_each_eye([&](Eye eye) {
+        eyeViewport(eye);
+        auto eyePose = _currentPresentFrameInfo.presentPose * getEyeToHeadTransform(eye);
+        auto view = glm::inverse(eyePose);
+        const auto& projection = _eyeProjections[eye];
+        for (int i = 0; i < 2; ++i) {
+            if (handLaserModelMatrices[i] == identity) {
+                continue;
+            }
+            Uniform<glm::mat4>(*_laserProgram, "mvp").Set(projection * view * handLaserModelMatrices[i]);
+            Uniform<glm::vec4>(*_laserProgram, "color").Set(handLasers[i].color);
+            _laserGeometry->Draw();
+            // TODO render some kind of visual indicator at the intersection point with the UI.
+        }
+    });
+}

libraries/display-plugins/src/display-plugins/hmd/HmdDisplayPlugin.h

@@ -30,7 +30,7 @@ public:
 
     virtual glm::mat4 getHeadPose() const override;
 
-
+    bool setHandLaser(uint32_t hands, HandLaserMode mode, const vec4& color, const vec3& direction) override;
 
 protected:
     virtual void hmdPresent() = 0;
@@ -47,6 +47,22 @@ protected:
     void uncustomizeContext() override;
     void updateFrameData() override;
 
+    void compositeLasers();
+
+
+    struct HandLaserInfo {
+        HandLaserMode mode { HandLaserMode::None };
+        vec4 color { 1.0f };
+        vec3 direction { 0, 0, -1 };
+
+        // Is this hand laser info suitable for drawing?
+        bool valid() const {
+            return (mode != HandLaserMode::None && color.a > 0.0f && direction != vec3());
+        }
+    };
+
+    std::array<HandLaserInfo, 2> _handLasers;
+    std::array<glm::mat4, 2> _handPoses;
     std::array<glm::mat4, 2> _eyeOffsets;
     std::array<glm::mat4, 2> _eyeProjections;
     std::array<glm::mat4, 2> _eyeInverseProjections;
@@ -75,5 +91,7 @@ private:
     bool _enableReprojection { true };
     ShapeWrapperPtr _sphereSection;
     ProgramPtr _reprojectionProgram;
+    ProgramPtr _laserProgram;
+    ShapeWrapperPtr _laserGeometry;
 };
 

libraries/gl/src/gl/OglplusHelpers.cpp

@@ -45,9 +45,11 @@ in vec2 vTexCoord;
 out vec4 FragColor;
 
 void main() {
-
     FragColor = texture(sampler, vTexCoord);
     FragColor.a *= alpha;
+    if (FragColor.a <= 0.0) {
+        discard;
+    }
 }
 
 )FS";
@@ -359,6 +361,94 @@ ShapeWrapperPtr loadSphereSection(ProgramPtr program, float fov, float aspect, i
     );
 }
 
+namespace oglplus {
+    namespace shapes {
+
+        class Laser : public DrawingInstructionWriter, public DrawMode {
+        public:
+            using IndexArray = std::vector<GLuint>;
+            using PosArray = std::vector<float>;
+            /// The type of the index container returned by Indices()
+            // vertex positions
+            PosArray _pos_data;
+            IndexArray _idx_data;
+            unsigned int _prim_count { 0 };
+
+        public:
+            Laser() {
+                int vertices = 2;
+                _pos_data.resize(vertices * 3);
+                _pos_data[0] = 0;
+                _pos_data[1] = 0;
+                _pos_data[2] = 0;
+
+                _pos_data[3] = 0;
+                _pos_data[4] = 0;
+                _pos_data[5] = -1;
+
+                _idx_data.push_back(0);
+                _idx_data.push_back(1);
+                _prim_count = 1;
+            }
+
+            /// Returns the winding direction of faces
+            FaceOrientation FaceWinding(void) const {
+                return FaceOrientation::CCW;
+            }
+
+            /// Queries the bounding sphere coordinates and dimensions
+            template <typename T>
+            void BoundingSphere(Sphere<T>& bounding_sphere) const {
+                bounding_sphere = Sphere<T>(0, 0, -0.5, 0.5);
+            }
+
+            typedef GLuint(Laser::*VertexAttribFunc)(std::vector<GLfloat>&) const;
+
+            /// Makes the vertex positions and returns the number of values per vertex
+            template <typename T>
+            GLuint Positions(std::vector<T>& dest) const {
+                dest.clear();
+                dest.insert(dest.begin(), _pos_data.begin(), _pos_data.end());
+                return 3;
+            }
+
+            typedef VertexAttribsInfo<
+                Laser,
+                std::tuple<VertexPositionsTag>
+            > VertexAttribs;
+
+
+            /// Returns element indices that are used with the drawing instructions
+            const IndexArray & Indices(Default = Default()) const {
+                return _idx_data;
+            }
+
+            /// Returns the instructions for rendering of faces
+            DrawingInstructions Instructions(PrimitiveType primitive) const {
+                DrawingInstructions instr = MakeInstructions();
+                DrawOperation operation;
+                operation.method = DrawOperation::Method::DrawElements;
+                operation.mode = primitive;
+                operation.first = 0;
+                operation.count = _prim_count * 3;
+                operation.restart_index = DrawOperation::NoRestartIndex();
+                operation.phase = 0;
+                AddInstruction(instr, operation);
+                return instr;
+            }
+
+            /// Returns the instructions for rendering of faces
+            DrawingInstructions Instructions(Default = Default()) const {
+                return Instructions(PrimitiveType::Lines);
+            }
+        };
+    }
+}
+
+ShapeWrapperPtr loadLaser(const ProgramPtr& program) {
+    return std::make_shared<shapes::ShapeWrapper>(shapes::ShapeWrapper("Position", shapes::Laser(), *program));
+}
+
 void TextureRecycler::setSize(const uvec2& size) {
     if (size == _size) {
         return;

libraries/gl/src/gl/OglplusHelpers.h

@@ -64,8 +64,9 @@ ProgramPtr loadCubemapShader();
 void compileProgram(ProgramPtr & result, const std::string& vs, const std::string& fs);
 ShapeWrapperPtr loadSkybox(ProgramPtr program);
 ShapeWrapperPtr loadPlane(ProgramPtr program, float aspect = 1.0f);
-ShapeWrapperPtr loadSphereSection(ProgramPtr program, float fov = PI / 3.0f * 2.0f, float aspect = 16.0f / 9.0f, int slices = 32, int stacks = 32);
-    
+ShapeWrapperPtr loadSphereSection(ProgramPtr program, float fov = PI / 3.0f * 2.0f, float aspect = 16.0f / 9.0f, int slices = 128, int stacks = 128);
+ShapeWrapperPtr loadLaser(const ProgramPtr& program);
+
 
 // A basic wrapper for constructing a framebuffer with a renderbuffer
 // for the depth attachment and an undefined type for the color attachement

libraries/plugins/src/plugins/DisplayPlugin.h

@@ -168,6 +168,26 @@ public:
 
     static const QString& MENU_PATH();
 
+    enum Hand {
+        LeftHand = 0x01,
+        RightHand = 0x02,
+    };
+
+    enum class HandLaserMode {
+        None, // Render no hand lasers
+        Overlay, // Render hand lasers only if they intersect with the UI layer, and stop at the UI layer
+    };
+
+    virtual bool setHandLaser(
+        uint32_t hands, // Bits from the Hand enum
+        HandLaserMode mode, // Mode in which to render
+        const vec4& color = vec4(1), // The color of the rendered laser
+        const vec3& direction = vec3(0, 0, -1) // The direction in which to render the hand lasers
+    ) {
+        return false;
+    }
+
+
 signals:
     void recommendedFramebufferSizeChanged(const QSize & size);
     // Indicates that this display plugin is no longer valid for use.

plugins/oculus/src/OculusBaseDisplayPlugin.cpp

@@ -24,8 +24,9 @@ bool OculusBaseDisplayPlugin::beginFrameRender(uint32_t frameIndex) {
     auto trackingState = ovr_GetTrackingState(_session, _currentRenderFrameInfo.predictedDisplayTime, ovrTrue);
     _currentRenderFrameInfo.renderPose = toGlm(trackingState.HeadPose.ThePose);
     _currentRenderFrameInfo.presentPose = _currentRenderFrameInfo.renderPose;
-    Lock lock(_mutex);
-    _frameInfos[frameIndex] = _currentRenderFrameInfo;
+    withRenderThreadLock([&] {
+        _frameInfos[frameIndex] = _currentRenderFrameInfo;
+    });
     return true;
 }
 

plugins/oculusLegacy/src/OculusLegacyDisplayPlugin.cpp

@@ -45,8 +45,9 @@ bool OculusLegacyDisplayPlugin::beginFrameRender(uint32_t frameIndex) {
     _currentRenderFrameInfo.predictedDisplayTime = _currentRenderFrameInfo.sensorSampleTime = ovr_GetTimeInSeconds();
     _trackingState = ovrHmd_GetTrackingState(_hmd, _currentRenderFrameInfo.predictedDisplayTime);
     _currentRenderFrameInfo.rawRenderPose = _currentRenderFrameInfo.renderPose = toGlm(_trackingState.HeadPose.ThePose);
-    Lock lock(_mutex);
-    _frameInfos[frameIndex] = _currentRenderFrameInfo;
+    withRenderThreadLock([&]{
+        _frameInfos[frameIndex] = _currentRenderFrameInfo;
+    })
     return true;
 }
 

plugins/openvr/src/OpenVrDisplayPlugin.cpp

@@ -29,11 +29,13 @@ Q_DECLARE_LOGGING_CATEGORY(displayplugins)
 const QString OpenVrDisplayPlugin::NAME("OpenVR (Vive)");
 const QString StandingHMDSensorMode = "Standing HMD Sensor Mode"; // this probably shouldn't be hardcoded here
 
-static vr::IVRCompositor* _compositor{ nullptr };
+static vr::IVRCompositor* _compositor { nullptr };
 vr::TrackedDevicePose_t _trackedDevicePose[vr::k_unMaxTrackedDeviceCount];
+
 mat4 _trackedDevicePoseMat4[vr::k_unMaxTrackedDeviceCount];
 vec3 _trackedDeviceLinearVelocities[vr::k_unMaxTrackedDeviceCount];
 vec3 _trackedDeviceAngularVelocities[vr::k_unMaxTrackedDeviceCount];
+
 static mat4 _sensorResetMat;
 static std::array<vr::Hmd_Eye, 2> VR_EYES { { vr::Eye_Left, vr::Eye_Right } };
 bool _openVrDisplayActive { false };
@@ -59,16 +61,14 @@ bool OpenVrDisplayPlugin::internalActivate() {
     // left + right eyes
     _renderTargetSize.x *= 2;
 
-    {
-        Lock lock(_poseMutex);
+    withRenderThreadLock([&] {
         openvr_for_each_eye([&](vr::Hmd_Eye eye) {
             _eyeOffsets[eye] = toGlm(_system->GetEyeToHeadTransform(eye));
             _eyeProjections[eye] = toGlm(_system->GetProjectionMatrix(eye, DEFAULT_NEAR_CLIP, DEFAULT_FAR_CLIP, vr::API_OpenGL));
         });
         // FIXME Calculate the proper combined projection by using GetProjectionRaw values from both eyes
         _cullingProjection = _eyeProjections[0];
-
-    }
+    });
 
     _compositor = vr::VRCompositor();
     Q_ASSERT(_compositor);
@@ -113,7 +113,7 @@ void OpenVrDisplayPlugin::internalDeactivate() {
 void OpenVrDisplayPlugin::customizeContext() {
     // Display plugins in DLLs must initialize glew locally
     static std::once_flag once;
-    std::call_once(once, []{
+    std::call_once(once, [] {
         glewExperimental = true;
         GLenum err = glewInit();
         glGetError(); // clear the potential error from glewExperimental
@@ -123,9 +123,10 @@ void OpenVrDisplayPlugin::customizeContext() {
 }
 
 void OpenVrDisplayPlugin::resetSensors() {
-    Lock lock(_poseMutex);
-    glm::mat4 m = toGlm(_trackedDevicePose[0].mDeviceToAbsoluteTracking);
-    _sensorResetMat = glm::inverse(cancelOutRollAndPitch(m));
+    withRenderThreadLock([&] {
+        glm::mat4 m = toGlm(_trackedDevicePose[0].mDeviceToAbsoluteTracking);
+        _sensorResetMat = glm::inverse(cancelOutRollAndPitch(m));
+    });
 }
 
 
@@ -150,6 +151,24 @@ bool OpenVrDisplayPlugin::beginFrameRender(uint32_t frameIndex) {
 
     _system->GetDeviceToAbsoluteTrackingPose(vr::TrackingUniverseStanding, _currentRenderFrameInfo.predictedDisplayTime, _trackedDevicePose, vr::k_unMaxTrackedDeviceCount);
 
+
+    vr::TrackedDeviceIndex_t handIndices[2] { vr::k_unTrackedDeviceIndexInvalid, vr::k_unTrackedDeviceIndexInvalid };
+    {
+        vr::TrackedDeviceIndex_t controllerIndices[2] ;
+        auto trackedCount = _system->GetSortedTrackedDeviceIndicesOfClass(vr::TrackedDeviceClass_Controller, controllerIndices, 2);
+        // Find the left and right hand controllers, if they exist
+        for (uint32_t i = 0; i < std::min<uint32_t>(trackedCount, 2); ++i) {
+            if (_trackedDevicePose[i].bPoseIsValid) {
+                auto role = _system->GetControllerRoleForTrackedDeviceIndex(controllerIndices[i]);
+                if (vr::TrackedControllerRole_LeftHand == role) {
+                    handIndices[0] = controllerIndices[i];
+                } else if (vr::TrackedControllerRole_RightHand == role) {
+                    handIndices[1] = controllerIndices[i];
+                }
+            }
+        }
+    }
+
     // copy and process predictedTrackedDevicePoses
     for (int i = 0; i < vr::k_unMaxTrackedDeviceCount; i++) {
         _trackedDevicePoseMat4[i] = _sensorResetMat * toGlm(_trackedDevicePose[i].mDeviceToAbsoluteTracking);
@@ -159,18 +178,27 @@ bool OpenVrDisplayPlugin::beginFrameRender(uint32_t frameIndex) {
     _currentRenderFrameInfo.rawRenderPose = toGlm(_trackedDevicePose[vr::k_unTrackedDeviceIndex_Hmd].mDeviceToAbsoluteTracking);
     _currentRenderFrameInfo.renderPose = _trackedDevicePoseMat4[vr::k_unTrackedDeviceIndex_Hmd];
 
-    Lock lock(_mutex);
-    _frameInfos[frameIndex] = _currentRenderFrameInfo;
+    bool keyboardVisible = isOpenVrKeyboardShown();
+    withRenderThreadLock([&] {
+        // Make controller poses available to the presentation thread
+        for (int i = 0; i < 2; ++i) {
+            if (keyboardVisible || handIndices[i] == vr::k_unTrackedDeviceIndexInvalid) {
+                _handPoses[i] = glm::mat4();
+            } else {
+                _handPoses[i] = _sensorResetMat * toGlm(_trackedDevicePose[handIndices[i]].mDeviceToAbsoluteTracking);
+            }
+        }
+        _frameInfos[frameIndex] = _currentRenderFrameInfo;
+    });
     return true;
 }
 
 void OpenVrDisplayPlugin::hmdPresent() {
-
     PROFILE_RANGE_EX(__FUNCTION__, 0xff00ff00, (uint64_t)_currentPresentFrameIndex)
 
     // Flip y-axis since GL UV coords are backwards.
-    static vr::VRTextureBounds_t leftBounds{ 0, 0, 0.5f, 1 };
-    static vr::VRTextureBounds_t rightBounds{ 0.5f, 0, 1, 1 };
+    static vr::VRTextureBounds_t leftBounds { 0, 0, 0.5f, 1 };
+    static vr::VRTextureBounds_t rightBounds { 0.5f, 0, 1, 1 };
 
     vr::Texture_t texture { (void*)oglplus::GetName(_compositeFramebuffer->color), vr::API_OpenGL, vr::ColorSpace_Auto };
 
@@ -191,6 +219,10 @@ bool OpenVrDisplayPlugin::isHmdMounted() const {
 }
 
 void OpenVrDisplayPlugin::updatePresentPose() {
+    mat4 sensorResetMat;
+    withPresentThreadLock([&] {
+        sensorResetMat = _sensorResetMat;
+    });
     {
         float fSecondsSinceLastVsync;
         _system->GetTimeSinceLastVsync(&fSecondsSinceLastVsync, nullptr);
@@ -202,9 +234,8 @@ void OpenVrDisplayPlugin::updatePresentPose() {
         _system->GetDeviceToAbsoluteTrackingPose(vr::TrackingUniverseStanding, fPredictedSecondsFromNow, &pose, 1);
         _currentPresentFrameInfo.rawPresentPose = toGlm(pose.mDeviceToAbsoluteTracking);
     }
-    _currentPresentFrameInfo.presentPose = _sensorResetMat * _currentPresentFrameInfo.rawPresentPose;
+    _currentPresentFrameInfo.presentPose = sensorResetMat * _currentPresentFrameInfo.rawPresentPose;
     mat3 renderRotation(_currentPresentFrameInfo.rawRenderPose);
     mat3 presentRotation(_currentPresentFrameInfo.rawPresentPose);
     _currentPresentFrameInfo.presentReprojection = glm::mat3(glm::inverse(renderRotation) * presentRotation);
 }
-

plugins/openvr/src/OpenVrDisplayPlugin.h

@@ -43,5 +43,4 @@ private:
     vr::IVRSystem* _system { nullptr };
     std::atomic<vr::EDeviceActivityLevel> _hmdActivityLevel { vr::k_EDeviceActivityLevel_Unknown };
     static const QString NAME;
-    mutable Mutex _poseMutex;
 };

plugins/openvr/src/OpenVrHelpers.cpp

@@ -208,6 +208,10 @@ void disableOpenVrKeyboard() {
     QObject::disconnect(_focusConnection);
 }
 
+bool isOpenVrKeyboardShown() {
+    return _keyboardShown;
+}
+
 
 void handleOpenVrEvents() {
     if (!activeHmd) {

plugins/openvr/src/OpenVrHelpers.h

@@ -20,6 +20,7 @@ void handleOpenVrEvents();
 bool openVrQuitRequested();
 void enableOpenVrKeyboard();
 void disableOpenVrKeyboard();
+bool isOpenVrKeyboardShown();
 
 
 template<typename F>