Merge pull request #7640 from jherico/vive_timewarp

Vive timewarp

interface/resources/shaders/reproject.frag

@@ -0,0 +1,71 @@
+#version 450 core
+
+uniform sampler2D sampler;
+layout (location = 0) uniform mat3 reprojection = mat3(1);
+layout (location = 4) uniform mat4 inverseProjections[2];
+layout (location = 12) uniform mat4 projections[2];
+
+in vec2 vTexCoord;
+in vec3 vPosition;
+
+out vec4 FragColor;
+
+void main() {
+    vec2 uv = vTexCoord;
+
+    mat4 eyeInverseProjection;
+    mat4 eyeProjection;
+    
+    float xoffset = 1.0;
+    vec2 uvmin = vec2(0.0);
+    vec2 uvmax = vec2(1.0);
+    // determine the correct projection and inverse projection to use.
+    if (vTexCoord.x < 0.5) {
+        uvmax.x = 0.5;
+        eyeInverseProjection = inverseProjections[0];
+        eyeProjection = projections[0];
+    } else {
+        xoffset = -1.0;
+        uvmin.x = 0.5;
+        uvmax.x = 1.0;
+        eyeInverseProjection = inverseProjections[1];
+        eyeProjection = projections[1];
+    }
+
+    // Account for stereo in calculating the per-eye NDC coordinates
+    vec4 ndcSpace = vec4(vPosition, 1.0);
+    ndcSpace.x *= 2.0;
+    ndcSpace.x += xoffset;
+    
+    // Convert from NDC to eyespace
+    vec4 eyeSpace = eyeInverseProjection * ndcSpace;
+    eyeSpace /= eyeSpace.w;
+
+    // Convert to a noramlized ray 
+    vec3 ray = eyeSpace.xyz;
+    ray = normalize(ray);
+
+    // Adjust the ray by the rotation
+    ray = reprojection * ray;
+
+    // Project back on to the texture plane
+    ray /= ray.z;
+    ray *= eyeSpace.z;
+
+    // Update the eyespace vector
+    eyeSpace.xyz = ray;
+
+    // Reproject back into NDC
+    ndcSpace = eyeProjection * eyeSpace;
+    ndcSpace /= ndcSpace.w;
+    ndcSpace.x -= xoffset;
+    ndcSpace.x /= 2.0;
+    
+    // Calculate the new UV coordinates
+    uv = (ndcSpace.xy / 2.0) + 0.5;
+    if (any(greaterThan(uv, uvmax)) || any(lessThan(uv, uvmin))) {
+        FragColor = vec4(0.0, 0.0, 0.0, 1.0);
+    } else {
+        FragColor = texture(sampler, uv);
+    }
+}

libraries/display-plugins/src/display-plugins/OpenGLDisplayPlugin.cpp

@@ -386,6 +386,11 @@ bool OpenGLDisplayPlugin::eventFilter(QObject* receiver, QEvent* event) {
 }
 
 void OpenGLDisplayPlugin::submitSceneTexture(uint32_t frameIndex, const gpu::TexturePointer& sceneTexture) {
+    if (_lockCurrentTexture) {
+        _container->releaseSceneTexture(sceneTexture);
+        return;
+    }
+
     {
         Lock lock(_mutex);
         _sceneTextureToFrameIndexMap[sceneTexture] = frameIndex;
@@ -424,9 +429,9 @@ void OpenGLDisplayPlugin::updateTextures() {
 
 void OpenGLDisplayPlugin::updateFrameData() {
     Lock lock(_mutex);
-    auto previousFrameIndex = _currentRenderFrameIndex;
+    auto previousFrameIndex = _currentPresentFrameIndex;
-    _currentRenderFrameIndex = _sceneTextureToFrameIndexMap[_currentSceneTexture];
+    _currentPresentFrameIndex = _sceneTextureToFrameIndexMap[_currentSceneTexture];
-    auto skippedCount = (_currentRenderFrameIndex - previousFrameIndex) - 1;
+    auto skippedCount = (_currentPresentFrameIndex - previousFrameIndex) - 1;
     _droppedFrameRate.increment(skippedCount);
 }
 
@@ -435,6 +440,7 @@ void OpenGLDisplayPlugin::compositeOverlay() {
 
     auto compositorHelper = DependencyManager::get<CompositorHelper>();
 
+    useProgram(_program);
     // check the alpha
     auto overlayAlpha = compositorHelper->getAlpha();
     if (overlayAlpha > 0.0f) {
@@ -461,6 +467,7 @@ void OpenGLDisplayPlugin::compositePointer() {
     using namespace oglplus;
     auto compositorHelper = DependencyManager::get<CompositorHelper>();
 
+    useProgram(_program);
     // check the alpha
     auto overlayAlpha = compositorHelper->getAlpha();
     if (overlayAlpha > 0.0f) {
@@ -481,6 +488,13 @@ void OpenGLDisplayPlugin::compositePointer() {
     Uniform<float>(*_program, _alphaUniform).Set(1.0);
 }
 
+void OpenGLDisplayPlugin::compositeScene() {
+    using namespace oglplus;
+    useProgram(_program);
+    Uniform<glm::mat4>(*_program, _mvpUniform).Set(mat4());
+    drawUnitQuad();
+}
+
 void OpenGLDisplayPlugin::compositeLayers() {
     using namespace oglplus;
     auto targetRenderSize = getRecommendedRenderSize();
@@ -492,9 +506,7 @@ void OpenGLDisplayPlugin::compositeLayers() {
         Context::Viewport(targetRenderSize.x, targetRenderSize.y);
         Context::Clear().DepthBuffer();
         glBindTexture(GL_TEXTURE_2D, getSceneTextureId());
-        _program->Bind();
+        compositeScene();
-        Uniform<glm::mat4>(*_program, _mvpUniform).Set(mat4());
-        drawUnitQuad();
         auto overlayTextureId = getOverlayTextureId();
         if (overlayTextureId) {
             glEnable(GL_BLEND);
@@ -540,6 +552,7 @@ void OpenGLDisplayPlugin::present() {
         // Take the composite framebuffer and send it to the output device
         internalPresent();
         _presentRate.increment();
+        _activeProgram.reset();
     }
 }
 
@@ -556,7 +569,7 @@ float OpenGLDisplayPlugin::presentRate() const {
 }
 
 void OpenGLDisplayPlugin::drawUnitQuad() {
-    _program->Bind();
+    useProgram(_program);
     _plane->Use();
     _plane->Draw();
 }
@@ -655,3 +668,10 @@ bool OpenGLDisplayPlugin::hasFocus() const {
     auto window = _container->getPrimaryWidget();
     return window ? window->hasFocus() : false;
 }
+
+void OpenGLDisplayPlugin::useProgram(const ProgramPtr& program) {
+    if (_activeProgram != program) {
+        program->Bind();
+        _activeProgram = program;
+    }
+}

libraries/display-plugins/src/display-plugins/OpenGLDisplayPlugin.h

@@ -71,6 +71,7 @@ protected:
     glm::uvec2 getSurfacePixels() const;
 
     void compositeLayers();
+    virtual void compositeScene();
     virtual void compositeOverlay();
     virtual void compositePointer();
 
@@ -93,6 +94,7 @@ protected:
 
     void withMainThreadContext(std::function<void()> f) const;
 
+    void useProgram(const ProgramPtr& program);
     void present();
     void updateTextures();
     void drawUnitQuad();
@@ -111,7 +113,7 @@ protected:
     RateCounter<> _newFrameRate;
     RateCounter<> _presentRate;
     QMap<gpu::TexturePointer, uint32_t> _sceneTextureToFrameIndexMap;
-    uint32_t _currentRenderFrameIndex { 0 };
+    uint32_t _currentPresentFrameIndex { 0 };
 
     gpu::TexturePointer _currentSceneTexture;
     gpu::TexturePointer _currentOverlayTexture;
@@ -130,6 +132,10 @@ protected:
 
     std::map<uint16_t, CursorData> _cursorsData;
     BasicFramebufferWrapperPtr _compositeFramebuffer;
+    bool _lockCurrentTexture { false };
+
+private:
+    ProgramPtr _activeProgram;
 };
 
 

libraries/display-plugins/src/display-plugins/hmd/HmdDisplayPlugin.cpp

@@ -25,7 +25,9 @@
 #include "../CompositorHelper.h"
 
 static const QString MONO_PREVIEW = "Mono Preview";
+static const QString REPROJECTION = "Allow Reprojection";
 static const QString FRAMERATE = DisplayPlugin::MENU_PATH() + ">Framerate";
+static const QString DEVELOPER_MENU_PATH = "Developer>" + DisplayPlugin::MENU_PATH();
 static const bool DEFAULT_MONO_VIEW = true;
 
 glm::uvec2 HmdDisplayPlugin::getRecommendedUiSize() const {
@@ -41,39 +43,219 @@ bool HmdDisplayPlugin::internalActivate() {
         _container->setBoolSetting("monoPreview", _monoPreview);
     }, true, _monoPreview);
     _container->removeMenu(FRAMERATE);
+    _container->addMenu(DEVELOPER_MENU_PATH);
+    _container->addMenuItem(PluginType::DISPLAY_PLUGIN, DEVELOPER_MENU_PATH, REPROJECTION,
+        [this](bool clicked) {
+            _enableReprojection = clicked;
+            _container->setBoolSetting("enableReprojection", _enableReprojection);
+    }, true, _enableReprojection);
+    
+    for_each_eye([&](Eye eye) {
+        _eyeInverseProjections[eye] = glm::inverse(_eyeProjections[eye]);
+    });
 
     return Parent::internalActivate();
 }
 
+
+static const char * REPROJECTION_VS = R"VS(#version 450 core
+in vec3 Position;
+in vec2 TexCoord;
+
+out vec3 vPosition;
+out vec2 vTexCoord;
+
+void main() {
+  gl_Position = vec4(Position, 1);
+  vTexCoord = TexCoord;
+  vPosition = Position;
+}
+
+)VS";
+
+static const GLint REPROJECTION_MATRIX_LOCATION = 0;
+static const GLint INVERSE_PROJECTION_MATRIX_LOCATION = 4;
+static const GLint PROJECTION_MATRIX_LOCATION = 12;
+static const char * REPROJECTION_FS = R"FS(#version 450 core
+
+uniform sampler2D sampler;
+layout (location = 0) uniform mat3 reprojection = mat3(1);
+layout (location = 4) uniform mat4 inverseProjections[2];
+layout (location = 12) uniform mat4 projections[2];
+
+in vec2 vTexCoord;
+in vec3 vPosition;
+
+out vec4 FragColor;
+
+void main() {
+    vec2 uv = vTexCoord;
+
+    mat4 eyeInverseProjection;
+    mat4 eyeProjection;
+    
+    float xoffset = 1.0;
+    vec2 uvmin = vec2(0.0);
+    vec2 uvmax = vec2(1.0);
+    // determine the correct projection and inverse projection to use.
+    if (vTexCoord.x < 0.5) {
+        uvmax.x = 0.5;
+        eyeInverseProjection = inverseProjections[0];
+        eyeProjection = projections[0];
+    } else {
+        xoffset = -1.0;
+        uvmin.x = 0.5;
+        uvmax.x = 1.0;
+        eyeInverseProjection = inverseProjections[1];
+        eyeProjection = projections[1];
+    }
+
+    // Account for stereo in calculating the per-eye NDC coordinates
+    vec4 ndcSpace = vec4(vPosition, 1.0);
+    ndcSpace.x *= 2.0;
+    ndcSpace.x += xoffset;
+    
+    // Convert from NDC to eyespace
+    vec4 eyeSpace = eyeInverseProjection * ndcSpace;
+    eyeSpace /= eyeSpace.w;
+
+    // Convert to a noramlized ray 
+    vec3 ray = eyeSpace.xyz;
+    ray = normalize(ray);
+
+    // Adjust the ray by the rotation
+    ray = reprojection * ray;
+
+    // Project back on to the texture plane
+    ray *= eyeSpace.z / ray.z;
+
+    // Update the eyespace vector
+    eyeSpace.xyz = ray;
+
+    // Reproject back into NDC
+    ndcSpace = eyeProjection * eyeSpace;
+    ndcSpace /= ndcSpace.w;
+    ndcSpace.x -= xoffset;
+    ndcSpace.x /= 2.0;
+    
+    // Calculate the new UV coordinates
+    uv = (ndcSpace.xy / 2.0) + 0.5;
+    if (any(greaterThan(uv, uvmax)) || any(lessThan(uv, uvmin))) {
+        FragColor = vec4(0.0, 0.0, 0.0, 1.0);
+    } else {
+        FragColor = texture(sampler, uv);
+    }
+}
+)FS";
+
+#ifdef DEBUG_REPROJECTION_SHADER
+#include <QtCore/QFile>
+#include <QtCore/QFileInfo>
+#include <QtCore/QDateTime>
+#include <PathUtils.h>
+
+static const QString REPROJECTION_FS_FILE =  "c:/Users/bdavis/Git/hifi/interface/resources/shaders/reproject.frag";
+
+static ProgramPtr getReprojectionProgram() {
+    static ProgramPtr _currentProgram;
+    uint64_t now = usecTimestampNow();
+    static uint64_t _lastFileCheck = now;
+
+    bool modified = false;
+    if ((now - _lastFileCheck) > USECS_PER_MSEC * 100) {
+        QFileInfo info(REPROJECTION_FS_FILE);
+        QDateTime lastModified = info.lastModified();
+        static QDateTime _lastModified = lastModified;
+        qDebug() << lastModified.toTime_t();
+        qDebug() << _lastModified.toTime_t();
+        if (lastModified > _lastModified) {
+            _lastModified = lastModified;
+            modified = true;
+        }
+    }
+
+    if (!_currentProgram || modified) {
+        _currentProgram.reset();
+        try {
+            QFile shaderFile(REPROJECTION_FS_FILE);
+            shaderFile.open(QIODevice::ReadOnly);
+            QString fragment = shaderFile.readAll();
+            compileProgram(_currentProgram, REPROJECTION_VS, fragment.toLocal8Bit().data());
+        } catch (const std::runtime_error& error) {
+            qDebug() << "Failed to build: " << error.what();
+        }
+        if (!_currentProgram) {
+            _currentProgram = loadDefaultShader();
+        }
+    }
+    return _currentProgram;
+}
+#endif
+
+
 void HmdDisplayPlugin::customizeContext() {
     Parent::customizeContext();
     // Only enable mirroring if we know vsync is disabled
     enableVsync(false);
     _enablePreview = !isVsyncEnabled();
     _sphereSection = loadSphereSection(_program, CompositorHelper::VIRTUAL_UI_TARGET_FOV.y, CompositorHelper::VIRTUAL_UI_ASPECT_RATIO);
+    compileProgram(_reprojectionProgram, REPROJECTION_VS, REPROJECTION_FS);
 }
 
 void HmdDisplayPlugin::uncustomizeContext() {
     _sphereSection.reset();
     _compositeFramebuffer.reset();
+    _reprojectionProgram.reset();
     Parent::uncustomizeContext();
 }
 
+// By default assume we'll present with the same pose as the render
+void HmdDisplayPlugin::updatePresentPose() {
+    _currentPresentFrameInfo.presentPose = _currentPresentFrameInfo.renderPose;
+}
+
+void HmdDisplayPlugin::compositeScene() {
+    updatePresentPose();
+
+    if (!_enableReprojection || glm::mat3() == _currentPresentFrameInfo.presentReprojection) {
+        // No reprojection required
+        Parent::compositeScene();
+        return;
+    }
+
+#ifdef DEBUG_REPROJECTION_SHADER
+    _reprojectionProgram = getReprojectionProgram();
+#endif
+    useProgram(_reprojectionProgram);
+
+    using namespace oglplus;
+    Texture::MinFilter(TextureTarget::_2D, TextureMinFilter::Linear);
+    Texture::MagFilter(TextureTarget::_2D, TextureMagFilter::Linear);
+    Uniform<glm::mat3>(*_reprojectionProgram, REPROJECTION_MATRIX_LOCATION).Set(_currentPresentFrameInfo.presentReprojection);
+    //Uniform<glm::mat4>(*_reprojectionProgram, PROJECTION_MATRIX_LOCATION).Set(_eyeProjections);
+    //Uniform<glm::mat4>(*_reprojectionProgram, INVERSE_PROJECTION_MATRIX_LOCATION).Set(_eyeInverseProjections);
+    // FIXME what's the right oglplus mechanism to do this?  It's not that ^^^ ... better yet, switch to a uniform buffer
+    glUniformMatrix4fv(INVERSE_PROJECTION_MATRIX_LOCATION, 2, GL_FALSE, &(_eyeInverseProjections[0][0][0]));
+    glUniformMatrix4fv(PROJECTION_MATRIX_LOCATION, 2, GL_FALSE, &(_eyeProjections[0][0][0]));
+    _plane->UseInProgram(*_reprojectionProgram);
+    _plane->Draw();
+}
+
 void HmdDisplayPlugin::compositeOverlay() {
     using namespace oglplus;
     auto compositorHelper = DependencyManager::get<CompositorHelper>();
 
     // check the alpha
+    useProgram(_program);
     auto overlayAlpha = compositorHelper->getAlpha();
     if (overlayAlpha > 0.0f) {
         // set the alpha
         Uniform<float>(*_program, _alphaUniform).Set(overlayAlpha);
 
-        auto eyePoses = _currentPresentFrameInfo.eyePoses;
         _sphereSection->Use();
         for_each_eye([&](Eye eye) {
             eyeViewport(eye);
-            auto modelView = glm::inverse(eyePoses[eye]); // *glm::translate(mat4(), vec3(0, 0, -1));
+            auto modelView = glm::inverse(_currentPresentFrameInfo.presentPose * getEyeToHeadTransform(eye));
             auto mvp = _eyeProjections[eye] * modelView;
             Uniform<glm::mat4>(*_program, _mvpUniform).Set(mvp);
             _sphereSection->Draw();
@@ -88,6 +270,7 @@ void HmdDisplayPlugin::compositePointer() {
     auto compositorHelper = DependencyManager::get<CompositorHelper>();
 
     // check the alpha
+    useProgram(_program);
     auto overlayAlpha = compositorHelper->getAlpha();
     if (overlayAlpha > 0.0f) {
         // set the alpha
@@ -96,10 +279,11 @@ void HmdDisplayPlugin::compositePointer() {
         // Mouse pointer
         _plane->Use();
         // Reconstruct the headpose from the eye poses
-        auto headPosition = vec3(_currentPresentFrameInfo.headPose[3]);
+        auto headPosition = vec3(_currentPresentFrameInfo.presentPose[3]);
         for_each_eye([&](Eye eye) {
             eyeViewport(eye);
-            auto reticleTransform = compositorHelper->getReticleTransform(_currentPresentFrameInfo.eyePoses[eye], headPosition);
+            auto eyePose = _currentPresentFrameInfo.presentPose * getEyeToHeadTransform(eye);
+            auto reticleTransform = compositorHelper->getReticleTransform(eyePose, headPosition);
             auto mvp = _eyeProjections[eye] * reticleTransform;
             Uniform<glm::mat4>(*_program, _mvpUniform).Set(mvp);
             _plane->Draw();
@@ -160,16 +344,13 @@ void HmdDisplayPlugin::internalPresent() {
 }
 
 void HmdDisplayPlugin::setEyeRenderPose(uint32_t frameIndex, Eye eye, const glm::mat4& pose) {
-    Lock lock(_mutex);
-    FrameInfo& frame = _frameInfos[frameIndex];
-    frame.eyePoses[eye] = pose;
 }
 
 void HmdDisplayPlugin::updateFrameData() {
     // Check if we have old frame data to discard
     {
         Lock lock(_mutex);
-        auto itr = _frameInfos.find(_currentRenderFrameIndex);
+        auto itr = _frameInfos.find(_currentPresentFrameIndex);
         if (itr != _frameInfos.end()) {
             _frameInfos.erase(itr);
         }
@@ -179,10 +360,10 @@ void HmdDisplayPlugin::updateFrameData() {
 
     {
         Lock lock(_mutex);
-        _currentPresentFrameInfo = _frameInfos[_currentRenderFrameIndex];
+        _currentPresentFrameInfo = _frameInfos[_currentPresentFrameIndex];
     }
 }
 
 glm::mat4 HmdDisplayPlugin::getHeadPose() const {
-    return _currentRenderFrameInfo.get().headPose;
+    return _currentRenderFrameInfo.renderPose;
 }

libraries/display-plugins/src/display-plugins/hmd/HmdDisplayPlugin.h

@@ -28,22 +28,16 @@ public:
 
     virtual glm::mat4 getHeadPose() const override;
 
-    using EyePoses = std::array<glm::mat4, 2>;
-
-    struct FrameInfo {
-        EyePoses eyePoses;
-        glm::mat4 headPose;
-        double sensorSampleTime { 0 };
-        double predictedDisplayTime { 0 };
-    };
 
 
 protected:
     virtual void hmdPresent() = 0;
     virtual bool isHmdMounted() const = 0;
     virtual void postPreview() {};
+    virtual void updatePresentPose();
 
     bool internalActivate() override;
+    void compositeScene() override;
     void compositeOverlay() override;
     void compositePointer() override;
     void internalPresent() override;
@@ -53,17 +47,31 @@ protected:
 
     std::array<glm::mat4, 2> _eyeOffsets;
     std::array<glm::mat4, 2> _eyeProjections;
+    std::array<glm::mat4, 2> _eyeInverseProjections;
+
     glm::mat4 _cullingProjection;
     glm::uvec2 _renderTargetSize;
     float _ipd { 0.064f };
 
+    struct FrameInfo {
+        glm::mat4 rawRenderPose;
+        glm::mat4 renderPose;
+        glm::mat4 rawPresentPose;
+        glm::mat4 presentPose;
+        double sensorSampleTime { 0 };
+        double predictedDisplayTime { 0 };
+        glm::mat3 presentReprojection;
+    };
+
     QMap<uint32_t, FrameInfo> _frameInfos;
     FrameInfo _currentPresentFrameInfo;
-    ThreadSafeValueCache<FrameInfo> _currentRenderFrameInfo;
+    FrameInfo _currentRenderFrameInfo;
 
 private:
     bool _enablePreview { false };
     bool _monoPreview { true };
+    bool _enableReprojection { true };
     ShapeWrapperPtr _sphereSection;
+    ProgramPtr _reprojectionProgram;
 };
 

plugins/oculus/src/OculusBaseDisplayPlugin.cpp

@@ -16,15 +16,14 @@ void OculusBaseDisplayPlugin::resetSensors() {
 }
 
 void OculusBaseDisplayPlugin::beginFrameRender(uint32_t frameIndex) {
-    FrameInfo frame;
+    _currentRenderFrameInfo = FrameInfo();
-    frame.sensorSampleTime = ovr_GetTimeInSeconds();;
+    _currentRenderFrameInfo.sensorSampleTime = ovr_GetTimeInSeconds();;
-    frame.predictedDisplayTime = ovr_GetPredictedDisplayTime(_session, frameIndex);
+    _currentRenderFrameInfo.predictedDisplayTime = ovr_GetPredictedDisplayTime(_session, frameIndex);
-    auto trackingState = ovr_GetTrackingState(_session, frame.predictedDisplayTime, ovrTrue);
+    auto trackingState = ovr_GetTrackingState(_session, _currentRenderFrameInfo.predictedDisplayTime, ovrTrue);
-    frame.headPose = toGlm(trackingState.HeadPose.ThePose);
+    _currentRenderFrameInfo.renderPose = toGlm(trackingState.HeadPose.ThePose);
-
+    _currentRenderFrameInfo.presentPose = _currentRenderFrameInfo.renderPose;
-    _currentRenderFrameInfo.set(frame);
     Lock lock(_mutex);
-    _frameInfos[frameIndex] = frame;
+    _frameInfos[frameIndex] = _currentRenderFrameInfo;
 }
 
 bool OculusBaseDisplayPlugin::isSupported() const {

plugins/oculus/src/OculusDisplayPlugin.cpp

@@ -84,7 +84,7 @@ void blit(const SrcFbo& srcFbo, const DstFbo& dstFbo) {
 
 void OculusDisplayPlugin::hmdPresent() {
 
-    PROFILE_RANGE_EX(__FUNCTION__, 0xff00ff00, (uint64_t)_currentRenderFrameIndex)
+    PROFILE_RANGE_EX(__FUNCTION__, 0xff00ff00, (uint64_t)_currentPresentFrameIndex)
 
     if (!_currentSceneTexture) {
         return;
@@ -94,10 +94,10 @@ void OculusDisplayPlugin::hmdPresent() {
     _sceneFbo->Commit();
     {
         _sceneLayer.SensorSampleTime = _currentPresentFrameInfo.sensorSampleTime;
-        _sceneLayer.RenderPose[ovrEyeType::ovrEye_Left] = ovrPoseFromGlm(_currentPresentFrameInfo.headPose);
+        _sceneLayer.RenderPose[ovrEyeType::ovrEye_Left] = ovrPoseFromGlm(_currentPresentFrameInfo.renderPose);
-        _sceneLayer.RenderPose[ovrEyeType::ovrEye_Right] = ovrPoseFromGlm(_currentPresentFrameInfo.headPose);
+        _sceneLayer.RenderPose[ovrEyeType::ovrEye_Right] = ovrPoseFromGlm(_currentPresentFrameInfo.renderPose);
         ovrLayerHeader* layers = &_sceneLayer.Header;
-        ovrResult result = ovr_SubmitFrame(_session, _currentRenderFrameIndex, &_viewScaleDesc, &layers, 1);
+        ovrResult result = ovr_SubmitFrame(_session, _currentPresentFrameIndex, &_viewScaleDesc, &layers, 1);
         if (!OVR_SUCCESS(result)) {
             logWarning("Failed to present");
         }

plugins/oculusLegacy/src/OculusLegacyDisplayPlugin.cpp

@@ -36,13 +36,12 @@ void OculusLegacyDisplayPlugin::resetSensors() {
 }
 
 void OculusLegacyDisplayPlugin::beginFrameRender(uint32_t frameIndex) {
-    FrameInfo frame;
+    _currentRenderFrameInfo = FrameInfo();
-    frame.predictedDisplayTime = frame.sensorSampleTime = ovr_GetTimeInSeconds();
+    _currentRenderFrameInfo.predictedDisplayTime = _currentRenderFrameInfo.sensorSampleTime = ovr_GetTimeInSeconds();
-    _trackingState = ovrHmd_GetTrackingState(_hmd, frame.predictedDisplayTime);
+    _trackingState = ovrHmd_GetTrackingState(_hmd, _currentRenderFrameInfo.predictedDisplayTime);
-    frame.headPose = toGlm(_trackingState.HeadPose.ThePose);
+    _currentRenderFrameInfo.renderPose = toGlm(_trackingState.HeadPose.ThePose);
-    _currentRenderFrameInfo.set(frame);
     Lock lock(_mutex);
-    _frameInfos[frameIndex] = frame;
+    _frameInfos[frameIndex] = _currentRenderFrameInfo;
 }
 
 bool OculusLegacyDisplayPlugin::isSupported() const {

plugins/openvr/src/OpenVrDisplayPlugin.cpp

@@ -9,6 +9,7 @@
 
 #include <memory>
 
+#include <QtCore/QProcessEnvironment>
 #include <QMainWindow>
 #include <QLoggingCategory>
 #include <QGLWidget>
@@ -29,6 +30,10 @@ Q_DECLARE_LOGGING_CATEGORY(displayplugins)
 const QString OpenVrDisplayPlugin::NAME("OpenVR (Vive)");
 const QString StandingHMDSensorMode = "Standing HMD Sensor Mode"; // this probably shouldn't be hardcoded here
 
+static const QString DEBUG_FLAG("HIFI_DEBUG_OPENVR");
+static bool enableDebugOpenVR = QProcessEnvironment::systemEnvironment().contains(DEBUG_FLAG);
+
+
 static vr::IVRCompositor* _compositor{ nullptr };
 vr::TrackedDevicePose_t _trackedDevicePose[vr::k_unMaxTrackedDeviceCount];
 mat4 _trackedDevicePoseMat4[vr::k_unMaxTrackedDeviceCount];
@@ -38,12 +43,10 @@ static mat4 _sensorResetMat;
 static std::array<vr::Hmd_Eye, 2> VR_EYES { { vr::Eye_Left, vr::Eye_Right } };
 
 bool OpenVrDisplayPlugin::isSupported() const {
-    return !isOculusPresent() && vr::VR_IsHmdPresent();
+    return (enableDebugOpenVR || !isOculusPresent()) && vr::VR_IsHmdPresent();
 }
 
 bool OpenVrDisplayPlugin::internalActivate() {
-    Parent::internalActivate();
-
     _container->setIsOptionChecked(StandingHMDSensorMode, true);
 
     if (!_system) {
@@ -91,7 +94,7 @@ bool OpenVrDisplayPlugin::internalActivate() {
         qDebug() << "OpenVR: error could not get chaperone pointer";
     }
 
-    return true;
+    return Parent::internalActivate();
 }
 
 void OpenVrDisplayPlugin::internalDeactivate() {
@@ -112,6 +115,7 @@ void OpenVrDisplayPlugin::customizeContext() {
         GLenum err = glewInit();
         glGetError(); // clear the potential error from glewExperimental
     });
+
     Parent::customizeContext();
 }
 
@@ -121,41 +125,39 @@ void OpenVrDisplayPlugin::resetSensors() {
     _sensorResetMat = glm::inverse(cancelOutRollAndPitch(m));
 }
 
-void OpenVrDisplayPlugin::beginFrameRender(uint32_t frameIndex) {
 
+void OpenVrDisplayPlugin::beginFrameRender(uint32_t frameIndex) {
     double displayFrequency = _system->GetFloatTrackedDeviceProperty(vr::k_unTrackedDeviceIndex_Hmd, vr::Prop_DisplayFrequency_Float);
     double frameDuration = 1.f / displayFrequency;
     double vsyncToPhotons = _system->GetFloatTrackedDeviceProperty(vr::k_unTrackedDeviceIndex_Hmd, vr::Prop_SecondsFromVsyncToPhotons_Float);
 
-    FrameInfo frame;
+    _currentRenderFrameInfo = FrameInfo();
 #if THREADED_PRESENT
     // 3 frames of prediction + vsyncToPhotons = 44ms total
     const double NUM_PREDICTION_FRAMES = 3.0f;
-    frame.predictedDisplayTime = NUM_PREDICTION_FRAMES * frameDuration + vsyncToPhotons;
+    _currentRenderFrameInfo.predictedDisplayTime = NUM_PREDICTION_FRAMES * frameDuration + vsyncToPhotons;
 #else
-    frame.predictedDisplayTime = frameDuration + vsyncToPhotons;
+    _currentRenderFrameInfo.predictedDisplayTime = frameDuration + vsyncToPhotons;
 #endif
 
-    vr::TrackedDevicePose_t predictedTrackedDevicePose[vr::k_unMaxTrackedDeviceCount];
+    _system->GetDeviceToAbsoluteTrackingPose(vr::TrackingUniverseStanding, _currentRenderFrameInfo.predictedDisplayTime, _trackedDevicePose, vr::k_unMaxTrackedDeviceCount);
-    _system->GetDeviceToAbsoluteTrackingPose(vr::TrackingUniverseStanding, frame.predictedDisplayTime, predictedTrackedDevicePose, vr::k_unMaxTrackedDeviceCount);
 
     // copy and process predictedTrackedDevicePoses
     for (int i = 0; i < vr::k_unMaxTrackedDeviceCount; i++) {
-        _trackedDevicePose[i] = predictedTrackedDevicePose[i];
         _trackedDevicePoseMat4[i] = _sensorResetMat * toGlm(_trackedDevicePose[i].mDeviceToAbsoluteTracking);
         _trackedDeviceLinearVelocities[i] = transformVectorFast(_sensorResetMat, toGlm(_trackedDevicePose[i].vVelocity));
         _trackedDeviceAngularVelocities[i] = transformVectorFast(_sensorResetMat, toGlm(_trackedDevicePose[i].vAngularVelocity));
     }
-    frame.headPose = _trackedDevicePoseMat4[0];
+    _currentRenderFrameInfo.rawRenderPose = toGlm(_trackedDevicePose[vr::k_unTrackedDeviceIndex_Hmd].mDeviceToAbsoluteTracking);
-    _currentRenderFrameInfo.set(frame);
+    _currentRenderFrameInfo.renderPose = _trackedDevicePoseMat4[vr::k_unTrackedDeviceIndex_Hmd];
 
     Lock lock(_mutex);
-    _frameInfos[frameIndex] = frame;
+    _frameInfos[frameIndex] = _currentRenderFrameInfo;
 }
 
 void OpenVrDisplayPlugin::hmdPresent() {
 
-    PROFILE_RANGE_EX(__FUNCTION__, 0xff00ff00, (uint64_t)_currentRenderFrameIndex)
+    PROFILE_RANGE_EX(__FUNCTION__, 0xff00ff00, (uint64_t)_currentPresentFrameIndex)
 
     // Flip y-axis since GL UV coords are backwards.
     static vr::VRTextureBounds_t leftBounds{ 0, 0, 0.5f, 1 };
@@ -168,7 +170,7 @@ void OpenVrDisplayPlugin::hmdPresent() {
 }
 
 void OpenVrDisplayPlugin::postPreview() {
-    PROFILE_RANGE_EX(__FUNCTION__, 0xff00ff00, (uint64_t)_currentRenderFrameIndex)
+    PROFILE_RANGE_EX(__FUNCTION__, 0xff00ff00, (uint64_t)_currentPresentFrameIndex)
 
     vr::TrackedDevicePose_t currentTrackedDevicePose[vr::k_unMaxTrackedDeviceCount];
     _compositor->WaitGetPoses(currentTrackedDevicePose, vr::k_unMaxTrackedDeviceCount, nullptr, 0);
@@ -179,3 +181,21 @@ bool OpenVrDisplayPlugin::isHmdMounted() const {
     return _hmdActivityLevel == vr::k_EDeviceActivityLevel_UserInteraction;
 }
 
+void OpenVrDisplayPlugin::updatePresentPose() {
+    {
+        float fSecondsSinceLastVsync;
+        _system->GetTimeSinceLastVsync(&fSecondsSinceLastVsync, nullptr);
+        float fDisplayFrequency = _system->GetFloatTrackedDeviceProperty(vr::k_unTrackedDeviceIndex_Hmd, vr::Prop_DisplayFrequency_Float);
+        float fFrameDuration = 1.f / fDisplayFrequency;
+        float fVsyncToPhotons = _system->GetFloatTrackedDeviceProperty(vr::k_unTrackedDeviceIndex_Hmd, vr::Prop_SecondsFromVsyncToPhotons_Float);
+        float fPredictedSecondsFromNow = fFrameDuration - fSecondsSinceLastVsync + fVsyncToPhotons;
+        vr::TrackedDevicePose_t pose;
+        _system->GetDeviceToAbsoluteTrackingPose(vr::TrackingUniverseStanding, fPredictedSecondsFromNow, &pose, 1);
+        _currentPresentFrameInfo.rawPresentPose = toGlm(pose.mDeviceToAbsoluteTracking);
+    }
+    _currentPresentFrameInfo.presentPose = _sensorResetMat * _currentPresentFrameInfo.rawPresentPose;
+    mat3 renderRotation(_currentPresentFrameInfo.rawRenderPose);
+    mat3 presentRotation(_currentPresentFrameInfo.rawPresentPose);
+    _currentPresentFrameInfo.presentReprojection = glm::mat3(glm::inverse(renderRotation) * presentRotation);
+}
+

plugins/openvr/src/OpenVrDisplayPlugin.h

@@ -28,10 +28,12 @@ public:
     // Stereo specific methods
     virtual void resetSensors() override;
     virtual void beginFrameRender(uint32_t frameIndex) override;
+    void cycleDebugOutput() override { _lockCurrentTexture = !_lockCurrentTexture; }
 
 protected:
     bool internalActivate() override;
     void internalDeactivate() override;
+    void updatePresentPose() override;
 
     void hmdPresent() override;
     bool isHmdMounted() const override;