From 1a0222088654af0df4792484b99290e81a846ca3 Mon Sep 17 00:00:00 2001 From: Brad Davis Date: Tue, 12 Apr 2016 14:11:36 -0700 Subject: [PATCH] Fixing wobble --- interface/resources/shaders/reproject.frag | 71 +++++++++++++ .../display-plugins/hmd/HmdDisplayPlugin.cpp | 99 +++++++++++++------ .../display-plugins/hmd/HmdDisplayPlugin.h | 18 ++-- plugins/openvr/src/OpenVrDisplayPlugin.cpp | 32 +++--- 4 files changed, 172 insertions(+), 48 deletions(-) create mode 100644 interface/resources/shaders/reproject.frag diff --git a/interface/resources/shaders/reproject.frag b/interface/resources/shaders/reproject.frag new file mode 100644 index 0000000000..c3afe58f37 --- /dev/null +++ b/interface/resources/shaders/reproject.frag @@ -0,0 +1,71 @@ +#version 450 core + +uniform sampler2D sampler; +layout (location = 0) uniform mat3 reprojection = mat3(1); +layout (location = 4) uniform mat4 inverseProjections[2]; +layout (location = 12) uniform mat4 projections[2]; + +in vec2 vTexCoord; +in vec3 vPosition; + +out vec4 FragColor; + +void main() { + vec2 uv = vTexCoord; + + mat4 eyeInverseProjection; + mat4 eyeProjection; + + float xoffset = 1.0; + vec2 uvmin = vec2(0.0); + vec2 uvmax = vec2(1.0); + // determine the correct projection and inverse projection to use. + if (vTexCoord.x < 0.5) { + uvmax.x = 0.5; + eyeInverseProjection = inverseProjections[0]; + eyeProjection = projections[0]; + } else { + xoffset = -1.0; + uvmin.x = 0.5; + uvmax.x = 1.0; + eyeInverseProjection = inverseProjections[1]; + eyeProjection = projections[1]; + } + + // Account for stereo in calculating the per-eye NDC coordinates + vec4 ndcSpace = vec4(vPosition, 1.0); + ndcSpace.x *= 2.0; + ndcSpace.x += xoffset; + + // Convert from NDC to eyespace + vec4 eyeSpace = eyeInverseProjection * ndcSpace; + eyeSpace /= eyeSpace.w; + + // Convert to a noramlized ray + vec3 ray = eyeSpace.xyz; + ray = normalize(ray); + + // Adjust the ray by the rotation + ray = reprojection * ray; + + // Project back on to the texture plane + ray /= ray.z; + ray *= eyeSpace.z; + + // Update the eyespace vector + eyeSpace.xyz = ray; + + // Reproject back into NDC + ndcSpace = eyeProjection * eyeSpace; + ndcSpace /= ndcSpace.w; + ndcSpace.x -= xoffset; + ndcSpace.x /= 2.0; + + // Calculate the new UV coordinates + uv = (ndcSpace.xy / 2.0) + 0.5; + if (any(greaterThan(uv, uvmax)) || any(lessThan(uv, uvmin))) { + FragColor = vec4(0.0, 0.0, 0.0, 1.0); + } else { + FragColor = texture(sampler, uv); + } +} diff --git a/libraries/display-plugins/src/display-plugins/hmd/HmdDisplayPlugin.cpp b/libraries/display-plugins/src/display-plugins/hmd/HmdDisplayPlugin.cpp index 6011bc2906..34e484a988 100644 --- a/libraries/display-plugins/src/display-plugins/hmd/HmdDisplayPlugin.cpp +++ b/libraries/display-plugins/src/display-plugins/hmd/HmdDisplayPlugin.cpp @@ -25,10 +25,11 @@ #include "../CompositorHelper.h" static const QString MONO_PREVIEW = "Mono Preview"; +static const QString REPROJECTION = "Allow Reprojection"; static const QString FRAMERATE = DisplayPlugin::MENU_PATH() + ">Framerate"; +static const QString DEVELOPER_MENU_PATH = "Developer>" + DisplayPlugin::MENU_PATH(); static const bool DEFAULT_MONO_VIEW = true; - glm::uvec2 HmdDisplayPlugin::getRecommendedUiSize() const { return CompositorHelper::VIRTUAL_SCREEN_SIZE; } @@ -42,7 +43,13 @@ bool HmdDisplayPlugin::internalActivate() { _container->setBoolSetting("monoPreview", _monoPreview); }, true, _monoPreview); _container->removeMenu(FRAMERATE); - + _container->addMenu(DEVELOPER_MENU_PATH); + _container->addMenuItem(PluginType::DISPLAY_PLUGIN, DEVELOPER_MENU_PATH, REPROJECTION, + [this](bool clicked) { + _enableReprojection = clicked; + _container->setBoolSetting("enableReprojection", _enableReprojection); + }, true, _enableReprojection); + for_each_eye([&](Eye eye) { _eyeInverseProjections[eye] = glm::inverse(_eyeProjections[eye]); }); @@ -70,6 +77,7 @@ static const GLint REPROJECTION_MATRIX_LOCATION = 0; static const GLint INVERSE_PROJECTION_MATRIX_LOCATION = 4; static const GLint PROJECTION_MATRIX_LOCATION = 12; static const char * REPROJECTION_FS = R"FS(#version 450 core + uniform sampler2D sampler; layout (location = 0) uniform mat3 reprojection = mat3(1); layout (location = 4) uniform mat4 inverseProjections[2]; @@ -81,20 +89,10 @@ in vec3 vPosition; out vec4 FragColor; void main() { - vec2 uv = vTexCoord; - vec3 Z_AXIS = vec3(0.0, 0.0, -1.0); - vec3 rotated = reprojection * Z_AXIS; - float angle = acos(dot(Z_AXIS, rotated)); - if (angle < 0.001) { - FragColor = texture(sampler, uv); - return; - } - mat4 eyeInverseProjection; mat4 eyeProjection; - float xoffset = 1.0; vec2 uvmin = vec2(0.0); @@ -127,10 +125,12 @@ void main() { // Adjust the ray by the rotation ray = reprojection * ray; - + // Project back on to the texture plane - eyeSpace.xyz = ray * eyeSpace.z; - //eyeSpace.xyz = ray; + ray *= eyeSpace.z / ray.z; + + // Update the eyespace vector + eyeSpace.xyz = ray; // Reproject back into NDC ndcSpace = eyeProjection * eyeSpace; @@ -148,6 +148,50 @@ void main() { } )FS"; +#ifdef DEBUG_REPROJECTION_SHADER +#include +#include +#include +#include + +static const QString REPROJECTION_FS_FILE = "c:/Users/bdavis/Git/hifi/interface/resources/shaders/reproject.frag"; + +static ProgramPtr getReprojectionProgram() { + static ProgramPtr _currentProgram; + uint64_t now = usecTimestampNow(); + static uint64_t _lastFileCheck = now; + + bool modified = false; + if ((now - _lastFileCheck) > USECS_PER_MSEC * 100) { + QFileInfo info(REPROJECTION_FS_FILE); + QDateTime lastModified = info.lastModified(); + static QDateTime _lastModified = lastModified; + qDebug() << lastModified.toTime_t(); + qDebug() << _lastModified.toTime_t(); + if (lastModified > _lastModified) { + _lastModified = lastModified; + modified = true; + } + } + + if (!_currentProgram || modified) { + _currentProgram.reset(); + try { + QFile shaderFile(REPROJECTION_FS_FILE); + shaderFile.open(QIODevice::ReadOnly); + QString fragment = shaderFile.readAll(); + compileProgram(_currentProgram, REPROJECTION_VS, fragment.toLocal8Bit().data()); + } catch (const std::runtime_error& error) { + qDebug() << "Failed to build: " << error.what(); + } + if (!_currentProgram) { + _currentProgram = loadDefaultShader(); + } + } + return _currentProgram; +} +#endif + void HmdDisplayPlugin::customizeContext() { Parent::customizeContext(); @@ -161,6 +205,7 @@ void HmdDisplayPlugin::customizeContext() { void HmdDisplayPlugin::uncustomizeContext() { _sphereSection.reset(); _compositeFramebuffer.reset(); + _reprojectionProgram.reset(); Parent::uncustomizeContext(); } @@ -169,31 +214,27 @@ void HmdDisplayPlugin::updatePresentPose() { _currentPresentFrameInfo.presentPose = _currentPresentFrameInfo.renderPose; } -glm::mat3 HmdDisplayPlugin::FrameInfo::presentRotation() const { - if (renderPose == presentPose) { - return glm::mat3(); - } - quat renderRotation = glm::quat_cast(renderPose); - quat presentRotation = glm::quat_cast(presentPose); - quat reprojection = glm::inverse(renderRotation) * presentRotation; - return glm::mat3_cast(reprojection); -} - void HmdDisplayPlugin::compositeScene() { updatePresentPose(); - glm::mat3 reprojection = _currentPresentFrameInfo.presentRotation(); - if (glm::mat3() == reprojection) { + if (!_enableReprojection || glm::mat3() == _currentPresentFrameInfo.presentReprojection) { // No reprojection required Parent::compositeScene(); return; } +#ifdef DEBUG_REPROJECTION_SHADER + _reprojectionProgram = getReprojectionProgram(); +#endif useProgram(_reprojectionProgram); using namespace oglplus; - Uniform(*_reprojectionProgram, REPROJECTION_MATRIX_LOCATION).Set(reprojection); - // FIXME what's the right oglplus mechanism to do this? + Texture::MinFilter(TextureTarget::_2D, TextureMinFilter::Linear); + Texture::MagFilter(TextureTarget::_2D, TextureMagFilter::Linear); + Uniform(*_reprojectionProgram, REPROJECTION_MATRIX_LOCATION).Set(_currentPresentFrameInfo.presentReprojection); + //Uniform(*_reprojectionProgram, PROJECTION_MATRIX_LOCATION).Set(_eyeProjections); + //Uniform(*_reprojectionProgram, INVERSE_PROJECTION_MATRIX_LOCATION).Set(_eyeInverseProjections); + // FIXME what's the right oglplus mechanism to do this? It's not that ^^^ ... better yet, switch to a uniform buffer glUniformMatrix4fv(INVERSE_PROJECTION_MATRIX_LOCATION, 2, GL_FALSE, &(_eyeInverseProjections[0][0][0])); glUniformMatrix4fv(PROJECTION_MATRIX_LOCATION, 2, GL_FALSE, &(_eyeProjections[0][0][0])); _plane->UseInProgram(*_reprojectionProgram); diff --git a/libraries/display-plugins/src/display-plugins/hmd/HmdDisplayPlugin.h b/libraries/display-plugins/src/display-plugins/hmd/HmdDisplayPlugin.h index a0f6f87d11..d16f03112f 100644 --- a/libraries/display-plugins/src/display-plugins/hmd/HmdDisplayPlugin.h +++ b/libraries/display-plugins/src/display-plugins/hmd/HmdDisplayPlugin.h @@ -28,13 +28,6 @@ public: virtual glm::mat4 getHeadPose() const override; - struct FrameInfo { - glm::mat4 renderPose; - glm::mat4 presentPose; - double sensorSampleTime { 0 }; - double predictedDisplayTime { 0 }; - glm::mat3 presentRotation() const; - }; protected: @@ -60,6 +53,16 @@ protected: glm::uvec2 _renderTargetSize; float _ipd { 0.064f }; + struct FrameInfo { + glm::mat4 rawRenderPose; + glm::mat4 renderPose; + glm::mat4 rawPresentPose; + glm::mat4 presentPose; + double sensorSampleTime { 0 }; + double predictedDisplayTime { 0 }; + glm::mat3 presentReprojection; + }; + QMap _frameInfos; FrameInfo _currentPresentFrameInfo; FrameInfo _currentRenderFrameInfo; @@ -67,6 +70,7 @@ protected: private: bool _enablePreview { false }; bool _monoPreview { true }; + bool _enableReprojection { true }; ShapeWrapperPtr _sphereSection; ProgramPtr _reprojectionProgram; }; diff --git a/plugins/openvr/src/OpenVrDisplayPlugin.cpp b/plugins/openvr/src/OpenVrDisplayPlugin.cpp index 7c26e998e6..956c5e8e1a 100644 --- a/plugins/openvr/src/OpenVrDisplayPlugin.cpp +++ b/plugins/openvr/src/OpenVrDisplayPlugin.cpp @@ -125,8 +125,8 @@ void OpenVrDisplayPlugin::resetSensors() { _sensorResetMat = glm::inverse(cancelOutRollAndPitch(m)); } -void OpenVrDisplayPlugin::beginFrameRender(uint32_t frameIndex) { +void OpenVrDisplayPlugin::beginFrameRender(uint32_t frameIndex) { double displayFrequency = _system->GetFloatTrackedDeviceProperty(vr::k_unTrackedDeviceIndex_Hmd, vr::Prop_DisplayFrequency_Float); double frameDuration = 1.f / displayFrequency; double vsyncToPhotons = _system->GetFloatTrackedDeviceProperty(vr::k_unTrackedDeviceIndex_Hmd, vr::Prop_SecondsFromVsyncToPhotons_Float); @@ -146,11 +146,12 @@ void OpenVrDisplayPlugin::beginFrameRender(uint32_t frameIndex) { // copy and process predictedTrackedDevicePoses for (int i = 0; i < vr::k_unMaxTrackedDeviceCount; i++) { _trackedDevicePose[i] = predictedTrackedDevicePose[i]; - _trackedDevicePoseMat4[i] = _sensorResetMat * toGlm(_trackedDevicePose[i].mDeviceToAbsoluteTracking); + _trackedDevicePoseMat4[i] = toGlm(_trackedDevicePose[i].mDeviceToAbsoluteTracking); _trackedDeviceLinearVelocities[i] = transformVectorFast(_sensorResetMat, toGlm(_trackedDevicePose[i].vVelocity)); _trackedDeviceAngularVelocities[i] = transformVectorFast(_sensorResetMat, toGlm(_trackedDevicePose[i].vAngularVelocity)); } - _currentRenderFrameInfo.renderPose = _trackedDevicePoseMat4[0]; + _currentRenderFrameInfo.rawRenderPose = _trackedDevicePoseMat4[vr::k_unTrackedDeviceIndex_Hmd]; + _currentRenderFrameInfo.renderPose = _sensorResetMat * _currentRenderFrameInfo.rawRenderPose; Lock lock(_mutex); _frameInfos[frameIndex] = _currentRenderFrameInfo; @@ -183,13 +184,20 @@ bool OpenVrDisplayPlugin::isHmdMounted() const { } void OpenVrDisplayPlugin::updatePresentPose() { - float fSecondsSinceLastVsync; - _system->GetTimeSinceLastVsync(&fSecondsSinceLastVsync, nullptr); - float fDisplayFrequency = _system->GetFloatTrackedDeviceProperty(vr::k_unTrackedDeviceIndex_Hmd, vr::Prop_DisplayFrequency_Float); - float fFrameDuration = 1.f / fDisplayFrequency; - float fVsyncToPhotons = _system->GetFloatTrackedDeviceProperty(vr::k_unTrackedDeviceIndex_Hmd, vr::Prop_SecondsFromVsyncToPhotons_Float); - float fPredictedSecondsFromNow = fFrameDuration - fSecondsSinceLastVsync + fVsyncToPhotons; - vr::TrackedDevicePose_t presentPoseOpenVR; - _system->GetDeviceToAbsoluteTrackingPose(vr::TrackingUniverseStanding, fPredictedSecondsFromNow, &presentPoseOpenVR, 1); - _currentPresentFrameInfo.presentPose = _sensorResetMat * toGlm(presentPoseOpenVR.mDeviceToAbsoluteTracking); + { + float fSecondsSinceLastVsync; + _system->GetTimeSinceLastVsync(&fSecondsSinceLastVsync, nullptr); + float fDisplayFrequency = _system->GetFloatTrackedDeviceProperty(vr::k_unTrackedDeviceIndex_Hmd, vr::Prop_DisplayFrequency_Float); + float fFrameDuration = 1.f / fDisplayFrequency; + float fVsyncToPhotons = _system->GetFloatTrackedDeviceProperty(vr::k_unTrackedDeviceIndex_Hmd, vr::Prop_SecondsFromVsyncToPhotons_Float); + float fPredictedSecondsFromNow = fFrameDuration - fSecondsSinceLastVsync + fVsyncToPhotons; + vr::TrackedDevicePose_t pose; + _system->GetDeviceToAbsoluteTrackingPose(vr::TrackingUniverseStanding, fPredictedSecondsFromNow, &pose, 1); + _currentPresentFrameInfo.rawPresentPose = toGlm(pose.mDeviceToAbsoluteTracking); + } + _currentPresentFrameInfo.presentPose = _sensorResetMat * _currentPresentFrameInfo.rawPresentPose; + mat3 renderRotation(_currentPresentFrameInfo.rawRenderPose); + mat3 presentRotation(_currentPresentFrameInfo.rawPresentPose); + _currentPresentFrameInfo.presentReprojection = glm::mat3(glm::inverse(renderRotation) * presentRotation); } +