Add debugging output, fix timewarp

interface/src/Application.cpp

@@ -1490,11 +1490,15 @@ void Application::paintGL() {
     // FIXME not needed anymore?
     _offscreenContext->makeCurrent();
 
-    displayPlugin->updateHeadPose(_frameCount);
+    displayPlugin->beginFrameRender(_frameCount);
 
     // update the avatar with a fresh HMD pose
     getMyAvatar()->updateFromHMDSensorMatrix(getHMDSensorPose());
 
+    // update sensorToWorldMatrix for camera and hand controllers
+    getMyAvatar()->updateSensorToWorldMatrix();
+
+
     auto lodManager = DependencyManager::get<LODManager>();
 
 
@@ -2007,6 +2011,12 @@ void Application::keyPressEvent(QKeyEvent* event) {
                 }
                 break;
 
+            case Qt::Key_Y:
+                if (isShifted && isMeta) {
+                    getActiveDisplayPlugin()->cycleDebugOutput();
+                }
+                break;
+
             case Qt::Key_B:
                 if (isMeta) {
                     auto offscreenUi = DependencyManager::get<OffscreenUi>();
@@ -2572,11 +2582,6 @@ void Application::idle(uint64_t now) {
         return; // bail early, nothing to do here.
     }
 
-    checkChangeCursor();
-
-    Stats::getInstance()->updateStats();
-    AvatarInputs::getInstance()->update();
-
     // These tasks need to be done on our first idle, because we don't want the showing of
     // overlay subwindows to do a showDesktop() until after the first time through
     static bool firstIdle = true;
@@ -2625,6 +2630,11 @@ void Application::idle(uint64_t now) {
     // We're going to execute idle processing, so restart the last idle timer
     _lastTimeUpdated.start();
 
+    checkChangeCursor();
+
+    Stats::getInstance()->updateStats();
+    AvatarInputs::getInstance()->update();
+
     {
         static uint64_t lastIdleStart{ now };
         uint64_t idleStartToStartDuration = now - lastIdleStart;
@@ -3389,9 +3399,6 @@ void Application::update(float deltaTime) {
 
         qApp->updateMyAvatarLookAtPosition();
 
-        // update sensorToWorldMatrix for camera and hand controllers
-        myAvatar->updateSensorToWorldMatrix();
-
         {
             PROFILE_RANGE_EX("MyAvatar", 0xffff00ff, (uint64_t)getActiveDisplayPlugin()->presentCount());
             avatarManager->updateMyAvatar(deltaTime);

libraries/display-plugins/src/display-plugins/OpenGLDisplayPlugin.cpp

@@ -552,9 +552,9 @@ float OpenGLDisplayPlugin::presentRate() {
     {
         Lock lock(_mutex);
         result = _usecsPerFrame.getAverage();
-        result = 1.0f / result; 
-        result *= USECS_PER_SECOND;
     }
+    result = 1.0f / result; 
+    result *= USECS_PER_SECOND;
     return result;
 }
 

libraries/display-plugins/src/display-plugins/OpenGLDisplayPlugin.h

@@ -17,9 +17,9 @@
 #include <GLMHelpers.h>
 #include <SimpleMovingAverage.h>
 #include <gl/OglplusHelpers.h>
+#include <gl/GLEscrow.h>
 
 #define THREADED_PRESENT 1
-#include <gl/GLEscrow.h>
 
 class OpenGLDisplayPlugin : public DisplayPlugin {
 protected:

libraries/display-plugins/src/display-plugins/hmd/HmdDisplayPlugin.cpp

@@ -69,10 +69,11 @@ void HmdDisplayPlugin::compositeOverlay() {
         // set the alpha
         Uniform<float>(*_program, _alphaUniform).Set(overlayAlpha);
 
+        auto eyePoses = _currentPresentFrameInfo.eyePoses;
         _sphereSection->Use();
         for_each_eye([&](Eye eye) {
             eyeViewport(eye);
-            auto modelView = glm::inverse(_currentRenderEyePoses[eye]); // *glm::translate(mat4(), vec3(0, 0, -1));
+            auto modelView = glm::inverse(eyePoses[eye]); // *glm::translate(mat4(), vec3(0, 0, -1));
             auto mvp = _eyeProjections[eye] * modelView;
             Uniform<glm::mat4>(*_program, _mvpUniform).Set(mvp);
             _sphereSection->Draw();
@@ -95,10 +96,10 @@ void HmdDisplayPlugin::compositePointer() {
         // Mouse pointer
         _plane->Use();
         // Reconstruct the headpose from the eye poses
-        auto headPosition = (vec3(_currentRenderEyePoses[Left][3]) + vec3(_currentRenderEyePoses[Right][3])) / 2.0f;
+        auto headPosition = vec3(_currentPresentFrameInfo.headPose[3]);
         for_each_eye([&](Eye eye) {
             eyeViewport(eye);
-            auto reticleTransform = compositorHelper->getReticleTransform(_currentRenderEyePoses[eye], headPosition);
+            auto reticleTransform = compositorHelper->getReticleTransform(_currentPresentFrameInfo.eyePoses[eye], headPosition);
             auto mvp = _eyeProjections[eye] * reticleTransform;
             Uniform<glm::mat4>(*_program, _mvpUniform).Set(mvp);
             _plane->Draw();
@@ -160,15 +161,28 @@ void HmdDisplayPlugin::internalPresent() {
 
 void HmdDisplayPlugin::setEyeRenderPose(uint32_t frameIndex, Eye eye, const glm::mat4& pose) {
     Lock lock(_mutex);
-    _renderEyePoses[frameIndex][eye] = pose;
+    FrameInfo& frame = _frameInfos[frameIndex];
+    frame.eyePoses[eye] = pose;
 }
 
 void HmdDisplayPlugin::updateFrameData() {
+    // Check if we have old frame data to discard
+    {
+        Lock lock(_mutex);
+        auto itr = _frameInfos.find(_currentRenderFrameIndex);
+        if (itr != _frameInfos.end()) {
+            _frameInfos.erase(itr);
+        }
+    }
+
     Parent::updateFrameData();
-    Lock lock(_mutex);
-    _currentRenderEyePoses = _renderEyePoses[_currentRenderFrameIndex];
+
+    {
+        Lock lock(_mutex);
+        _currentPresentFrameInfo = _frameInfos[_currentRenderFrameIndex];
+    }
 }
 
 glm::mat4 HmdDisplayPlugin::getHeadPose() const {
-    return _headPoseCache.get();
+    return _currentRenderFrameInfo.get().headPose;
 }

libraries/display-plugins/src/display-plugins/hmd/HmdDisplayPlugin.h

@@ -28,6 +28,16 @@ public:
 
     virtual glm::mat4 getHeadPose() const override;
 
+    using EyePoses = std::array<glm::mat4, 2>;
+
+    struct FrameInfo {
+        EyePoses eyePoses;
+        glm::mat4 headPose;
+        double sensorSampleTime { 0 };
+        double predictedDisplayTime { 0 };
+    };
+
+
 protected:
     virtual void hmdPresent() = 0;
     virtual bool isHmdMounted() const = 0;
@@ -46,10 +56,10 @@ protected:
     glm::mat4 _cullingProjection;
     glm::uvec2 _renderTargetSize;
     float _ipd { 0.064f };
-    using EyePoses = std::array<glm::mat4, 2>;
-    QMap<uint32_t, EyePoses> _renderEyePoses;
-    EyePoses _currentRenderEyePoses;
-    ThreadSafeValueCache<glm::mat4> _headPoseCache { glm::mat4() };
+
+    QMap<uint32_t, FrameInfo> _frameInfos;
+    FrameInfo _currentPresentFrameInfo;
+    ThreadSafeValueCache<FrameInfo> _currentRenderFrameInfo;
 
 private:
     bool _enablePreview { false };

libraries/plugins/src/plugins/DisplayPlugin.h

@@ -122,7 +122,7 @@ public:
     }
 
     // will query the underlying hmd api to compute the most recent head pose
-    virtual void updateHeadPose(uint32_t frameIndex) {}
+    virtual void beginFrameRender(uint32_t frameIndex) {}
 
     // returns a copy of the most recent head pose, computed via updateHeadPose
     virtual glm::mat4 getHeadPose() const {
@@ -142,6 +142,8 @@ public:
     virtual float presentRate() { return -1.0f; }
     uint32_t presentCount() const { return _presentedFrameIndex; }
 
+    virtual void cycleDebugOutput() {}
+
     static const QString& MENU_PATH();
 
 signals:

libraries/shared/src/ThreadSafeValueCache.h

@@ -23,6 +23,7 @@
 template <typename T>
 class ThreadSafeValueCache {
 public:
+    ThreadSafeValueCache() {}
     ThreadSafeValueCache(const T& v) : _value { v } {}
 
     // returns atomic copy of the cached value.

plugins/oculus/src/OculusBaseDisplayPlugin.cpp

@@ -15,11 +15,16 @@ void OculusBaseDisplayPlugin::resetSensors() {
     ovr_RecenterTrackingOrigin(_session);
 }
 
-void OculusBaseDisplayPlugin::updateHeadPose(uint32_t frameIndex) {
-    auto displayTime = ovr_GetPredictedDisplayTime(_session, frameIndex);
-    auto trackingState = ovr_GetTrackingState(_session, displayTime, true);
-    mat4 headPose = toGlm(trackingState.HeadPose.ThePose);
-    _headPoseCache.set(headPose);
+void OculusBaseDisplayPlugin::beginFrameRender(uint32_t frameIndex) {
+    FrameInfo frame;
+    frame.sensorSampleTime = ovr_GetTimeInSeconds();;
+    frame.predictedDisplayTime = ovr_GetPredictedDisplayTime(_session, frameIndex);
+    auto trackingState = ovr_GetTrackingState(_session, frame.predictedDisplayTime, ovrTrue);
+    frame.headPose = toGlm(trackingState.HeadPose.ThePose);
+
+    _currentRenderFrameInfo.set(frame);
+    Lock lock(_mutex);
+    _frameInfos[frameIndex] = frame;
 }
 
 bool OculusBaseDisplayPlugin::isSupported() const {

plugins/oculus/src/OculusBaseDisplayPlugin.h

@@ -20,7 +20,8 @@ public:
 
     // Stereo specific methods
     virtual void resetSensors() override final;
-    virtual void updateHeadPose(uint32_t frameIndex) override;
+    virtual void beginFrameRender(uint32_t frameIndex) override;
+    
 
 protected:
     void customizeContext() override;

plugins/oculus/src/OculusDisplayPlugin.cpp

@@ -10,6 +10,23 @@
 #include "OculusHelpers.h"
 
 const QString OculusDisplayPlugin::NAME("Oculus Rift");
+static ovrPerfHudMode currentDebugMode = ovrPerfHud_Off;
+
+bool OculusDisplayPlugin::internalActivate() {
+    bool result = Parent::internalActivate();
+    currentDebugMode = ovrPerfHud_Off;
+    if (result && _session) {
+        ovr_SetInt(_session, OVR_PERF_HUD_MODE, currentDebugMode);
+    }
+    return result;
+}
+
+void OculusDisplayPlugin::cycleDebugOutput() {
+    if (_session) {
+        currentDebugMode = static_cast<ovrPerfHudMode>((currentDebugMode + 1) % ovrPerfHud_Count);
+        ovr_SetInt(_session, OVR_PERF_HUD_MODE, currentDebugMode);
+    }
+}
 
 void OculusDisplayPlugin::customizeContext() {
     Parent::customizeContext();
@@ -48,12 +65,6 @@ void blit(const SrcFbo& srcFbo, const DstFbo& dstFbo) {
     });
 }
 
-void OculusDisplayPlugin::updateFrameData() {
-    Parent::updateFrameData();
-    _sceneLayer.RenderPose[ovrEyeType::ovrEye_Left] = ovrPoseFromGlm(_currentRenderEyePoses[Left]);
-    _sceneLayer.RenderPose[ovrEyeType::ovrEye_Right] = ovrPoseFromGlm(_currentRenderEyePoses[Right]);
-}
-
 void OculusDisplayPlugin::hmdPresent() {
 
     PROFILE_RANGE_EX(__FUNCTION__, 0xff00ff00, (uint64_t)_currentRenderFrameIndex)
@@ -63,12 +74,15 @@ void OculusDisplayPlugin::hmdPresent() {
     }
 
     blit(_compositeFramebuffer, _sceneFbo);
+    _sceneFbo->Commit();
     {
+        _sceneLayer.SensorSampleTime = _currentPresentFrameInfo.sensorSampleTime;
+        _sceneLayer.RenderPose[ovrEyeType::ovrEye_Left] = ovrPoseFromGlm(_currentPresentFrameInfo.headPose);
+        _sceneLayer.RenderPose[ovrEyeType::ovrEye_Right] = ovrPoseFromGlm(_currentPresentFrameInfo.headPose);
         ovrLayerHeader* layers = &_sceneLayer.Header;
         ovrResult result = ovr_SubmitFrame(_session, _currentRenderFrameIndex, &_viewScaleDesc, &layers, 1);
         if (!OVR_SUCCESS(result)) {
             logWarning("Failed to present");
         }
     }
-    _sceneFbo->Commit();
 }

plugins/oculus/src/OculusDisplayPlugin.h

@@ -22,12 +22,13 @@ public:
     float getTargetFrameRate() override { return TARGET_RATE_Oculus; }
 
 protected:
+    bool internalActivate() override;
     void hmdPresent() override;
     // FIXME update with Oculus API call once it's available in the SDK
     bool isHmdMounted() const override { return true; }
     void customizeContext() override;
     void uncustomizeContext() override;
-    void updateFrameData() override;
+    void cycleDebugOutput() override;
 
 private:
     static const QString NAME;

plugins/oculus/src/OculusHelpers.cpp

@@ -116,7 +116,8 @@ SwapFramebufferWrapper::~SwapFramebufferWrapper() {
 }
 
 void SwapFramebufferWrapper::Commit() {
-    ovr_CommitTextureSwapChain(_session, color);
+    auto result = ovr_CommitTextureSwapChain(_session, color);
+    Q_ASSERT(OVR_SUCCESS(result));
 }
 
 void SwapFramebufferWrapper::Resize(const uvec2 & size) {

plugins/oculusLegacy/src/OculusLegacyDisplayPlugin.cpp

@@ -35,10 +35,14 @@ void OculusLegacyDisplayPlugin::resetSensors() {
     ovrHmd_RecenterPose(_hmd);
 }
 
-void OculusLegacyDisplayPlugin::updateHeadPose(uint32_t frameIndex) {
+void OculusLegacyDisplayPlugin::beginFrameRender(uint32_t frameIndex) {
+    FrameInfo frame;
+    frame.predictedDisplayTime = frame.sensorSampleTime = ovr_GetTimeInSeconds();
+    _trackingState = ovrHmd_GetTrackingState(_hmd, frame.predictedDisplayTime);
+    frame.headPose = toGlm(_trackingState.HeadPose.ThePose);
+    _currentRenderFrameInfo.set(frame);
     Lock lock(_mutex);
-    _trackingState = ovrHmd_GetTrackingState(_hmd, ovr_GetTimeInSeconds());
-    _headPoseCache.set(toGlm(_trackingState.HeadPose.ThePose));
+    _frameInfos[frameIndex] = frame;
 }
 
 bool OculusLegacyDisplayPlugin::isSupported() const {

plugins/oculusLegacy/src/OculusLegacyDisplayPlugin.h

@@ -26,7 +26,7 @@ public:
 
     // Stereo specific methods
     virtual void resetSensors() override;
-    virtual void updateHeadPose(uint32_t frameIndex) override;
+    virtual void beginFrameRender(uint32_t frameIndex) override;
 
     virtual float getTargetFrameRate() override;
 

plugins/openvr/src/OpenVrDisplayPlugin.cpp

@@ -121,22 +121,23 @@ void OpenVrDisplayPlugin::resetSensors() {
     _sensorResetMat = glm::inverse(cancelOutRollAndPitch(m));
 }
 
-void OpenVrDisplayPlugin::updateHeadPose(uint32_t frameIndex) {
+void OpenVrDisplayPlugin::beginFrameRender(uint32_t frameIndex) {
 
-    float displayFrequency = _system->GetFloatTrackedDeviceProperty(vr::k_unTrackedDeviceIndex_Hmd, vr::Prop_DisplayFrequency_Float);
-    float frameDuration = 1.f / displayFrequency;
-    float vsyncToPhotons = _system->GetFloatTrackedDeviceProperty(vr::k_unTrackedDeviceIndex_Hmd, vr::Prop_SecondsFromVsyncToPhotons_Float);
+    double displayFrequency = _system->GetFloatTrackedDeviceProperty(vr::k_unTrackedDeviceIndex_Hmd, vr::Prop_DisplayFrequency_Float);
+    double frameDuration = 1.f / displayFrequency;
+    double vsyncToPhotons = _system->GetFloatTrackedDeviceProperty(vr::k_unTrackedDeviceIndex_Hmd, vr::Prop_SecondsFromVsyncToPhotons_Float);
 
+    FrameInfo frame;
 #if THREADED_PRESENT
     // 3 frames of prediction + vsyncToPhotons = 44ms total
-    const float NUM_PREDICTION_FRAMES = 3.0f;
-    float predictedSecondsFromNow = NUM_PREDICTION_FRAMES * frameDuration + vsyncToPhotons;
+    const double NUM_PREDICTION_FRAMES = 3.0f;
+    frame.predictedDisplayTime = NUM_PREDICTION_FRAMES * frameDuration + vsyncToPhotons;
 #else
-    float predictedSecondsFromNow = frameDuration + vsyncToPhotons;
+    frame.predictedDisplayTime = frameDuration + vsyncToPhotons;
 #endif
 
     vr::TrackedDevicePose_t predictedTrackedDevicePose[vr::k_unMaxTrackedDeviceCount];
-    _system->GetDeviceToAbsoluteTrackingPose(vr::TrackingUniverseStanding, predictedSecondsFromNow, predictedTrackedDevicePose, vr::k_unMaxTrackedDeviceCount);
+    _system->GetDeviceToAbsoluteTrackingPose(vr::TrackingUniverseStanding, frame.predictedDisplayTime, predictedTrackedDevicePose, vr::k_unMaxTrackedDeviceCount);
 
     // copy and process predictedTrackedDevicePoses
     for (int i = 0; i < vr::k_unMaxTrackedDeviceCount; i++) {
@@ -145,8 +146,11 @@ void OpenVrDisplayPlugin::updateHeadPose(uint32_t frameIndex) {
         _trackedDeviceLinearVelocities[i] = transformVectorFast(_sensorResetMat, toGlm(_trackedDevicePose[i].vVelocity));
         _trackedDeviceAngularVelocities[i] = transformVectorFast(_sensorResetMat, toGlm(_trackedDevicePose[i].vAngularVelocity));
     }
+    frame.headPose = _trackedDevicePoseMat4[0];
+    _currentRenderFrameInfo.set(frame);
 
-    _headPoseCache.set(_trackedDevicePoseMat4[0]);
+    Lock lock(_mutex);
+    _frameInfos[frameIndex] = frame;
 }
 
 void OpenVrDisplayPlugin::hmdPresent() {

plugins/openvr/src/OpenVrDisplayPlugin.h

@@ -27,7 +27,7 @@ public:
 
     // Stereo specific methods
     virtual void resetSensors() override;
-    virtual void updateHeadPose(uint32_t frameIndex) override;
+    virtual void beginFrameRender(uint32_t frameIndex) override;
 
 protected:
     bool internalActivate() override;