Merge pull request #8043 from hyperlogic/bug-fix/hmd-ui-center-improvements

HMD UI re-centering while driving improvements.

interface/src/Application.cpp

@@ -4208,6 +4208,7 @@ void Application::resetSensors(bool andReload) {
     DependencyManager::get<DdeFaceTracker>()->reset();
     DependencyManager::get<EyeTracker>()->reset();
     getActiveDisplayPlugin()->resetSensors();
+    _overlayConductor.centerUI();
     getMyAvatar()->reset(andReload);
     QMetaObject::invokeMethod(DependencyManager::get<AudioClient>().data(), "reset", Qt::QueuedConnection);
 }

interface/src/Menu.cpp

@@ -256,8 +256,7 @@ Menu::Menu() {
         UNSPECIFIED_POSITION, "Advanced");
 
     // View > Overlays
-    addCheckableActionToQMenuAndActionHash(viewMenu, MenuOption::Overlays, 0, true,
+    addCheckableActionToQMenuAndActionHash(viewMenu, MenuOption::Overlays, 0, true);
-        qApp, SLOT(setOverlaysVisible(bool)));
 
     // Navigate menu ----------------------------------
     MenuWrapper* navigateMenu = addMenu("Navigate");

interface/src/avatar/MyAvatar.cpp

@@ -1246,8 +1246,7 @@ void MyAvatar::prepareForPhysicsSimulation() {
 
     _characterController.setPositionAndOrientation(getPosition(), getOrientation());
     if (qApp->isHMDMode()) {
-        bool hasDriveInput = fabsf(_driveKeys[TRANSLATE_X]) > 0.0f || fabsf(_driveKeys[TRANSLATE_Z]) > 0.0f;
+        _follow.prePhysicsUpdate(*this, deriveBodyFromHMDSensor(), _bodySensorMatrix, hasDriveInput());
-        _follow.prePhysicsUpdate(*this, deriveBodyFromHMDSensor(), _bodySensorMatrix, hasDriveInput);
     } else {
         _follow.deactivate();
     }
@@ -2132,3 +2131,7 @@ bool MyAvatar::didTeleport() {
     lastPosition = pos;
     return (changeInPosition.length() > MAX_AVATAR_MOVEMENT_PER_FRAME);
 }
+
+bool MyAvatar::hasDriveInput() const {
+    return fabsf(_driveKeys[TRANSLATE_X]) > 0.0f || fabsf(_driveKeys[TRANSLATE_Y]) > 0.0f || fabsf(_driveKeys[TRANSLATE_Z]) > 0.0f;
+}

interface/src/avatar/MyAvatar.h

@@ -266,6 +266,8 @@ public:
     controller::Pose getLeftHandControllerPoseInAvatarFrame() const;
     controller::Pose getRightHandControllerPoseInAvatarFrame() const;
 
+    bool hasDriveInput() const;
+
     Q_INVOKABLE void setCharacterControllerEnabled(bool enabled);
     Q_INVOKABLE bool getCharacterControllerEnabled();
 

interface/src/ui/OverlayConductor.cpp

@@ -17,122 +17,164 @@
 #include "OverlayConductor.h"
 
 OverlayConductor::OverlayConductor() {
+
 }
 
 OverlayConductor::~OverlayConductor() {
 }
 
+bool OverlayConductor::headOutsideOverlay() const {
+    glm::mat4 hmdMat = qApp->getHMDSensorPose();
+    glm::vec3 hmdPos = extractTranslation(hmdMat);
+    glm::vec3 hmdForward = transformVectorFast(hmdMat, glm::vec3(0.0f, 0.0f, -1.0f));
+
+    Transform uiTransform = qApp->getApplicationCompositor().getModelTransform();
+    glm::vec3 uiPos = uiTransform.getTranslation();
+    glm::vec3 uiForward = uiTransform.getRotation() * glm::vec3(0.0f, 0.0f, -1.0f);
+
+    const float MAX_COMPOSITOR_DISTANCE = 0.6f;
+    const float MAX_COMPOSITOR_ANGLE = 180.0f;  // rotation check is effectively disabled
+    if (glm::distance(uiPos, hmdPos) > MAX_COMPOSITOR_DISTANCE ||
+        glm::dot(uiForward, hmdForward) < cosf(glm::radians(MAX_COMPOSITOR_ANGLE))) {
+        return true;
+    }
+    return false;
+}
+
+bool OverlayConductor::updateAvatarIsAtRest() {
+
+    MyAvatar* myAvatar = DependencyManager::get<AvatarManager>()->getMyAvatar();
+
+    const quint64 REST_ENABLE_TIME_USECS = 1000 * 1000; // 1 s
+    const quint64 REST_DISABLE_TIME_USECS = 200 * 1000;  // 200 ms
+
+    const float AT_REST_THRESHOLD = 0.01f;
+    bool desiredAtRest = glm::length(myAvatar->getVelocity()) < AT_REST_THRESHOLD;
+    if (desiredAtRest != _desiredAtRest) {
+        // start timer
+        _desiredAtRestTimer = usecTimestampNow() + (desiredAtRest ? REST_ENABLE_TIME_USECS : REST_DISABLE_TIME_USECS);
+    }
+
+    _desiredAtRest = desiredAtRest;
+
+    if (_desiredAtRestTimer != 0 && usecTimestampNow() > _desiredAtRestTimer) {
+        // timer expired
+        // change state!
+        _currentAtRest = _desiredAtRest;
+        // disable timer
+        _desiredAtRestTimer = 0;
+    }
+
+    return _currentAtRest;
+}
+
+bool OverlayConductor::updateAvatarHasDriveInput() {
+    MyAvatar* myAvatar = DependencyManager::get<AvatarManager>()->getMyAvatar();
+
+    const quint64 DRIVE_ENABLE_TIME_USECS = 200 * 1000;  // 200 ms
+    const quint64 DRIVE_DISABLE_TIME_USECS = 1000 * 1000; // 1 s
+
+    bool desiredDriving = myAvatar->hasDriveInput();
+    if (desiredDriving != _desiredDriving) {
+        // start timer
+        _desiredDrivingTimer = usecTimestampNow() + (desiredDriving ? DRIVE_ENABLE_TIME_USECS : DRIVE_DISABLE_TIME_USECS);
+    }
+
+    _desiredDriving = desiredDriving;
+
+    if (_desiredDrivingTimer != 0 && usecTimestampNow() > _desiredDrivingTimer) {
+        // timer expired
+        // change state!
+        _currentDriving = _desiredDriving;
+        // disable timer
+        _desiredDrivingTimer = 0;
+    }
+
+    return _currentDriving;
+}
+
+void OverlayConductor::centerUI() {
+    // place the overlay at the current hmd position in sensor space
+    auto camMat = cancelOutRollAndPitch(qApp->getHMDSensorPose());
+    qApp->getApplicationCompositor().setModelTransform(Transform(camMat));
+}
+
+bool OverlayConductor::userWishesToHide() const {
+    // user pressed toggle button.
+    return Menu::getInstance()->isOptionChecked(MenuOption::Overlays) != _prevOverlayMenuChecked && Menu::getInstance()->isOptionChecked(MenuOption::Overlays);
+}
+
+bool OverlayConductor::userWishesToShow() const {
+    // user pressed toggle button.
+    return Menu::getInstance()->isOptionChecked(MenuOption::Overlays) != _prevOverlayMenuChecked && !Menu::getInstance()->isOptionChecked(MenuOption::Overlays);
+}
+
+void OverlayConductor::setState(State state) {
+#ifdef WANT_DEBUG
+    static QString stateToString[NumStates] = { "Enabled", "DisabledByDrive", "DisabledByHead", "DisabledByToggle" };
+    qDebug() << "OverlayConductor " << stateToString[state] << "<--" << stateToString[_state];
+#endif
+    _state = state;
+}
+
+OverlayConductor::State OverlayConductor::getState() const {
+    return _state;
+}
+
 void OverlayConductor::update(float dt) {
 
-    updateMode();
-
-    switch (_mode) {
-    case SITTING: {
-        // when sitting, the overlay is at the origin, facing down the -z axis.
-        // the camera is taken directly from the HMD.
-        Transform identity;
-        qApp->getApplicationCompositor().setModelTransform(identity);
-        qApp->getApplicationCompositor().setCameraBaseTransform(identity);
-        break;
-    }
-    case STANDING: {
-        // when standing, the overlay is at a reference position, which is set when the overlay is
-        // enabled.  The camera is taken directly from the HMD, but in world space.
-        // So the sensorToWorldMatrix must be applied.
     MyAvatar* myAvatar = DependencyManager::get<AvatarManager>()->getMyAvatar();
-        Transform t;
-        t.evalFromRawMatrix(myAvatar->getSensorToWorldMatrix());
-        qApp->getApplicationCompositor().setCameraBaseTransform(t);
 
-        // detect when head moves out side of sweet spot, or looks away.
+    // centerUI when hmd mode is first enabled
-        mat4 headMat = myAvatar->getSensorToWorldMatrix() * qApp->getHMDSensorPose();
+    if (qApp->isHMDMode() && !_hmdMode) {
-        vec3 headWorldPos = extractTranslation(headMat);
+        centerUI();
-        vec3 headForward = glm::quat_cast(headMat) * glm::vec3(0.0f, 0.0f, -1.0f);
+    }
-        Transform modelXform = qApp->getApplicationCompositor().getModelTransform();
+    _hmdMode = qApp->isHMDMode();
-        vec3 compositorWorldPos = modelXform.getTranslation();
+
-        vec3 compositorForward = modelXform.getRotation() * glm::vec3(0.0f, 0.0f, -1.0f);
+    bool prevDriving = _currentDriving;
-        const float MAX_COMPOSITOR_DISTANCE = 0.6f;
+    bool isDriving = updateAvatarHasDriveInput();
-        const float MAX_COMPOSITOR_ANGLE = 110.0f;
+    bool drivingChanged = prevDriving != isDriving;
-        if (_enabled && (glm::distance(headWorldPos, compositorWorldPos) > MAX_COMPOSITOR_DISTANCE ||
+
-                         glm::dot(headForward, compositorForward) < cosf(glm::radians(MAX_COMPOSITOR_ANGLE)))) {
+    bool isAtRest = updateAvatarIsAtRest();
-            // fade out the overlay
+
+    switch (getState()) {
+        case Enabled:
+            if (myAvatar->getClearOverlayWhenDriving() && qApp->isHMDMode() && headOutsideOverlay()) {
+                setState(DisabledByHead);
+                setEnabled(false);
+            }
+            if (userWishesToHide()) {
+                setState(DisabledByToggle);
+                setEnabled(false);
+            }
+            if (myAvatar->getClearOverlayWhenDriving() && drivingChanged && isDriving) {
+                setState(DisabledByDrive);
                 setEnabled(false);
             }
             break;
+        case DisabledByDrive:
+            if (!isDriving || userWishesToShow()) {
+                setState(Enabled);
+                setEnabled(true);
             }
-    case FLAT:
-        // do nothing
             break;
+        case DisabledByHead:
+            if (isAtRest || userWishesToShow()) {
+                setState(Enabled);
+                setEnabled(true);
             }
-}
-
-void OverlayConductor::updateMode() {
-    MyAvatar* myAvatar = DependencyManager::get<AvatarManager>()->getMyAvatar();
-    if (myAvatar->getClearOverlayWhenDriving()) {
-        float speed = glm::length(myAvatar->getVelocity());
-        const float MIN_DRIVING = 0.2f;
-        const float MAX_NOT_DRIVING = 0.01f;
-        const quint64 REQUIRED_USECS_IN_NEW_MODE_BEFORE_INVISIBLE = 200 * 1000;
-        const quint64 REQUIRED_USECS_IN_NEW_MODE_BEFORE_VISIBLE = 1000 * 1000;
-        bool nowDriving = _driving; // Assume current _driving mode unless...
-        if (speed > MIN_DRIVING) {  // ... we're definitely moving...
-            nowDriving = true;
-        } else if (speed < MAX_NOT_DRIVING) { // ... or definitely not.
-            nowDriving = false;
-        }
-        // Check that we're in this new mode for long enough to really trigger a transition.
-        if (nowDriving == _driving) {  // If there's no change in state, clear any attepted timer.
-            _timeInPotentialMode = 0;
-        } else if (_timeInPotentialMode == 0) { // We've just changed with no timer, so start timing now.
-            _timeInPotentialMode = usecTimestampNow();
-        } else if ((usecTimestampNow() - _timeInPotentialMode) > (nowDriving ? REQUIRED_USECS_IN_NEW_MODE_BEFORE_INVISIBLE : REQUIRED_USECS_IN_NEW_MODE_BEFORE_VISIBLE)) {
-            _timeInPotentialMode = 0; // a real transition
-            if (nowDriving) {
-                _wantsOverlays = Menu::getInstance()->isOptionChecked(MenuOption::Overlays);
-            } else { // reset when coming out of driving
-                _mode = FLAT;  // Seems appropriate to let things reset, below, after the following.
-                // All reset of, e.g., room-scale location as though by apostrophe key, without all the other adjustments.
-                qApp->getActiveDisplayPlugin()->resetSensors();
-                myAvatar->reset(true, false, false);
-            }
-            if (_wantsOverlays) {
-                setEnabled(!nowDriving);
-            }
-            _driving = nowDriving;
-        } // Else haven't accumulated enough time in new mode, but keep timing.
-    }
-
-    Mode newMode;
-    if (qApp->isHMDMode()) {
-        newMode = SITTING;
-    } else {
-        newMode = FLAT;
-    }
-
-    if (newMode != _mode) {
-        switch (newMode) {
-        case SITTING: {
-            // enter the SITTING state
-            // place the overlay at origin
-            qApp->getApplicationCompositor().setModelTransform(Transform());
             break;
+        case DisabledByToggle:
+            if (userWishesToShow()) {
+                setState(Enabled);
+                setEnabled(true);
             }
-        case STANDING: {  // STANDING mode is not currently used.
+            break;
-            // enter the STANDING state
+        default:
-            // place the overlay at the current hmd position in world space
-            auto camMat = cancelOutRollAndPitch(myAvatar->getSensorToWorldMatrix() * qApp->getHMDSensorPose());
-            qApp->getApplicationCompositor().setModelTransform(Transform(camMat));
             break;
     }
 
-        case FLAT:
+    _prevOverlayMenuChecked = Menu::getInstance()->isOptionChecked(MenuOption::Overlays);
-            // do nothing
-            break;
-        }
-    }
-
-    _mode = newMode;
-
 }
 
 void OverlayConductor::setEnabled(bool enabled) {
@@ -143,12 +185,14 @@ void OverlayConductor::setEnabled(bool enabled) {
     _enabled = enabled; // set the new value
     auto offscreenUi = DependencyManager::get<OffscreenUi>();
     offscreenUi->setPinned(!_enabled);
+
+    // ensure that the the state of the menu item reflects the state of the overlay.
+    Menu::getInstance()->setIsOptionChecked(MenuOption::Overlays, _enabled);
+    _prevOverlayMenuChecked = _enabled;
+
     // if the new state is visible/enabled...
-    if (_enabled && _mode == STANDING) {
+    if (_enabled && qApp->isHMDMode()) {
-        // place the overlay at the current hmd position in world space
+        centerUI();
-        MyAvatar* myAvatar = DependencyManager::get<AvatarManager>()->getMyAvatar();
-        auto camMat = cancelOutRollAndPitch(myAvatar->getSensorToWorldMatrix() * qApp->getHMDSensorPose());
-        qApp->getApplicationCompositor().setModelTransform(Transform(camMat));
     }
 }
 

interface/src/ui/OverlayConductor.h

@@ -20,20 +20,41 @@ public:
     void setEnabled(bool enable);
     bool getEnabled() const;
 
-private:
+    void centerUI();
-    void updateMode();
 
-    enum Mode {
+private:
-        FLAT,
+    bool headOutsideOverlay() const;
-        SITTING,
+    bool updateAvatarHasDriveInput();
-        STANDING
+    bool updateAvatarIsAtRest();
+    bool userWishesToHide() const;
+    bool userWishesToShow() const;
+
+    enum State {
+        Enabled = 0,
+        DisabledByDrive,
+        DisabledByHead,
+        DisabledByToggle,
+        NumStates
     };
 
-    Mode _mode { FLAT };
+    void setState(State state);
+    State getState() const;
+
+    State _state { DisabledByDrive };
+
+    bool _prevOverlayMenuChecked { true };
     bool _enabled { false };
-    bool _driving { false };
+    bool _hmdMode { false };
-    quint64 _timeInPotentialMode { 0 };
+
-    bool _wantsOverlays { true };
+    // used by updateAvatarHasDriveInput
+    quint64 _desiredDrivingTimer { 0 };
+    bool _desiredDriving { false };
+    bool _currentDriving { false };
+
+    // used by updateAvatarIsAtRest
+    quint64 _desiredAtRestTimer { 0 };
+    bool _desiredAtRest { true };
+    bool _currentAtRest { true };
 };
 
 #endif

libraries/display-plugins/src/display-plugins/CompositorHelper.cpp

@@ -336,7 +336,9 @@ void CompositorHelper::computeHmdPickRay(const glm::vec2& cursorPos, glm::vec3&
 }
 
 glm::mat4 CompositorHelper::getUiTransform() const {
-    return _currentCamera * glm::inverse(_currentDisplayPlugin->getHeadPose());
+    glm::mat4 modelMat;
+    _modelTransform.getMatrix(modelMat);
+    return _currentCamera * glm::inverse(_currentDisplayPlugin->getHeadPose()) * modelMat;
 }
 
 //Finds the collision point of a world space ray

libraries/display-plugins/src/display-plugins/hmd/HmdDisplayPlugin.cpp

@@ -253,12 +253,13 @@ void HmdDisplayPlugin::compositeScene() {
 void HmdDisplayPlugin::compositeOverlay() {
     using namespace oglplus;
     auto compositorHelper = DependencyManager::get<CompositorHelper>();
+    glm::mat4 modelMat = compositorHelper->getModelTransform().getMatrix();
 
     useProgram(_program);
     _sphereSection->Use();
     for_each_eye([&](Eye eye) {
         eyeViewport(eye);
-        auto modelView = glm::inverse(_currentPresentFrameInfo.presentPose * getEyeToHeadTransform(eye));
+        auto modelView = glm::inverse(_currentPresentFrameInfo.presentPose * getEyeToHeadTransform(eye)) * modelMat;
         auto mvp = _eyeProjections[eye] * modelView;
         Uniform<glm::mat4>(*_program, _mvpUniform).Set(mvp);
         _sphereSection->Draw();