From 1d93abf90c921097b13bbcc925005ad58acd17cb Mon Sep 17 00:00:00 2001 From: "Anthony J. Thibault" Date: Mon, 20 Jul 2015 10:13:39 -0700 Subject: [PATCH] Overlay can be shown and hidden and has better behavior in Standing HMD Mode. Added OverlayConductor class that "orchestrates" Overlays, ApplicationCompositor & AppliationOverlay instances so that the overlay can be smoothly animated on and off, mouse/key events are blocked when overlay is disabled. --- interface/src/Application.cpp | 31 +++-- interface/src/Application.h | 5 + interface/src/ui/ApplicationCompositor.cpp | 41 +++++- interface/src/ui/ApplicationCompositor.h | 13 ++ interface/src/ui/OverlayConductor.cpp | 148 +++++++++++++++++++++ interface/src/ui/OverlayConductor.h | 36 +++++ interface/src/ui/overlays/Overlays.cpp | 13 ++ interface/src/ui/overlays/Overlays.h | 3 + 8 files changed, 274 insertions(+), 16 deletions(-) create mode 100644 interface/src/ui/OverlayConductor.cpp create mode 100644 interface/src/ui/OverlayConductor.h diff --git a/interface/src/Application.cpp b/interface/src/Application.cpp index b4a285a680..8a322c5f5b 100644 --- a/interface/src/Application.cpp +++ b/interface/src/Application.cpp @@ -1123,6 +1123,7 @@ void Application::audioMuteToggled() { } void Application::faceTrackerMuteToggled() { + QAction* muteAction = Menu::getInstance()->getActionForOption(MenuOption::MuteFaceTracking); Q_CHECK_PTR(muteAction); bool isMuted = getSelectedFaceTracker()->isMuted(); @@ -1682,6 +1683,9 @@ void Application::mousePressEvent(QMouseEvent* event, unsigned int deviceID) { } else if (event->button() == Qt::RightButton) { // right click items here + + // toggle the overlay + _overlayConductor.setEnabled(!_overlayConductor.getEnabled()); } } } @@ -1973,17 +1977,13 @@ void Application::idle() { _idleLoopStdev.reset(); } } - - // depending on whether we're throttling or not. - // Once rendering is off on another thread we should be able to have Application::idle run at start(0) in - // perpetuity and not expect events to get backed up. - - static const int IDLE_TIMER_DELAY_MS = 0; - int desiredInterval = getActiveDisplayPlugin()->isThrottled() ? THROTTLED_IDLE_TIMER_DELAY : IDLE_TIMER_DELAY_MS; - - if (idleTimer->interval() != desiredInterval) { - idleTimer->start(desiredInterval); - } + + float secondsSinceLastUpdate = (float)timeSinceLastUpdate / 1000.0f; + _overlayConductor.update(secondsSinceLastUpdate); + + // After finishing all of the above work, ensure the idle timer is set to the proper interval, + // depending on whether we're throttling or not + idleTimer->start(getActiveDisplayPlugin()->isThrottled() ? THROTTLED_IDLE_TIMER_DELAY : 0); } // check for any requested background downloads. @@ -4882,6 +4882,15 @@ mat4 Application::getEyePose(int eye) const { return mat4(); } +mat4 Application::getEyeOffset(int eye) const { + if (isHMDMode()) { + mat4 identity; + return getActiveDisplayPlugin()->getModelview((Eye)eye, identity); + } + + return mat4(); +} + mat4 Application::getHMDSensorPose() const { if (isHMDMode()) { return getActiveDisplayPlugin()->getHeadPose(); diff --git a/interface/src/Application.h b/interface/src/Application.h index 10605d3e57..e4b13cce8b 100644 --- a/interface/src/Application.h +++ b/interface/src/Application.h @@ -61,6 +61,7 @@ #include "ui/overlays/Overlays.h" #include "ui/ApplicationOverlay.h" #include "ui/ApplicationCompositor.h" +#include "ui/OverlayConductor.h" #include "ui/RunningScriptsWidget.h" #include "ui/ToolWindow.h" #include "octree/OctreePacketProcessor.h" @@ -317,6 +318,7 @@ public: bool isHMDMode() const; glm::mat4 getHMDSensorPose() const; glm::mat4 getEyePose(int eye) const; + glm::mat4 getEyeOffset(int eye) const; glm::mat4 getEyeProjection(int eye) const; QRect getDesirableApplicationGeometry(); @@ -652,11 +654,14 @@ private: Overlays _overlays; ApplicationOverlay _applicationOverlay; ApplicationCompositor _compositor; + OverlayConductor _overlayConductor; int _oldHandMouseX[2]; int _oldHandMouseY[2]; bool _oldHandLeftClick[2]; bool _oldHandRightClick[2]; + + bool _overlayEnabled = true; }; #endif // hifi_Application_h diff --git a/interface/src/ui/ApplicationCompositor.cpp b/interface/src/ui/ApplicationCompositor.cpp index fcb37a0066..64b4cda0a1 100644 --- a/interface/src/ui/ApplicationCompositor.cpp +++ b/interface/src/ui/ApplicationCompositor.cpp @@ -186,7 +186,8 @@ void ApplicationCompositor::bindCursorTexture(gpu::Batch& batch, uint8_t cursorI // Draws the FBO texture for the screen void ApplicationCompositor::displayOverlayTexture(RenderArgs* renderArgs) { PROFILE_RANGE(__FUNCTION__); - if (_alpha == 0.0f) { + + if (_alpha <= 0.0f) { return; } @@ -253,7 +254,8 @@ vec2 getPolarCoordinates(const PalmData& palm) { // Draws the FBO texture for Oculus rift. void ApplicationCompositor::displayOverlayTextureHmd(RenderArgs* renderArgs, int eye) { PROFILE_RANGE(__FUNCTION__); - if (_alpha == 0.0f) { + + if (_alpha <= 0.0f) { return; } @@ -280,7 +282,11 @@ void ApplicationCompositor::displayOverlayTextureHmd(RenderArgs* renderArgs, int batch.setResourceTexture(0, overlayFramebuffer->getRenderBuffer(0)); - batch.setViewTransform(Transform()); + mat4 camMat; + _cameraTransform.getMatrix(camMat); + camMat = camMat * qApp->getEyeOffset(eye); + batch.setViewTransform(camMat); + batch.setProjectionTransform(qApp->getEyeProjection(eye)); mat4 eyePose = qApp->getEyePose(eye); @@ -293,7 +299,9 @@ void ApplicationCompositor::displayOverlayTextureHmd(RenderArgs* renderArgs, int } #else { - batch.setModelTransform(overlayXfm); + //batch.setModelTransform(overlayXfm); + + batch.setModelTransform(_modelTransform); drawSphereSection(batch); } #endif @@ -304,8 +312,8 @@ void ApplicationCompositor::displayOverlayTextureHmd(RenderArgs* renderArgs, int bindCursorTexture(batch); - MyAvatar* myAvatar = DependencyManager::get()->getMyAvatar(); //Controller Pointers + MyAvatar* myAvatar = DependencyManager::get()->getMyAvatar(); for (int i = 0; i < (int)myAvatar->getHand()->getNumPalms(); i++) { PalmData& palm = myAvatar->getHand()->getPalms()[i]; if (palm.isActive()) { @@ -619,6 +627,19 @@ void ApplicationCompositor::drawSphereSection(gpu::Batch& batch) { batch.setInputFormat(streamFormat); static const int VERTEX_STRIDE = sizeof(vec3) + sizeof(vec2) + sizeof(vec4); + + if (_prevAlpha != _alpha) { + // adjust alpha by munging vertex color alpha. + // FIXME we should probably just use a uniform for this. + float* floatPtr = reinterpret_cast(_hemiVertices->editData()); + const auto ALPHA_FLOAT_OFFSET = (sizeof(vec3) + sizeof(vec2) + sizeof(vec3)) / sizeof(float); + const auto VERTEX_FLOAT_STRIDE = (sizeof(vec3) + sizeof(vec2) + sizeof(vec4)) / sizeof(float); + const auto NUM_VERTS = _hemiVertices->getSize() / VERTEX_STRIDE; + for (size_t i = 0; i < NUM_VERTS; i++) { + floatPtr[i * VERTEX_FLOAT_STRIDE + ALPHA_FLOAT_OFFSET] = _alpha; + } + } + gpu::BufferView posView(_hemiVertices, 0, _hemiVertices->getSize(), VERTEX_STRIDE, streamFormat->getAttributes().at(gpu::Stream::POSITION)._element); gpu::BufferView uvView(_hemiVertices, sizeof(vec3), _hemiVertices->getSize(), VERTEX_STRIDE, streamFormat->getAttributes().at(gpu::Stream::TEXCOORD)._element); gpu::BufferView colView(_hemiVertices, sizeof(vec3) + sizeof(vec2), _hemiVertices->getSize(), VERTEX_STRIDE, streamFormat->getAttributes().at(gpu::Stream::COLOR)._element); @@ -708,3 +729,13 @@ void ApplicationCompositor::updateTooltips() { } } } + +void ApplicationCompositor::update(float dt) { + const int ALPHA_FADE_RATE = 1.0f; + _prevAlpha = _alpha; + if (_fadeInAlpha && _alpha < 1.0f) { + _alpha = std::min(_alpha + ALPHA_FADE_RATE * dt, 1.0f); + } else if (!_fadeInAlpha && _alpha > 0.0f) { + _alpha = std::max(_alpha - ALPHA_FADE_RATE * dt, 0.0f); + } +} diff --git a/interface/src/ui/ApplicationCompositor.h b/interface/src/ui/ApplicationCompositor.h index 8ae6f0930e..f82119d40f 100644 --- a/interface/src/ui/ApplicationCompositor.h +++ b/interface/src/ui/ApplicationCompositor.h @@ -64,6 +64,14 @@ public: void computeHmdPickRay(glm::vec2 cursorPos, glm::vec3& origin, glm::vec3& direction) const; uint32_t getOverlayTexture() const; + void setCameraTransform(const Transform& transform) { _cameraTransform = transform; } + void setModelTransform(const Transform& transform) { _modelTransform = transform; } + + void fadeIn() { _fadeInAlpha = true; } + void fadeOut() { _fadeInAlpha = false; } + void toggle() { _fadeInAlpha = !_fadeInAlpha; } + void update(float dt); + static glm::vec2 directionToSpherical(const glm::vec3 & direction); static glm::vec3 sphericalToDirection(const glm::vec2 & sphericalPos); static glm::vec2 screenToSpherical(const glm::vec2 & screenPos); @@ -100,6 +108,8 @@ private: bool _magnifier{ true }; float _alpha{ 1.0f }; + float _prevAlpha{ 1.0f }; + float _fadeInAlpha{ true }; float _oculusUIRadius{ 1.0f }; QMap _cursors; @@ -115,6 +125,9 @@ private: glm::vec3 _previousMagnifierBottomRight; glm::vec3 _previousMagnifierTopLeft; glm::vec3 _previousMagnifierTopRight; + + Transform _modelTransform; + Transform _cameraTransform; }; #endif // hifi_ApplicationCompositor_h diff --git a/interface/src/ui/OverlayConductor.cpp b/interface/src/ui/OverlayConductor.cpp new file mode 100644 index 0000000000..112b3c1820 --- /dev/null +++ b/interface/src/ui/OverlayConductor.cpp @@ -0,0 +1,148 @@ +// +// OverlayConductor.cpp +// interface/src/ui +// +// Copyright 2015 High Fidelity, Inc. +// +// Distributed under the Apache License, Version 2.0. +// See the accompanying file LICENSE or http://www.apache.org/licenses/LICENSE-2.0.html +// + +#include "Application.h" +#include "avatar/AvatarManager.h" + +#include "OverlayConductor.h" + +OverlayConductor::OverlayConductor() { +} + +OverlayConductor::~OverlayConductor() { +} + +void OverlayConductor::update(float dt) { + + updateMode(); + + switch (_mode) { + case SITTING: { + // when sitting, the overlay is at the origin, facing down the -z axis. + // the camera is taken directly from the HMD. + Transform identity; + qApp->getApplicationCompositor().setModelTransform(identity); + Transform t; + t.evalFromRawMatrix(qApp->getHMDSensorPose()); + qApp->getApplicationCompositor().setCameraTransform(t); + break; + } + case STANDING: { + // when standing, the overlay is at a reference position, which is set when the overlay is + // enabled. The camera is taken directly from the HMD in world space. + MyAvatar* myAvatar = DependencyManager::get()->getMyAvatar(); + Transform t; + t.evalFromRawMatrix(myAvatar->getSensorToWorldMatrix() * qApp->getHMDSensorPose()); + qApp->getApplicationCompositor().setCameraTransform(t); + break; + } + case FLAT: + // do nothing + break; + } + + // TODO: detect when head moves out side of sweet spot. + // TODO: set reference position when HMD is on, etc are changed. + + // process alpha fade animations + qApp->getApplicationCompositor().update(dt); +} + +void OverlayConductor::updateMode() { + + Mode newMode; + if (qApp->isHMDMode()) { + MyAvatar* myAvatar = DependencyManager::get()->getMyAvatar(); + if (myAvatar->getStandingHMDSensorMode()) { + newMode = STANDING; + } else { + newMode = SITTING; + } + } else { + newMode = FLAT; + } + + if (newMode != _mode) { + switch (newMode) { + case SITTING: { + // enter the SITTING state + // place the overlay at origin + Transform identity; + qApp->getApplicationCompositor().setModelTransform(identity); + break; + } + case STANDING: { + // enter the STANDING state + // place the overlay at the current hmd position in world space + MyAvatar* myAvatar = DependencyManager::get()->getMyAvatar(); + auto camMat = cancelOutRollAndPitch(myAvatar->getSensorToWorldMatrix() * qApp->getHMDSensorPose()); + Transform t; + t.setTranslation(extractTranslation(camMat)); + t.setRotation(glm::quat_cast(camMat)); + qApp->getApplicationCompositor().setModelTransform(t); + break; + } + + case FLAT: + // do nothing + break; + } + } + + _mode = newMode; +} + +void OverlayConductor::setEnabled(bool enabled) { + + if (enabled == _enabled) { + return; + } + + if (_enabled) { + // alpha fadeOut the overlay mesh. + qApp->getApplicationCompositor().fadeOut(); + + // disable mouse clicks from script + qApp->getOverlays().disable(); + + // disable QML events + auto offscreenUi = DependencyManager::get(); + offscreenUi->getRootItem()->setEnabled(false); + + _enabled = false; + } else { + // alpha fadeIn the overlay mesh. + qApp->getApplicationCompositor().fadeIn(); + + // enable mouse clicks from script + qApp->getOverlays().enable(); + + // enable QML events + auto offscreenUi = DependencyManager::get(); + offscreenUi->getRootItem()->setEnabled(true); + + if (_mode == STANDING) { + // place the overlay at the current hmd position in world space + MyAvatar* myAvatar = DependencyManager::get()->getMyAvatar(); + auto camMat = cancelOutRollAndPitch(myAvatar->getSensorToWorldMatrix() * qApp->getHMDSensorPose()); + Transform t; + t.setTranslation(extractTranslation(camMat)); + t.setRotation(glm::quat_cast(camMat)); + qApp->getApplicationCompositor().setModelTransform(t); + } + + _enabled = true; + } +} + +bool OverlayConductor::getEnabled() const { + return _enabled; +} + diff --git a/interface/src/ui/OverlayConductor.h b/interface/src/ui/OverlayConductor.h new file mode 100644 index 0000000000..4b8c0134b5 --- /dev/null +++ b/interface/src/ui/OverlayConductor.h @@ -0,0 +1,36 @@ +// +// OverlayConductor.h +// interface/src/ui +// +// Copyright 2015 High Fidelity, Inc. +// +// Distributed under the Apache License, Version 2.0. +// See the accompanying file LICENSE or http://www.apache.org/licenses/LICENSE-2.0.html +// + +#ifndef hifi_OverlayConductor_h +#define hifi_OverlayConductor_h + +class OverlayConductor { +public: + OverlayConductor(); + ~OverlayConductor(); + + void update(float dt); + void setEnabled(bool enable); + bool getEnabled() const; + +private: + void updateMode(); + + enum Mode { + FLAT, + SITTING, + STANDING + }; + + Mode _mode = FLAT; + bool _enabled = true; +}; + +#endif diff --git a/interface/src/ui/overlays/Overlays.cpp b/interface/src/ui/overlays/Overlays.cpp index ff218db844..35eecfdc20 100644 --- a/interface/src/ui/overlays/Overlays.cpp +++ b/interface/src/ui/overlays/Overlays.cpp @@ -124,6 +124,16 @@ void Overlays::renderHUD(RenderArgs* renderArgs) { } } +void Overlays::disable() { + QWriteLocker lock(&_lock); + _enabled = false; +} + +void Overlays::enable() { + QWriteLocker lock(&_lock); + _enabled = true; +} + unsigned int Overlays::addOverlay(const QString& type, const QScriptValue& properties) { unsigned int thisID = 0; Overlay* thisOverlay = NULL; @@ -269,6 +279,9 @@ unsigned int Overlays::getOverlayAtPoint(const glm::vec2& point) { } QReadLocker lock(&_lock); + if (!_enabled) { + return 0; + } QMapIterator i(_overlaysHUD); i.toBack(); diff --git a/interface/src/ui/overlays/Overlays.h b/interface/src/ui/overlays/Overlays.h index cd5b0f1d10..17c59684e1 100644 --- a/interface/src/ui/overlays/Overlays.h +++ b/interface/src/ui/overlays/Overlays.h @@ -56,6 +56,8 @@ public: void init(); void update(float deltatime); void renderHUD(RenderArgs* renderArgs); + void disable(); + void enable(); public slots: /// adds an overlay with the specific properties @@ -99,6 +101,7 @@ private: QReadWriteLock _lock; QReadWriteLock _deleteLock; QScriptEngine* _scriptEngine; + bool _enabled = true; };