Fix picking in the HMD

interface/src/Application.cpp

@@ -3077,13 +3077,10 @@ PickRay Application::computePickRay(float x, float y) const {
     y /= size.y;
     PickRay result;
     if (isHMDMode()) {
-        ApplicationOverlay::computeHmdPickRay(glm::vec2(x, y), result.origin, result.direction);
+        getApplicationOverlay().computeHmdPickRay(glm::vec2(x, y), result.origin, result.direction);
     } else {
-        if (activeRenderingThread) {
-            getDisplayViewFrustum()->computePickRay(x, y, result.origin, result.direction);
-        } else {
-            getViewFrustum()->computePickRay(x, y, result.origin, result.direction);
-        }
+        auto frustum = activeRenderingThread ? getDisplayViewFrustum() : getViewFrustum();
+        frustum->computePickRay(x, y, result.origin, result.direction);
     }
     return result;
 }
@@ -3111,8 +3108,8 @@ QImage Application::renderAvatarBillboard() {
 ViewFrustum* Application::getViewFrustum() {
 #ifdef DEBUG
     if (QThread::currentThread() == activeRenderingThread) {
-        // FIXME, should this be an assert?
-        qWarning() << "Calling Application::getViewFrustum() from the active rendering thread, did you mean Application::getDisplayViewFrustum()?";
+        // FIXME, figure out a better way to do this
+        //qWarning() << "Calling Application::getViewFrustum() from the active rendering thread, did you mean Application::getDisplayViewFrustum()?";
     }
 #endif
     return &_viewFrustum;
@@ -3121,8 +3118,8 @@ ViewFrustum* Application::getViewFrustum() {
 const ViewFrustum* Application::getViewFrustum() const {
 #ifdef DEBUG
     if (QThread::currentThread() == activeRenderingThread) {
-        // FIXME, should this be an assert?
-        qWarning() << "Calling Application::getViewFrustum() from the active rendering thread, did you mean Application::getDisplayViewFrustum()?";
+        // FIXME, figure out a better way to do this
+        //qWarning() << "Calling Application::getViewFrustum() from the active rendering thread, did you mean Application::getDisplayViewFrustum()?";
     }
 #endif
     return &_viewFrustum;
@@ -3131,8 +3128,8 @@ const ViewFrustum* Application::getViewFrustum() const {
 ViewFrustum* Application::getDisplayViewFrustum() {
 #ifdef DEBUG
     if (QThread::currentThread() != activeRenderingThread) {
-        // FIXME, should this be an assert?
-        qWarning() << "Calling Application::getDisplayViewFrustum() from outside the active rendering thread or outside rendering, did you mean Application::getViewFrustum()?";
+        // FIXME, figure out a better way to do this
+        // qWarning() << "Calling Application::getDisplayViewFrustum() from outside the active rendering thread or outside rendering, did you mean Application::getViewFrustum()?";
     }
 #endif
     return &_displayViewFrustum;
@@ -3141,8 +3138,8 @@ ViewFrustum* Application::getDisplayViewFrustum() {
 const ViewFrustum* Application::getDisplayViewFrustum() const {
 #ifdef DEBUG
     if (QThread::currentThread() != activeRenderingThread) {
-        // FIXME, should this be an assert?
-        qWarning() << "Calling Application::getDisplayViewFrustum() from outside the active rendering thread or outside rendering, did you mean Application::getViewFrustum()?";
+        // FIXME, figure out a better way to do this
+        // qWarning() << "Calling Application::getDisplayViewFrustum() from outside the active rendering thread or outside rendering, did you mean Application::getViewFrustum()?";
     }
 #endif
     return &_displayViewFrustum;

interface/src/ui/ApplicationOverlay.cpp

@@ -301,9 +301,14 @@ void ApplicationOverlay::displayOverlayTextureHmd(Camera& whichCamera) {
     //Update and draw the magnifiers
     MyAvatar* myAvatar = DependencyManager::get<AvatarManager>()->getMyAvatar();
     const glm::quat& orientation = myAvatar->getOrientation();
-    const glm::vec3& position = myAvatar->getDefaultEyePosition();
+    // Always display the HMD overlay relative to the camera position but 
+    // remove the HMD pose offset.  This results in an overlay that sticks with you 
+    // even in third person mode, but isn't drawn at a fixed distance.
+    glm::vec3 position = whichCamera.getPosition();
+    position -= qApp->getCamera()->getHmdPosition();
     const float scale = myAvatar->getScale() * _oculusUIRadius;
     
+//    glm::vec3 eyeOffset = setEyeOffsetPosition;
     glMatrixMode(GL_MODELVIEW);
     glPushMatrix(); {
         glTranslatef(position.x, position.y, position.z);
@@ -453,19 +458,32 @@ void ApplicationOverlay::displayOverlayTextureStereo(Camera& whichCamera, float
     glEnable(GL_LIGHTING);
 }
 
-void ApplicationOverlay::computeHmdPickRay(glm::vec2 cursorPos, glm::vec3& origin, glm::vec3& direction) {
-    const MyAvatar* myAvatar = DependencyManager::get<AvatarManager>()->getMyAvatar();
-    cursorPos = 0.5f - cursorPos;
-    cursorPos *= MOUSE_RANGE;
-    const glm::quat orientation(glm::vec3(cursorPos, 0.0f));
-    const glm::vec3 localDirection = orientation * IDENTITY_FRONT;
+void ApplicationOverlay::computeHmdPickRay(glm::vec2 cursorPos, glm::vec3& origin, glm::vec3& direction) const {
+    cursorPos *= qApp->getCanvasSize();
+    const glm::vec2 projection = screenToSpherical(cursorPos);
+    // The overlay space orientation of the mouse coordinates
+    const glm::quat orientation(glm::vec3(-projection.y, projection.x, 0.0f));
+    // FIXME We now have the direction of the ray FROM THE DEFAULT HEAD POSE.  
+    // Now we need to account for the actual camera position relative to the overlay
+    glm::vec3 overlaySpaceDirection = glm::normalize(orientation * IDENTITY_FRONT);
 
-    // Get cursor position
-    const glm::vec3 cursorDir = myAvatar->getDefaultEyePosition() + myAvatar->getOrientation() * localDirection;
 
-    // Ray start where the eye position is and stop where the cursor is
-    origin = myAvatar->getEyePosition();
-    direction = cursorDir - origin;
+    const glm::vec3& hmdPosition = qApp->getCamera()->getHmdPosition();
+    const glm::quat& hmdOrientation = qApp->getCamera()->getHmdRotation();
+
+    // We need the RAW camera orientation and position, because this is what the overlay is
+    // rendered relative to
+    const glm::vec3 overlayPosition = qApp->getCamera()->getPosition() - hmdPosition;
+    const glm::quat overlayOrientation = qApp->getCamera()->getRotation() * glm::inverse(hmdOrientation);
+
+    // Intersection UI overlay space
+    glm::vec3 worldSpaceDirection = overlayOrientation * overlaySpaceDirection;
+    glm::vec3 intersectionWithUi = glm::normalize(worldSpaceDirection) * _oculusUIRadius;
+    intersectionWithUi += overlayPosition;
+
+    // Intersection in world space
+    origin = overlayPosition + hmdPosition;
+    direction = glm::normalize(intersectionWithUi - origin);
 }
 
 //Caculate the click location using one of the sixense controllers. Scale is not applied

interface/src/ui/ApplicationOverlay.h

@@ -58,12 +58,12 @@ public:
     glm::vec2 overlayToSpherical(const glm::vec2 & overlayPos) const;
     glm::vec2 screenToOverlay(const glm::vec2 & screenPos) const;
     glm::vec2 overlayToScreen(const glm::vec2 & overlayPos) const;
+    void computeHmdPickRay(glm::vec2 cursorPos, glm::vec3& origin, glm::vec3& direction) const;
 
     static glm::vec2 directionToSpherical(const glm::vec3 & direction);
     static glm::vec3 sphericalToDirection(const glm::vec2 & sphericalPos);
     static glm::vec2 screenToSpherical(const glm::vec2 & screenPos);
     static glm::vec2 sphericalToScreen(const glm::vec2 & sphericalPos);
-    static void computeHmdPickRay(glm::vec2 cursorPos, glm::vec3& origin, glm::vec3& direction);
     
 private:
     // Interleaved vertex data