make all volume3DOverlays handle proper ray picking against their rotated bounding box, make model overlays property ray pick as well

interface/src/ui/overlays/ModelOverlay.cpp

@@ -87,8 +87,56 @@ void ModelOverlay::setProperties(const QScriptValue &properties) {
         }
         _updateModel = true;
     }
-    
+
+    QScriptValue dimensionsValue = properties.property("dimensions");
+    if (dimensionsValue.isValid()) {
+        QScriptValue x = dimensionsValue.property("x");
+        QScriptValue y = dimensionsValue.property("y");
+        QScriptValue z = dimensionsValue.property("z");
+        if (x.isValid() && y.isValid() && z.isValid()) {
+            glm::vec3 dimensions;
+            dimensions.x = x.toVariant().toFloat();
+            dimensions.y = y.toVariant().toFloat();
+            dimensions.z = z.toVariant().toFloat();
+            _model.setScaleToFit(true, dimensions);
+        }
+        _updateModel = true;
+    }
+
     if (properties.property("position").isValid()) {
         _updateModel = true;
     }
-}
+}
+
+bool ModelOverlay::findRayIntersection(const glm::vec3& origin, const glm::vec3& direction,
+                                                        float& distance, BoxFace& face) const {
+
+    // if our model isn't active, we can't ray pick yet...
+    if (!_model.isActive()) {
+        return false;
+    }
+
+    // extents is the entity relative, scaled, centered extents of the entity
+    glm::vec3 position = getPosition();
+    glm::mat4 rotation = glm::mat4_cast(getRotation());
+    glm::mat4 translation = glm::translate(position);
+    glm::mat4 entityToWorldMatrix = translation * rotation;
+    glm::mat4 worldToEntityMatrix = glm::inverse(entityToWorldMatrix);
+
+    Extents modelExtents = _model.getMeshExtents(); // NOTE: unrotated
+    
+    glm::vec3 dimensions = modelExtents.maximum - modelExtents.minimum;
+    glm::vec3 corner = dimensions * -0.5f; // since we're going to do the ray picking in the overlay frame of reference
+    AABox overlayFrameBox(corner, dimensions);
+
+    glm::vec3 overlayFrameOrigin = glm::vec3(worldToEntityMatrix * glm::vec4(origin, 1.0f));
+    glm::vec3 overlayFrameDirection = glm::vec3(worldToEntityMatrix * glm::vec4(direction, 0.0f));
+
+    // we can use the AABox's ray intersection by mapping our origin and direction into the entity frame
+    // and testing intersection there.
+    if (overlayFrameBox.findRayIntersection(overlayFrameOrigin, overlayFrameDirection, distance, face)) {
+        return true;
+    }
+
+    return false;
+}

interface/src/ui/overlays/ModelOverlay.h

@@ -24,6 +24,8 @@ public:
     virtual void update(float deltatime);
     virtual void render();
     virtual void setProperties(const QScriptValue& properties);
+    virtual bool findRayIntersection(const glm::vec3& origin, const glm::vec3& direction, float& distance, BoxFace& face) const;
+
 private:
     
     Model _model;

interface/src/ui/overlays/Volume3DOverlay.cpp

@@ -8,6 +8,9 @@
 //  See the accompanying file LICENSE or http://www.apache.org/licenses/LICENSE-2.0.html
 //
 
+#include <glm/glm.hpp>
+#include <glm/gtx/transform.hpp>
+
 // include this before QGLWidget, which includes an earlier version of OpenGL
 #include "InterfaceConfig.h"
 
@@ -85,8 +88,23 @@ void Volume3DOverlay::setProperties(const QScriptValue& properties) {
 bool Volume3DOverlay::findRayIntersection(const glm::vec3& origin, const glm::vec3& direction,
                                                         float& distance, BoxFace& face) const {
 
-    // TODO: this is not exactly accurate because it doesn't properly handle rotation... but it's close enough for our
+    // extents is the entity relative, scaled, centered extents of the entity
-    // current use cases. We do need to improve it to be more accurate
+    glm::vec3 position = getPosition();
-    AABox myBox(getCorner(), _dimensions);
+    glm::mat4 rotation = glm::mat4_cast(getRotation());
-    return myBox.findRayIntersection(origin, direction, distance, face);
+    glm::mat4 translation = glm::translate(position);
+    glm::mat4 entityToWorldMatrix = translation * rotation;
+    glm::mat4 worldToEntityMatrix = glm::inverse(entityToWorldMatrix);
+
+    glm::vec3 dimensions = _dimensions;
+    glm::vec3 corner = dimensions * -0.5f; // since we're going to do the ray picking in the overlay frame of reference
+    AABox overlayFrameBox(corner, dimensions);
+    glm::vec3 overlayFrameOrigin = glm::vec3(worldToEntityMatrix * glm::vec4(origin, 1.0f));
+    glm::vec3 overlayFrameDirection = glm::vec3(worldToEntityMatrix * glm::vec4(direction, 0.0f));
+
+    // we can use the AABox's ray intersection by mapping our origin and direction into the entity frame
+    // and testing intersection there.
+    if (overlayFrameBox.findRayIntersection(overlayFrameOrigin, overlayFrameDirection, distance, face)) {
+        return true;
+    }
+    return false;
 }