support additional .extraInfo for ray intersections

interface/src/Application.cpp

@@ -1844,7 +1844,7 @@ Application::Application(int& argc, char** argv, QElapsedTimer& startupTimer, bo
                 entityResult.distance = pickResult->distance;
                 entityResult.surfaceNormal = pickResult->surfaceNormal;
                 entityResult.entityID = pickResult->objectID;
-                entityResult.entity = DependencyManager::get<EntityTreeRenderer>()->getTree()->findEntityByID(entityResult.entityID);
+                entityResult.extraInfo = pickResult->extraInfo;
             }
         }
         return entityResult;

interface/src/avatar/AvatarManager.cpp

@@ -546,7 +546,7 @@ RayToAvatarIntersectionResult AvatarManager::findRayIntersectionVector(const Pic
             continue;
         }
 
-        QString extraInfo;
+        QVariantMap extraInfo;
         intersects = avatarModel->findRayIntersectionAgainstSubMeshes(ray.origin, normDirection,
                                                                       distance, face, surfaceNormal, extraInfo, true);
 
@@ -554,6 +554,7 @@ RayToAvatarIntersectionResult AvatarManager::findRayIntersectionVector(const Pic
             result.intersects = true;
             result.avatarID = avatar->getID();
             result.distance = distance;
+            result.extraInfo = extraInfo;
         }
     }
 

interface/src/avatar/MyAvatar.cpp

@@ -2401,7 +2401,6 @@ bool MyAvatar::requiresSafeLanding(const glm::vec3& positionIn, glm::vec3& bette
     };
     auto findIntersection = [&](const glm::vec3& startPointIn, const glm::vec3& directionIn, glm::vec3& intersectionOut, EntityItemID& entityIdOut, glm::vec3& normalOut) {
         OctreeElementPointer element;
-        EntityItemPointer intersectedEntity = NULL;
         float distance;
         BoxFace face;
         const bool visibleOnly = false;
@@ -2413,13 +2412,14 @@ bool MyAvatar::requiresSafeLanding(const glm::vec3& positionIn, glm::vec3& bette
         const auto lockType = Octree::Lock; // Should we refactor to take a lock just once?
         bool* accurateResult = NULL;
 
-        bool intersects = entityTree->findRayIntersection(startPointIn, directionIn, include, ignore, visibleOnly, collidableOnly, precisionPicking,
+        QVariantMap extraInfo;
-            element, distance, face, normalOut, (void**)&intersectedEntity, lockType, accurateResult);
+        EntityItemID entityID = entityTree->findRayIntersection(startPointIn, directionIn, include, ignore, visibleOnly, collidableOnly, precisionPicking,
-        if (!intersects || !intersectedEntity) {
+            element, distance, face, normalOut, extraInfo, lockType, accurateResult);
+        if (entityID.isNull()) {
              return false;
         }
         intersectionOut = startPointIn + (directionIn * distance);
-        entityIdOut = intersectedEntity->getEntityItemID();
+        entityIdOut = entityID;
         return true;
     };
 

interface/src/raypick/RayPick.h

@@ -29,6 +29,7 @@ public:
         distance = rayPickResult.distance;
         intersection = rayPickResult.intersection;
         surfaceNormal = rayPickResult.surfaceNormal;
+        extraInfo = rayPickResult.extraInfo;
     }
 
     IntersectionType type { NONE };
@@ -37,6 +38,7 @@ public:
     float distance { FLT_MAX };
     glm::vec3 intersection { NAN };
     glm::vec3 surfaceNormal { NAN };
+    QVariantMap extraInfo;
 
     virtual QVariantMap toVariantMap() const override {
         QVariantMap toReturn;
@@ -47,6 +49,7 @@ public:
         toReturn["intersection"] = vec3toVariant(intersection);
         toReturn["surfaceNormal"] = vec3toVariant(surfaceNormal);
         toReturn["searchRay"] = PickResult::toVariantMap();
+        toReturn["extraInfo"] = extraInfo;
         return toReturn;
     }
 

interface/src/ui/overlays/Base3DOverlay.h

@@ -68,7 +68,7 @@ public:
                                         BoxFace& face, glm::vec3& surfaceNormal);
 
     virtual bool findRayIntersectionExtraInfo(const glm::vec3& origin, const glm::vec3& direction,
-                                        float& distance, BoxFace& face, glm::vec3& surfaceNormal, QString& extraInfo) {
+                                        float& distance, BoxFace& face, glm::vec3& surfaceNormal, QVariantMap& extraInfo) {
         return findRayIntersection(origin, direction, distance, face, surfaceNormal);
     }
 

interface/src/ui/overlays/ModelOverlay.cpp

@@ -445,12 +445,12 @@ QVariant ModelOverlay::getProperty(const QString& property) {
 bool ModelOverlay::findRayIntersection(const glm::vec3& origin, const glm::vec3& direction,
                                         float& distance, BoxFace& face, glm::vec3& surfaceNormal) {
 
-    QString subMeshNameTemp;
+    QVariantMap extraInfo;
-    return _model->findRayIntersectionAgainstSubMeshes(origin, direction, distance, face, surfaceNormal, subMeshNameTemp);
+    return _model->findRayIntersectionAgainstSubMeshes(origin, direction, distance, face, surfaceNormal, extraInfo);
 }
 
 bool ModelOverlay::findRayIntersectionExtraInfo(const glm::vec3& origin, const glm::vec3& direction,
-                                        float& distance, BoxFace& face, glm::vec3& surfaceNormal, QString& extraInfo) {
+                                        float& distance, BoxFace& face, glm::vec3& surfaceNormal, QVariantMap& extraInfo) {
 
     return _model->findRayIntersectionAgainstSubMeshes(origin, direction, distance, face, surfaceNormal, extraInfo);
 }

interface/src/ui/overlays/ModelOverlay.h

@@ -41,7 +41,7 @@ public:
     virtual bool findRayIntersection(const glm::vec3& origin, const glm::vec3& direction, float& distance,
                                         BoxFace& face, glm::vec3& surfaceNormal) override;
     virtual bool findRayIntersectionExtraInfo(const glm::vec3& origin, const glm::vec3& direction,
-        float& distance, BoxFace& face, glm::vec3& surfaceNormal, QString& extraInfo) override;
+        float& distance, BoxFace& face, glm::vec3& surfaceNormal, QVariantMap& extraInfo) override;
 
     virtual ModelOverlay* createClone() const override;
 

interface/src/ui/overlays/Overlays.cpp

@@ -584,7 +584,7 @@ RayToOverlayIntersectionResult Overlays::findRayIntersectionVector(const PickRay
             float thisDistance;
             BoxFace thisFace;
             glm::vec3 thisSurfaceNormal;
-            QString thisExtraInfo;
+            QVariantMap thisExtraInfo;
             if (thisOverlay->findRayIntersectionExtraInfo(ray.origin, ray.direction, thisDistance,
                                                           thisFace, thisSurfaceNormal, thisExtraInfo)) {
                 bool isDrawInFront = thisOverlay->getDrawInFront();
@@ -642,7 +642,7 @@ QScriptValue RayToOverlayIntersectionResultToScriptValue(QScriptEngine* engine,
     obj.setProperty("face", faceName);
     auto intersection = vec3toScriptValue(engine, value.intersection);
     obj.setProperty("intersection", intersection);
-    obj.setProperty("extraInfo", value.extraInfo);
+    obj.setProperty("extraInfo", engine->toScriptValue(value.extraInfo));
     return obj;
 }
 
@@ -676,7 +676,7 @@ void RayToOverlayIntersectionResultFromScriptValue(const QScriptValue& objectVar
             value.intersection = newIntersection;
         }
     }
-    value.extraInfo = object["extraInfo"].toString();
+    value.extraInfo = object["extraInfo"].toMap();
 }
 
 bool Overlays::isLoaded(OverlayID id) {

interface/src/ui/overlays/Overlays.h

@@ -52,6 +52,7 @@ const OverlayID UNKNOWN_OVERLAY_ID = OverlayID();
  * @property {number} distance - The distance from the {@link PickRay} origin to the intersection point.
  * @property {Vec3} surfaceNormal - The normal of the overlay surface at the intersection point.
  * @property {Vec3} intersection - The position of the intersection point.
+ * @property {Object} extraInfo Additional intersection details, if available.
  */
 class RayToOverlayIntersectionResult {
 public:
@@ -61,7 +62,7 @@ public:
     BoxFace face;
     glm::vec3 surfaceNormal;
     glm::vec3 intersection;
-    QString extraInfo;
+    QVariantMap extraInfo;
 };
 
 

libraries/avatars/src/AvatarData.cpp

@@ -2420,6 +2420,7 @@ QScriptValue RayToAvatarIntersectionResultToScriptValue(QScriptEngine* engine, c
     obj.setProperty("distance", value.distance);
     QScriptValue intersection = vec3toScriptValue(engine, value.intersection);
     obj.setProperty("intersection", intersection);
+    obj.setProperty("extraInfo", engine->toScriptValue(value.extraInfo));
     return obj;
 }
 
@@ -2432,6 +2433,7 @@ void RayToAvatarIntersectionResultFromScriptValue(const QScriptValue& object, Ra
     if (intersection.isValid()) {
         vec3FromScriptValue(intersection, value.intersection);
     }
+    value.extraInfo = object.property("extraInfo").toVariant().toMap();
 }
 
 const float AvatarData::OUT_OF_VIEW_PENALTY = -10.0f;

libraries/avatars/src/AvatarData.h

@@ -968,6 +968,7 @@ RayToAvatarIntersectionResult() : intersects(false), avatarID(), distance(0) {}
     QUuid avatarID;
     float distance;
     glm::vec3 intersection;
+    QVariantMap extraInfo;
 };
 
 Q_DECLARE_METATYPE(RayToAvatarIntersectionResult)

libraries/entities-renderer/src/EntityTreeRenderer.cpp

@@ -668,15 +668,16 @@ void EntityTreeRenderer::mousePressEvent(QMouseEvent* event) {
     auto entityScriptingInterface = DependencyManager::get<EntityScriptingInterface>();
     PickRay ray = _viewState->computePickRay(event->x(), event->y());
     RayToEntityIntersectionResult rayPickResult = _getPrevRayPickResultOperator(_mouseRayPickID);
-    if (rayPickResult.intersects && rayPickResult.entity) {
+    EntityItemPointer entity;
-        auto properties = rayPickResult.entity->getProperties();
+    if (rayPickResult.intersects && (entity = getTree()->findEntityByID(rayPickResult.entityID))) {
+        auto properties = entity->getProperties();
         QString urlString = properties.getHref();
         QUrl url = QUrl(urlString, QUrl::StrictMode);
         if (url.isValid() && !url.isEmpty()){
             DependencyManager::get<AddressManager>()->handleLookupString(urlString);
         }
 
-        glm::vec2 pos2D = projectOntoEntityXYPlane(rayPickResult.entity, ray, rayPickResult);
+        glm::vec2 pos2D = projectOntoEntityXYPlane(entity, ray, rayPickResult);
         PointerEvent pointerEvent(PointerEvent::Press, PointerManager::MOUSE_POINTER_ID,
                                   pos2D, rayPickResult.intersection,
                                   rayPickResult.surfaceNormal, ray.direction,
@@ -707,8 +708,9 @@ void EntityTreeRenderer::mouseDoublePressEvent(QMouseEvent* event) {
     auto entityScriptingInterface = DependencyManager::get<EntityScriptingInterface>();
     PickRay ray = _viewState->computePickRay(event->x(), event->y());
     RayToEntityIntersectionResult rayPickResult = _getPrevRayPickResultOperator(_mouseRayPickID);
-    if (rayPickResult.intersects && rayPickResult.entity) {
+    EntityItemPointer entity;
-        glm::vec2 pos2D = projectOntoEntityXYPlane(rayPickResult.entity, ray, rayPickResult);
+    if (rayPickResult.intersects && (entity = getTree()->findEntityByID(rayPickResult.entityID))) {
+        glm::vec2 pos2D = projectOntoEntityXYPlane(entity, ray, rayPickResult);
         PointerEvent pointerEvent(PointerEvent::Press, PointerManager::MOUSE_POINTER_ID,
             pos2D, rayPickResult.intersection,
             rayPickResult.surfaceNormal, ray.direction,
@@ -737,10 +739,11 @@ void EntityTreeRenderer::mouseReleaseEvent(QMouseEvent* event) {
     auto entityScriptingInterface = DependencyManager::get<EntityScriptingInterface>();
     PickRay ray = _viewState->computePickRay(event->x(), event->y());
     RayToEntityIntersectionResult rayPickResult = _getPrevRayPickResultOperator(_mouseRayPickID);
-    if (rayPickResult.intersects && rayPickResult.entity) {
+    EntityItemPointer entity;
+    if (rayPickResult.intersects && (entity = getTree()->findEntityByID(rayPickResult.entityID))) {
         // qCDebug(entitiesrenderer) << "mouseReleaseEvent over entity:" << rayPickResult.entityID;
 
-        glm::vec2 pos2D = projectOntoEntityXYPlane(rayPickResult.entity, ray, rayPickResult);
+        glm::vec2 pos2D = projectOntoEntityXYPlane(entity, ray, rayPickResult);
         PointerEvent pointerEvent(PointerEvent::Release, PointerManager::MOUSE_POINTER_ID,
                                   pos2D, rayPickResult.intersection,
                                   rayPickResult.surfaceNormal, ray.direction,
@@ -756,7 +759,7 @@ void EntityTreeRenderer::mouseReleaseEvent(QMouseEvent* event) {
     // Even if we're no longer intersecting with an entity, if we started clicking on it, and now
     // we're releasing the button, then this is considered a clickReleaseOn event
     if (!_currentClickingOnEntityID.isInvalidID()) {
-        glm::vec2 pos2D = projectOntoEntityXYPlane(rayPickResult.entity, ray, rayPickResult);
+        glm::vec2 pos2D = projectOntoEntityXYPlane( entity, ray, rayPickResult);
         PointerEvent pointerEvent(PointerEvent::Release, PointerManager::MOUSE_POINTER_ID,
                                   pos2D, rayPickResult.intersection,
                                   rayPickResult.surfaceNormal, ray.direction,
@@ -781,8 +784,9 @@ void EntityTreeRenderer::mouseMoveEvent(QMouseEvent* event) {
     auto entityScriptingInterface = DependencyManager::get<EntityScriptingInterface>();
     PickRay ray = _viewState->computePickRay(event->x(), event->y());
     RayToEntityIntersectionResult rayPickResult = _getPrevRayPickResultOperator(_mouseRayPickID);
-    if (rayPickResult.intersects && rayPickResult.entity) {
+    EntityItemPointer entity;
-        glm::vec2 pos2D = projectOntoEntityXYPlane(rayPickResult.entity, ray, rayPickResult);
+    if (rayPickResult.intersects && (entity = getTree()->findEntityByID(rayPickResult.entityID))) {
+        glm::vec2 pos2D = projectOntoEntityXYPlane(entity, ray, rayPickResult);
         PointerEvent pointerEvent(PointerEvent::Move, PointerManager::MOUSE_POINTER_ID,
                                   pos2D, rayPickResult.intersection,
                                   rayPickResult.surfaceNormal, ray.direction,
@@ -796,7 +800,7 @@ void EntityTreeRenderer::mouseMoveEvent(QMouseEvent* event) {
         // if we were previously hovering over an entity, and this new entity is not the same as our previous entity
         // then we need to send the hover leave.
         if (!_currentHoverOverEntityID.isInvalidID() && rayPickResult.entityID != _currentHoverOverEntityID) {
-            glm::vec2 pos2D = projectOntoEntityXYPlane(rayPickResult.entity, ray, rayPickResult);
+            glm::vec2 pos2D = projectOntoEntityXYPlane(entity, ray, rayPickResult);
             PointerEvent pointerEvent(PointerEvent::Move, PointerManager::MOUSE_POINTER_ID,
                                       pos2D, rayPickResult.intersection,
                                       rayPickResult.surfaceNormal, ray.direction,
@@ -827,7 +831,7 @@ void EntityTreeRenderer::mouseMoveEvent(QMouseEvent* event) {
         // if we were previously hovering over an entity, and we're no longer hovering over any entity then we need to
         // send the hover leave for our previous entity
         if (!_currentHoverOverEntityID.isInvalidID()) {
-            glm::vec2 pos2D = projectOntoEntityXYPlane(rayPickResult.entity, ray, rayPickResult);
+            glm::vec2 pos2D = projectOntoEntityXYPlane(entity, ray, rayPickResult);
             PointerEvent pointerEvent(PointerEvent::Move, PointerManager::MOUSE_POINTER_ID,
                                   pos2D, rayPickResult.intersection,
                                   rayPickResult.surfaceNormal, ray.direction,

libraries/entities-renderer/src/RenderableModelEntityItem.cpp

@@ -282,7 +282,7 @@ bool RenderableModelEntityItem::supportsDetailedRayIntersection() const {
 
 bool RenderableModelEntityItem::findDetailedRayIntersection(const glm::vec3& origin, const glm::vec3& direction,
                          bool& keepSearching, OctreeElementPointer& element, float& distance, BoxFace& face,
-                         glm::vec3& surfaceNormal, void** intersectedObject, bool precisionPicking) const {
+                         glm::vec3& surfaceNormal, QVariantMap& extraInfo, bool precisionPicking) const {
     auto model = getModel();
     if (!model) {
         return true;
@@ -290,9 +290,8 @@ bool RenderableModelEntityItem::findDetailedRayIntersection(const glm::vec3& ori
     // qCDebug(entitiesrenderer) << "RenderableModelEntityItem::findDetailedRayIntersection() precisionPicking:"
     //                           << precisionPicking;
 
-    QString extraInfo;
     return model->findRayIntersectionAgainstSubMeshes(origin, direction, distance,
-                                                       face, surfaceNormal, extraInfo, precisionPicking, false);
+               face, surfaceNormal, extraInfo, precisionPicking, false);
 }
 
 void RenderableModelEntityItem::getCollisionGeometryResource() {

libraries/entities-renderer/src/RenderableModelEntityItem.h

@@ -70,7 +70,7 @@ public:
     virtual bool findDetailedRayIntersection(const glm::vec3& origin, const glm::vec3& direction,
                         bool& keepSearching, OctreeElementPointer& element, float& distance,
                         BoxFace& face, glm::vec3& surfaceNormal,
-                        void** intersectedObject, bool precisionPicking) const override;
+                        QVariantMap& extraInfo, bool precisionPicking) const override;
 
     virtual void setShapeType(ShapeType type) override;
     virtual void setCompoundShapeURL(const QString& url) override;

libraries/entities-renderer/src/RenderablePolyVoxEntityItem.cpp

@@ -565,7 +565,7 @@ public:
 bool RenderablePolyVoxEntityItem::findDetailedRayIntersection(const glm::vec3& origin, const glm::vec3& direction,
                                                               bool& keepSearching, OctreeElementPointer& element,
                                                               float& distance, BoxFace& face, glm::vec3& surfaceNormal,
-                                                              void** intersectedObject, bool precisionPicking) const
+                                                              QVariantMap& extraInfo, bool precisionPicking) const
 {
     // TODO -- correctly pick against marching-cube generated meshes
     if (!precisionPicking) {

libraries/entities-renderer/src/RenderablePolyVoxEntityItem.h

@@ -55,7 +55,7 @@ public:
     virtual bool findDetailedRayIntersection(const glm::vec3& origin, const glm::vec3& direction,
                         bool& keepSearching, OctreeElementPointer& element, float& distance,
                         BoxFace& face, glm::vec3& surfaceNormal,
-                        void** intersectedObject, bool precisionPicking) const override;
+                        QVariantMap& extraInfo, bool precisionPicking) const override;
 
     virtual void setVoxelData(const QByteArray& voxelData) override;
     virtual void setVoxelVolumeSize(const glm::vec3& voxelVolumeSize) override;

libraries/entities/src/EntityItem.h

@@ -160,7 +160,7 @@ public:
     virtual bool findDetailedRayIntersection(const glm::vec3& origin, const glm::vec3& direction,
                          bool& keepSearching, OctreeElementPointer& element, float& distance,
                          BoxFace& face, glm::vec3& surfaceNormal,
-                         void** intersectedObject, bool precisionPicking) const { return true; }
+                         QVariantMap& extraInfo, bool precisionPicking) const { return true; }
 
     // attributes applicable to all entity types
     EntityTypes::EntityType getType() const { return _type; }

libraries/entities/src/EntityScriptingInterface.cpp

@@ -778,13 +778,12 @@ RayToEntityIntersectionResult EntityScriptingInterface::findRayIntersectionWorke
     RayToEntityIntersectionResult result;
     if (_entityTree) {
         OctreeElementPointer element;
-        EntityItemPointer intersectedEntity = NULL;
+        result.entityID = _entityTree->findRayIntersection(ray.origin, ray.direction,
-        result.intersects = _entityTree->findRayIntersection(ray.origin, ray.direction,
             entityIdsToInclude, entityIdsToDiscard, visibleOnly, collidableOnly, precisionPicking,
             element, result.distance, result.face, result.surfaceNormal,
-            (void**)&intersectedEntity, lockType, &result.accurate);
+            result.extraInfo, lockType, &result.accurate);
-        if (result.intersects && intersectedEntity) {
+        result.intersects = !result.entityID.isNull();
-            result.entityID = intersectedEntity->getEntityItemID();
+        if (result.intersects) {
             result.intersection = ray.origin + (ray.direction * result.distance);
         }
     }
@@ -950,8 +949,7 @@ RayToEntityIntersectionResult::RayToEntityIntersectionResult() :
     accurate(true), // assume it's accurate
     entityID(),
     distance(0),
-    face(),
+    face()
-    entity(NULL)
 {
 }
 
@@ -998,6 +996,7 @@ QScriptValue RayToEntityIntersectionResultToScriptValue(QScriptEngine* engine, c
 
     QScriptValue surfaceNormal = vec3toScriptValue(engine, value.surfaceNormal);
     obj.setProperty("surfaceNormal", surfaceNormal);
+    obj.setProperty("extraInfo", engine->toScriptValue(value.extraInfo));
     return obj;
 }
 
@@ -1033,6 +1032,7 @@ void RayToEntityIntersectionResultFromScriptValue(const QScriptValue& object, Ra
     if (surfaceNormal.isValid()) {
         vec3FromScriptValue(surfaceNormal, value.surfaceNormal);
     }
+    value.extraInfo = object.property("extraInfo").toVariant().toMap();
 }
 
 bool EntityScriptingInterface::polyVoxWorker(QUuid entityID, std::function<bool(PolyVoxEntityItem&)> actor) {

libraries/entities/src/EntityScriptingInterface.h

@@ -62,7 +62,7 @@ public:
     BoxFace face;
     glm::vec3 intersection;
     glm::vec3 surfaceNormal;
-    EntityItemPointer entity;
+    QVariantMap extraInfo;
 };
 
 Q_DECLARE_METATYPE(RayToEntityIntersectionResult)

libraries/entities/src/EntityTree.cpp

@@ -61,8 +61,8 @@ public:
     float& distance;
     BoxFace& face;
     glm::vec3& surfaceNormal;
-    void** intersectedObject;
+    QVariantMap& extraInfo;
-    bool found;
+    EntityItemID entityID;
 };
 
 
@@ -750,23 +750,24 @@ bool findRayIntersectionOp(const OctreeElementPointer& element, void* extraData)
     RayArgs* args = static_cast<RayArgs*>(extraData);
     bool keepSearching = true;
     EntityTreeElementPointer entityTreeElementPointer = std::static_pointer_cast<EntityTreeElement>(element);
-    if (entityTreeElementPointer->findRayIntersection(args->origin, args->direction, keepSearching,
+    EntityItemID entityID = entityTreeElementPointer->findRayIntersection(args->origin, args->direction, keepSearching,
         args->element, args->distance, args->face, args->surfaceNormal, args->entityIdsToInclude,
-        args->entityIdsToDiscard, args->visibleOnly, args->collidableOnly, args->intersectedObject, args->precisionPicking)) {
+        args->entityIdsToDiscard, args->visibleOnly, args->collidableOnly, args->extraInfo, args->precisionPicking);
-        args->found = true;
+    if (!entityID.isNull()) {
+        args->entityID = entityID;
     }
     return keepSearching;
 }
 
-bool EntityTree::findRayIntersection(const glm::vec3& origin, const glm::vec3& direction,
+EntityItemID EntityTree::findRayIntersection(const glm::vec3& origin, const glm::vec3& direction,
                                     QVector<EntityItemID> entityIdsToInclude, QVector<EntityItemID> entityIdsToDiscard,
                                     bool visibleOnly, bool collidableOnly, bool precisionPicking, 
                                     OctreeElementPointer& element, float& distance,
-                                    BoxFace& face, glm::vec3& surfaceNormal, void** intersectedObject,
+                                    BoxFace& face, glm::vec3& surfaceNormal, QVariantMap& extraInfo,
                                     Octree::lockType lockType, bool* accurateResult) {
     RayArgs args = { origin, direction, entityIdsToInclude, entityIdsToDiscard,
             visibleOnly, collidableOnly, precisionPicking,
-            element, distance, face, surfaceNormal,  intersectedObject, false };
+            element, distance, face, surfaceNormal, extraInfo, EntityItemID() };
     distance = FLT_MAX;
 
     bool requireLock = lockType == Octree::Lock;
@@ -778,7 +779,7 @@ bool EntityTree::findRayIntersection(const glm::vec3& origin, const glm::vec3& d
         *accurateResult = lockResult; // if user asked to accuracy or result, let them know this is accurate
     }
 
-    return args.found;
+    return args.entityID;
 }
 
 

libraries/entities/src/EntityTree.h

@@ -97,11 +97,11 @@ public:
     virtual void processChallengeOwnershipReplyPacket(ReceivedMessage& message, const SharedNodePointer& sourceNode) override;
     virtual void processChallengeOwnershipPacket(ReceivedMessage& message, const SharedNodePointer& sourceNode) override;
 
-    virtual bool findRayIntersection(const glm::vec3& origin, const glm::vec3& direction,
+    virtual EntityItemID findRayIntersection(const glm::vec3& origin, const glm::vec3& direction,
         QVector<EntityItemID> entityIdsToInclude, QVector<EntityItemID> entityIdsToDiscard,
         bool visibleOnly, bool collidableOnly, bool precisionPicking, 
         OctreeElementPointer& node, float& distance,
-        BoxFace& face, glm::vec3& surfaceNormal, void** intersectedObject = NULL,
+        BoxFace& face, glm::vec3& surfaceNormal, QVariantMap& extraInfo,
         Octree::lockType lockType = Octree::TryLock, bool* accurateResult = NULL);
 
     virtual bool rootElementHasData() const override { return true; }

libraries/entities/src/EntityTreeElement.cpp

@@ -588,57 +588,60 @@ bool EntityTreeElement::bestFitBounds(const glm::vec3& minPoint, const glm::vec3
     return false;
 }
 
-bool EntityTreeElement::findRayIntersection(const glm::vec3& origin, const glm::vec3& direction,
+EntityItemID EntityTreeElement::findRayIntersection(const glm::vec3& origin, const glm::vec3& direction,
     bool& keepSearching, OctreeElementPointer& element, float& distance,
     BoxFace& face, glm::vec3& surfaceNormal, const QVector<EntityItemID>& entityIdsToInclude,
     const QVector<EntityItemID>& entityIdsToDiscard, bool visibleOnly, bool collidableOnly,
-    void** intersectedObject, bool precisionPicking) {
+    QVariantMap& extraInfo, bool precisionPicking) {
 
     keepSearching = true; // assume that we will continue searching after this.
 
+    EntityItemID result;
     float distanceToElementCube = std::numeric_limits<float>::max();
     float distanceToElementDetails = distance;
     BoxFace localFace;
     glm::vec3 localSurfaceNormal;
+    QVariantMap localExtraInfo;
 
     // if the ray doesn't intersect with our cube, we can stop searching!
     if (!_cube.findRayIntersection(origin, direction, distanceToElementCube, localFace, localSurfaceNormal)) {
         keepSearching = false; // no point in continuing to search
-        return false; // we did not intersect
+        return result; // we did not intersect
     }
 
     // by default, we only allow intersections with leaves with content
     if (!canRayIntersect()) {
-        return false; // we don't intersect with non-leaves, and we keep searching
+        return result; // we don't intersect with non-leaves, and we keep searching
     }
 
     // if the distance to the element cube is not less than the current best distance, then it's not possible
     // for any details inside the cube to be closer so we don't need to consider them.
     if (_cube.contains(origin) || distanceToElementCube < distance) {
 
-        if (findDetailedRayIntersection(origin, direction, keepSearching, element, distanceToElementDetails,
+        EntityItemID entityID = findDetailedRayIntersection(origin, direction, keepSearching, element, distanceToElementDetails,
                 face, localSurfaceNormal, entityIdsToInclude, entityIdsToDiscard, visibleOnly, collidableOnly,
-                intersectedObject, precisionPicking, distanceToElementCube)) {
+                localExtraInfo, precisionPicking, distanceToElementCube);
-
+        if (!entityID.isNull()) {
             if (distanceToElementDetails < distance) {
                 distance = distanceToElementDetails;
                 face = localFace;
                 surfaceNormal = localSurfaceNormal;
-                return true;
+                extraInfo = localExtraInfo;
+                result = entityID;
             }
         }
     }
-    return false;
+    return result;
 }
 
-bool EntityTreeElement::findDetailedRayIntersection(const glm::vec3& origin, const glm::vec3& direction, bool& keepSearching,
+EntityItemID EntityTreeElement::findDetailedRayIntersection(const glm::vec3& origin, const glm::vec3& direction, bool& keepSearching,
                                     OctreeElementPointer& element, float& distance, BoxFace& face, glm::vec3& surfaceNormal,
                                     const QVector<EntityItemID>& entityIdsToInclude, const QVector<EntityItemID>& entityIDsToDiscard,
-                                    bool visibleOnly, bool collidableOnly, void** intersectedObject, bool precisionPicking, float distanceToElementCube) {
+                                    bool visibleOnly, bool collidableOnly, QVariantMap& extraInfo, bool precisionPicking, float distanceToElementCube) {
 
     // only called if we do intersect our bounding cube, but find if we actually intersect with entities...
     int entityNumber = 0;
-    bool somethingIntersected = false;
+    EntityItemID somethingIntersected;
     forEachEntity([&](EntityItemPointer entity) {
         if ( (visibleOnly && !entity->isVisible()) || (collidableOnly && (entity->getCollisionless() || entity->getShapeType() == SHAPE_TYPE_NONE))
             || (entityIdsToInclude.size() > 0 && !entityIdsToInclude.contains(entity->getID()))
@@ -655,6 +658,7 @@ bool EntityTreeElement::findDetailedRayIntersection(const glm::vec3& origin, con
         float localDistance;
         BoxFace localFace;
         glm::vec3 localSurfaceNormal;
+        QVariantMap localExtraInfo;
 
         // if the ray doesn't intersect with our cube, we can stop searching!
         if (!entityBox.findRayIntersection(origin, direction, localDistance, localFace, localSurfaceNormal)) {
@@ -684,14 +688,14 @@ bool EntityTreeElement::findDetailedRayIntersection(const glm::vec3& origin, con
                 // now ask the entity if we actually intersect
                 if (entity->supportsDetailedRayIntersection()) {
                     if (entity->findDetailedRayIntersection(origin, direction, keepSearching, element, localDistance,
-                        localFace, localSurfaceNormal, intersectedObject, precisionPicking)) {
+                        localFace, localSurfaceNormal, localExtraInfo, precisionPicking)) {
 
                         if (localDistance < distance) {
                             distance = localDistance;
                             face = localFace;
                             surfaceNormal = localSurfaceNormal;
-                            *intersectedObject = (void*)entity.get();
+                            extraInfo = localExtraInfo;
-                            somethingIntersected = true;
+                            somethingIntersected = entity->getEntityItemID();
                         }
                     }
                 } else {
@@ -701,8 +705,7 @@ bool EntityTreeElement::findDetailedRayIntersection(const glm::vec3& origin, con
                         distance = localDistance;
                         face = localFace;
                         surfaceNormal = glm::vec3(rotation * glm::vec4(localSurfaceNormal, 1.0f));
-                        *intersectedObject = (void*)entity.get();
+                        somethingIntersected = entity->getEntityItemID();
-                        somethingIntersected = true;
                     }
                 }
             }

libraries/entities/src/EntityTreeElement.h

@@ -146,16 +146,16 @@ public:
     virtual bool deleteApproved() const override { return !hasEntities(); }
 
     virtual bool canRayIntersect() const override { return hasEntities(); }
-    virtual bool findRayIntersection(const glm::vec3& origin, const glm::vec3& direction,
+    virtual EntityItemID findRayIntersection(const glm::vec3& origin, const glm::vec3& direction,
         bool& keepSearching, OctreeElementPointer& node, float& distance,
         BoxFace& face, glm::vec3& surfaceNormal, const QVector<EntityItemID>& entityIdsToInclude,
-        const QVector<EntityItemID>& entityIdsToDiscard, bool visibleOnly = false, bool collidableOnly = false,
+        const QVector<EntityItemID>& entityIdsToDiscard, bool visibleOnly, bool collidableOnly,
-        void** intersectedObject = NULL, bool precisionPicking = false);
+        QVariantMap& extraInfo, bool precisionPicking = false);
-    virtual bool findDetailedRayIntersection(const glm::vec3& origin, const glm::vec3& direction,
+    virtual EntityItemID findDetailedRayIntersection(const glm::vec3& origin, const glm::vec3& direction,
                          bool& keepSearching, OctreeElementPointer& element, float& distance,
                          BoxFace& face, glm::vec3& surfaceNormal, const QVector<EntityItemID>& entityIdsToInclude,
                          const QVector<EntityItemID>& entityIdsToDiscard, bool visibleOnly, bool collidableOnly,
-                         void** intersectedObject, bool precisionPicking, float distanceToElementCube);
+                         QVariantMap& extraInfo, bool precisionPicking, float distanceToElementCube);
     virtual bool findSpherePenetration(const glm::vec3& center, float radius,
                         glm::vec3& penetration, void** penetratedObject) const override;
 

libraries/entities/src/LightEntityItem.cpp

@@ -300,7 +300,7 @@ void LightEntityItem::resetLightPropertiesChanged() {
 bool LightEntityItem::findDetailedRayIntersection(const glm::vec3& origin, const glm::vec3& direction,
                         bool& keepSearching, OctreeElementPointer& element, float& distance,
                         BoxFace& face, glm::vec3& surfaceNormal,
-                        void** intersectedObject, bool precisionPicking) const {
+                        QVariantMap& extraInfo, bool precisionPicking) const {
 
     // TODO: consider if this is really what we want to do. We've made it so that "lights are pickable" is a global state
     // this is probably reasonable since there's typically only one tree you'd be picking on at a time. Technically we could

libraries/entities/src/LightEntityItem.h

@@ -88,7 +88,7 @@ public:
     virtual bool findDetailedRayIntersection(const glm::vec3& origin, const glm::vec3& direction,
                             bool& keepSearching, OctreeElementPointer& element, float& distance,
                             BoxFace& face, glm::vec3& surfaceNormal,
-                            void** intersectedObject, bool precisionPicking) const override;
+                            QVariantMap& extraInfo, bool precisionPicking) const override;
 
 private:
     // properties of a light

libraries/entities/src/LineEntityItem.h

@@ -63,7 +63,7 @@ class LineEntityItem : public EntityItem {
     virtual bool findDetailedRayIntersection(const glm::vec3& origin, const glm::vec3& direction,
                                              bool& keepSearching, OctreeElementPointer& element, float& distance,
                                              BoxFace& face, glm::vec3& surfaceNormal,
-                                             void** intersectedObject,
+                                             QVariantMap& extraInfo,
                                              bool precisionPicking) const override { return false; }
     bool pointsChanged() const { return _pointsChanged; }
     void resetPointsChanged();

libraries/entities/src/PolyLineEntityItem.h

@@ -96,7 +96,7 @@ class PolyLineEntityItem : public EntityItem {
     virtual bool findDetailedRayIntersection(const glm::vec3& origin, const glm::vec3& direction,
                                              bool& keepSearching, OctreeElementPointer& element, float& distance,
                                              BoxFace& face, glm::vec3& surfaceNormal,
-                                             void** intersectedObject, bool precisionPicking) const override { return false; }
+                                             QVariantMap& extraInfo, bool precisionPicking) const override { return false; }
 
     // disable these external interfaces as PolyLineEntities caculate their own dimensions based on the points they contain
     virtual void setRegistrationPoint(const glm::vec3& value) override {}; // FIXME: this is suspicious! 

libraries/entities/src/PolyVoxEntityItem.h

@@ -47,7 +47,7 @@ class PolyVoxEntityItem : public EntityItem {
     virtual bool findDetailedRayIntersection(const glm::vec3& origin, const glm::vec3& direction,
                                              bool& keepSearching, OctreeElementPointer& element, float& distance,
                                              BoxFace& face, glm::vec3& surfaceNormal,
-                                             void** intersectedObject, bool precisionPicking) const override { return false; }
+                                             QVariantMap& extraInfo, bool precisionPicking) const override { return false; }
 
     virtual void debugDump() const override;
 

libraries/entities/src/ShapeEntityItem.cpp

@@ -223,7 +223,7 @@ bool ShapeEntityItem::supportsDetailedRayIntersection() const {
 bool ShapeEntityItem::findDetailedRayIntersection(const glm::vec3& origin, const glm::vec3& direction,
                                                    bool& keepSearching, OctreeElementPointer& element,
                                                    float& distance, BoxFace& face, glm::vec3& surfaceNormal,
-                                                   void** intersectedObject, bool precisionPicking) const {
+                                                   QVariantMap& extraInfo, bool precisionPicking) const {
     // determine the ray in the frame of the entity transformed from a unit sphere
     glm::mat4 entityToWorldMatrix = getEntityToWorldMatrix();
     glm::mat4 worldToEntityMatrix = glm::inverse(entityToWorldMatrix);

libraries/entities/src/ShapeEntityItem.h

@@ -94,7 +94,7 @@ public:
     bool findDetailedRayIntersection(const glm::vec3& origin, const glm::vec3& direction,
                                                 bool& keepSearching, OctreeElementPointer& element, float& distance,
                                                 BoxFace& face, glm::vec3& surfaceNormal,
-                                                void** intersectedObject, bool precisionPicking) const override;
+                                                QVariantMap& extraInfo, bool precisionPicking) const override;
 
     void debugDump() const override;
 

libraries/entities/src/TextEntityItem.cpp

@@ -131,7 +131,7 @@ void TextEntityItem::appendSubclassData(OctreePacketData* packetData, EncodeBits
 bool TextEntityItem::findDetailedRayIntersection(const glm::vec3& origin, const glm::vec3& direction,
                      bool& keepSearching, OctreeElementPointer& element, float& distance, 
                      BoxFace& face, glm::vec3& surfaceNormal,
-                     void** intersectedObject, bool precisionPicking) const {
+                     QVariantMap& extraInfo, bool precisionPicking) const {
     glm::vec3 dimensions = getDimensions();
     glm::vec2 xyDimensions(dimensions.x, dimensions.y);
     glm::quat rotation = getWorldOrientation();

libraries/entities/src/TextEntityItem.h

@@ -50,7 +50,7 @@ public:
     virtual bool findDetailedRayIntersection(const glm::vec3& origin, const glm::vec3& direction,
                          bool& keepSearching, OctreeElementPointer& element, float& distance,
                          BoxFace& face, glm::vec3& surfaceNormal,
-                         void** intersectedObject, bool precisionPicking) const override;
+                         QVariantMap& extraInfo, bool precisionPicking) const override;
 
     static const QString DEFAULT_TEXT;
     void setText(const QString& value);

libraries/entities/src/WebEntityItem.cpp

@@ -108,7 +108,7 @@ void WebEntityItem::appendSubclassData(OctreePacketData* packetData, EncodeBitst
 bool WebEntityItem::findDetailedRayIntersection(const glm::vec3& origin, const glm::vec3& direction,
                                                 bool& keepSearching, OctreeElementPointer& element, float& distance,
                                                 BoxFace& face, glm::vec3& surfaceNormal,
-                                                void** intersectedObject, bool precisionPicking) const {
+                                                QVariantMap& extraInfo, bool precisionPicking) const {
     glm::vec3 dimensions = getDimensions();
     glm::vec2 xyDimensions(dimensions.x, dimensions.y);
     glm::quat rotation = getWorldOrientation();

libraries/entities/src/WebEntityItem.h

@@ -49,7 +49,7 @@ public:
     virtual bool findDetailedRayIntersection(const glm::vec3& origin, const glm::vec3& direction,
                          bool& keepSearching, OctreeElementPointer& element, float& distance,
                          BoxFace& face, glm::vec3& surfaceNormal,
-                         void** intersectedObject, bool precisionPicking) const override;
+                         QVariantMap& extraInfo, bool precisionPicking) const override;
 
     virtual void setSourceUrl(const QString& value);
     QString getSourceUrl() const;

libraries/entities/src/ZoneEntityItem.cpp

@@ -274,7 +274,7 @@ void ZoneEntityItem::setCompoundShapeURL(const QString& url) {
 bool ZoneEntityItem::findDetailedRayIntersection(const glm::vec3& origin, const glm::vec3& direction,
                          bool& keepSearching, OctreeElementPointer& element, float& distance, 
                          BoxFace& face, glm::vec3& surfaceNormal,
-                         void** intersectedObject, bool precisionPicking) const {
+                         QVariantMap& extraInfo, bool precisionPicking) const {
 
     return _zonesArePickable;
 }

libraries/entities/src/ZoneEntityItem.h

@@ -102,7 +102,7 @@ public:
     virtual bool findDetailedRayIntersection(const glm::vec3& origin, const glm::vec3& direction,
                          bool& keepSearching, OctreeElementPointer& element, float& distance,
                          BoxFace& face, glm::vec3& surfaceNormal,
-                         void** intersectedObject, bool precisionPicking) const override;
+                         QVariantMap& extraInfo, bool precisionPicking) const override;
 
     virtual void debugDump() const override;
 

libraries/render-utils/src/Model.cpp

@@ -337,8 +337,8 @@ void Model::initJointStates() {
 }
 
 bool Model::findRayIntersectionAgainstSubMeshes(const glm::vec3& origin, const glm::vec3& direction, float& distance,
-                                                    BoxFace& face, glm::vec3& surfaceNormal,
+                                                BoxFace& face, glm::vec3& surfaceNormal, QVariantMap& extraInfo,
-                                                    QString& extraInfo, bool pickAgainstTriangles, bool allowBackface) {
+                                                bool pickAgainstTriangles, bool allowBackface) {
 
     bool intersectedSomething = false;
 
@@ -369,6 +369,10 @@ bool Model::findRayIntersectionAgainstSubMeshes(const glm::vec3& origin, const g
         QMutexLocker locker(&_mutex);
 
         float bestDistance = std::numeric_limits<float>::max();
+        Triangle bestModelTriangle;
+        Triangle bestWorldTriangle;
+        int bestSubMeshIndex = 0;
+
         int subMeshIndex = 0;
         const FBXGeometry& geometry = getFBXGeometry();
 
@@ -386,8 +390,8 @@ bool Model::findRayIntersectionAgainstSubMeshes(const glm::vec3& origin, const g
         for (auto& triangleSet : _modelSpaceMeshTriangleSets) {
             float triangleSetDistance = 0.0f;
             BoxFace triangleSetFace;
-            glm::vec3 triangleSetNormal;
+            Triangle triangleSetTriangle;
-            if (triangleSet.findRayIntersection(meshFrameOrigin, meshFrameDirection, triangleSetDistance, triangleSetFace, triangleSetNormal, pickAgainstTriangles, allowBackface)) {
+            if (triangleSet.findRayIntersection(meshFrameOrigin, meshFrameDirection, triangleSetDistance, triangleSetFace, triangleSetTriangle, pickAgainstTriangles, allowBackface)) {
 
                 glm::vec3 meshIntersectionPoint = meshFrameOrigin + (meshFrameDirection * triangleSetDistance);
                 glm::vec3 worldIntersectionPoint = glm::vec3(meshToWorldMatrix * glm::vec4(meshIntersectionPoint, 1.0f));
@@ -397,8 +401,11 @@ bool Model::findRayIntersectionAgainstSubMeshes(const glm::vec3& origin, const g
                     bestDistance = worldDistance;
                     intersectedSomething = true;
                     face = triangleSetFace;
-                    surfaceNormal = glm::vec3(meshToWorldMatrix * glm::vec4(triangleSetNormal, 0.0f));
+                    bestModelTriangle = triangleSetTriangle;
-                    extraInfo = geometry.getModelNameOfMesh(subMeshIndex);
+                    bestWorldTriangle = triangleSetTriangle * meshToWorldMatrix;
+                    extraInfo["_worldIntersectionPoint"] = vec3toVariant(worldIntersectionPoint);
+                    extraInfo["_meshIntersectionPoint"] = vec3toVariant(meshIntersectionPoint);
+                    bestSubMeshIndex = subMeshIndex;
                 }
             }
             subMeshIndex++;
@@ -406,9 +413,24 @@ bool Model::findRayIntersectionAgainstSubMeshes(const glm::vec3& origin, const g
 
         if (intersectedSomething) {
             distance = bestDistance;
+            surfaceNormal = bestWorldTriangle.getNormal();
+            if (pickAgainstTriangles) {
+                extraInfo["subMeshIndex"] = bestSubMeshIndex;
+                extraInfo["subMeshName"] = geometry.getModelNameOfMesh(bestSubMeshIndex);
+                extraInfo["subMeshTriangleWorld"] = QVariantMap{
+                    { "v0", vec3toVariant(bestWorldTriangle.v0) },
+                    { "v1", vec3toVariant(bestWorldTriangle.v1) },
+                    { "v2", vec3toVariant(bestWorldTriangle.v2) },
+                };
+                extraInfo["subMeshNormal"] = vec3toVariant(bestModelTriangle.getNormal());
+                extraInfo["subMeshTriangle"] = QVariantMap{
+                    { "v0", vec3toVariant(bestModelTriangle.v0) },
+                    { "v1", vec3toVariant(bestModelTriangle.v1) },
+                    { "v2", vec3toVariant(bestModelTriangle.v2) },
+                };
+            }
+            
         }
-
-        return intersectedSomething;
     }
 
     return intersectedSomething;

libraries/render-utils/src/Model.h

@@ -158,8 +158,8 @@ public:
     void setJointTranslation(int index, bool valid, const glm::vec3& translation, float priority);
 
     bool findRayIntersectionAgainstSubMeshes(const glm::vec3& origin, const glm::vec3& direction, float& distance,
-                                             BoxFace& face, glm::vec3& surfaceNormal, 
+                                             BoxFace& face, glm::vec3& surfaceNormal,
-                                             QString& extraInfo, bool pickAgainstTriangles = false, bool allowBackface = false);
+                                             QVariantMap& extraInfo, bool pickAgainstTriangles = false, bool allowBackface = false);
 
     void setOffset(const glm::vec3& offset);
     const glm::vec3& getOffset() const { return _offset; }

libraries/script-engine/src/Mat4.cpp

@@ -11,7 +11,9 @@
 
 #include <QDebug>
 #include <GLMHelpers.h>
+#include <glm/glm.hpp>
 #include <glm/gtx/string_cast.hpp>
+#include <glm/gtc/type_ptr.hpp>
 #include "ScriptEngineLogging.h"
 #include "ScriptEngine.h"
 #include "Mat4.h"
@@ -32,6 +34,14 @@ glm::mat4 Mat4::createFromColumns(const glm::vec4& col0, const glm::vec4& col1,
     return glm::mat4(col0, col1, col2, col3);
 }
 
+glm::mat4 Mat4::createFromArray(const QVector<float>& floats) const {
+    if (floats.size() != 16 && floats.size() != 9) {
+        context()->throwError("createFromVector requires 16 floats for mat4 (or 9 if providing a mat3)");
+        return glm::mat4();
+    }
+    return floats.size() == 9 ? glm::mat4(glm::make_mat3(floats.constData())) : glm::make_mat4(floats.constData());
+}
+
 glm::vec3 Mat4::extractTranslation(const glm::mat4& m) const {
     return ::extractTranslation(m);
 }

libraries/script-engine/src/Mat4.h

@@ -17,6 +17,9 @@
 #include <QObject>
 #include <QString>
 #include <QtScript/QScriptable>
+#include <QVector>
+#include <glm/glm.hpp>
+#include "RegisteredMetaTypes.h"
 
 /// Scriptable Mat4 object.  Used exclusively in the JavaScript API
 class Mat4 : public QObject, protected QScriptable {
@@ -28,6 +31,7 @@ public slots:
     glm::mat4 createFromRotAndTrans(const glm::quat& rot, const glm::vec3& trans) const;
     glm::mat4 createFromScaleRotAndTrans(const glm::vec3& scale, const glm::quat& rot, const glm::vec3& trans) const;
     glm::mat4 createFromColumns(const glm::vec4& col0, const glm::vec4& col1, const glm::vec4& col2, const glm::vec4& col3) const;
+    glm::mat4 createFromArray(const QVector<float>& floats) const;
 
     glm::vec3 extractTranslation(const glm::mat4& m) const;
     glm::quat extractRotation(const glm::mat4& m) const;

libraries/shared/src/GeometryUtil.cpp

@@ -292,6 +292,14 @@ glm::vec3 Triangle::getNormal() const {
     return glm::normalize(glm::cross(edge1, edge2));
 }
 
+Triangle Triangle::operator*(const glm::mat4& transform) const {
+    return {
+        glm::vec3(transform * glm::vec4(v0, 1.0f)),
+        glm::vec3(transform * glm::vec4(v1, 1.0f)),
+        glm::vec3(transform * glm::vec4(v2, 1.0f))
+    };
+}
+
 bool findRayTriangleIntersection(const glm::vec3& origin, const glm::vec3& direction,
         const glm::vec3& v0, const glm::vec3& v1, const glm::vec3& v2, float& distance, bool allowBackface) {
     glm::vec3 firstSide = v0 - v1;

libraries/shared/src/GeometryUtil.h

@@ -104,6 +104,7 @@ public:
     glm::vec3 v1;
     glm::vec3 v2;
     glm::vec3 getNormal() const;
+    Triangle operator*(const glm::mat4& transform) const;
 };
 
 inline bool findRayTriangleIntersection(const glm::vec3& origin, const glm::vec3& direction,

libraries/shared/src/TriangleSet.cpp

@@ -31,7 +31,7 @@ void TriangleSet::clear() {
 }
 
 bool TriangleSet::findRayIntersection(const glm::vec3& origin, const glm::vec3& direction,
-    float& distance, BoxFace& face, glm::vec3& surfaceNormal, bool precision, bool allowBackface) {
+         float& distance, BoxFace& face, Triangle& triangle, bool precision, bool allowBackface) {
 
     // reset our distance to be the max possible, lower level tests will store best distance here
     distance = std::numeric_limits<float>::max();
@@ -41,7 +41,7 @@ bool TriangleSet::findRayIntersection(const glm::vec3& origin, const glm::vec3&
     }
 
     int trianglesTouched = 0;
-    auto result = _triangleOctree.findRayIntersection(origin, direction, distance, face, surfaceNormal, precision, trianglesTouched, allowBackface);
+    auto result = _triangleOctree.findRayIntersection(origin, direction, distance, face, triangle, precision, trianglesTouched, allowBackface);
 
     #if WANT_DEBUGGING
     if (precision) {
@@ -95,11 +95,12 @@ void TriangleSet::balanceOctree() {
 // Determine of the given ray (origin/direction) in model space intersects with any triangles
 // in the set. If an intersection occurs, the distance and surface normal will be provided.
 bool TriangleSet::TriangleOctreeCell::findRayIntersectionInternal(const glm::vec3& origin, const glm::vec3& direction,
-            float& distance, BoxFace& face, glm::vec3& surfaceNormal, bool precision, int& trianglesTouched, bool allowBackface) {
+            float& distance, BoxFace& face, Triangle& triangle, bool precision, int& trianglesTouched, bool allowBackface) {
 
     bool intersectedSomething = false;
     float boxDistance = distance;
     float bestDistance = distance;
+    glm::vec3 surfaceNormal;
 
     if (_bounds.findRayIntersection(origin, direction, boxDistance, face, surfaceNormal)) {
 
@@ -112,14 +113,14 @@ bool TriangleSet::TriangleOctreeCell::findRayIntersectionInternal(const glm::vec
 
         if (precision) {
             for (const auto& triangleIndex : _triangleIndices) {
-                const auto& triangle = _allTriangles[triangleIndex];
+                const auto& thisTriangle = _allTriangles[triangleIndex];
                 float thisTriangleDistance;
                 trianglesTouched++;
-                if (findRayTriangleIntersection(origin, direction, triangle, thisTriangleDistance, allowBackface)) {
+                if (findRayTriangleIntersection(origin, direction, thisTriangle, thisTriangleDistance, allowBackface)) {
                     if (thisTriangleDistance < bestDistance) {
                         bestDistance = thisTriangleDistance;
                         intersectedSomething = true;
-                        surfaceNormal = triangle.getNormal();
+                        triangle = thisTriangle;
                         distance = bestDistance;
                     }
                 }
@@ -204,7 +205,8 @@ void TriangleSet::TriangleOctreeCell::insert(size_t triangleIndex) {
 }
 
 bool TriangleSet::TriangleOctreeCell::findRayIntersection(const glm::vec3& origin, const glm::vec3& direction,
-                float& distance, BoxFace& face, glm::vec3& surfaceNormal, bool precision, int& trianglesTouched, bool allowBackface) {
+                float& distance, BoxFace& face, Triangle& triangle, bool precision, int& trianglesTouched,
+                bool allowBackface) {
 
     if (_population < 1) {
         return false; // no triangles below here, so we can't intersect
@@ -212,6 +214,7 @@ bool TriangleSet::TriangleOctreeCell::findRayIntersection(const glm::vec3& origi
 
     float bestLocalDistance = distance;
     BoxFace bestLocalFace;
+    Triangle bestLocalTriangle;
     glm::vec3 bestLocalNormal;
     bool intersects = false;
 
@@ -229,7 +232,7 @@ bool TriangleSet::TriangleOctreeCell::findRayIntersection(const glm::vec3& origi
 
         float childDistance = distance;
         BoxFace childFace;
-        glm::vec3 childNormal;
+        Triangle childTriangle;
 
         // if we're not yet at the max depth, then check which child the triangle fits in
         if (_depth < MAX_DEPTH) {
@@ -237,22 +240,22 @@ bool TriangleSet::TriangleOctreeCell::findRayIntersection(const glm::vec3& origi
                 // check each child, if there's an intersection, it will return some distance that we need
                 // to compare against the other results, because there might be multiple intersections and
                 // we will always choose the best (shortest) intersection
-                if (child.second.findRayIntersection(origin, direction, childDistance, childFace, childNormal, precision, trianglesTouched)) {
+                if (child.second.findRayIntersection(origin, direction, childDistance, childFace, childTriangle, precision, trianglesTouched)) {
                     if (childDistance < bestLocalDistance) {
                         bestLocalDistance = childDistance;
                         bestLocalFace = childFace;
-                        bestLocalNormal = childNormal;
+                        bestLocalTriangle = childTriangle;
                         intersects = true;
                     }
                 }
             }
         }
         // also check our local triangle set
-        if (findRayIntersectionInternal(origin, direction, childDistance, childFace, childNormal, precision, trianglesTouched, allowBackface)) {
+        if (findRayIntersectionInternal(origin, direction, childDistance, childFace, childTriangle, precision, trianglesTouched, allowBackface)) {
             if (childDistance < bestLocalDistance) {
                 bestLocalDistance = childDistance;
                 bestLocalFace = childFace;
-                bestLocalNormal = childNormal;
+                bestLocalTriangle = childTriangle;
                 intersects = true;
             }
         }
@@ -260,7 +263,7 @@ bool TriangleSet::TriangleOctreeCell::findRayIntersection(const glm::vec3& origi
     if (intersects) {
         distance = bestLocalDistance;
         face = bestLocalFace;
-        surfaceNormal = bestLocalNormal;
+        triangle = bestLocalTriangle;
     }
     return intersects;
 }

libraries/shared/src/TriangleSet.h

@@ -27,7 +27,8 @@ class TriangleSet {
         void clear();
 
         bool findRayIntersection(const glm::vec3& origin, const glm::vec3& direction,
-            float& distance, BoxFace& face, glm::vec3& surfaceNormal, bool precision, int& trianglesTouched, bool allowBackface = false);
+            float& distance, BoxFace& face, Triangle& triangle, bool precision, int& trianglesTouched,
+            bool allowBackface = false);
 
         const AABox& getBounds() const { return _bounds; }
 
@@ -38,7 +39,8 @@ class TriangleSet {
 
         // checks our internal list of triangles
         bool findRayIntersectionInternal(const glm::vec3& origin, const glm::vec3& direction,
-            float& distance, BoxFace& face, glm::vec3& surfaceNormal, bool precision, int& trianglesTouched, bool allowBackface = false);
+            float& distance, BoxFace& face, Triangle& triangle, bool precision, int& trianglesTouched,
+            bool allowBackface = false);
 
         std::vector<Triangle>& _allTriangles;
         std::map<AABox::OctreeChild, TriangleOctreeCell> _children;
@@ -60,7 +62,7 @@ public:
     void insert(const Triangle& t);
 
     bool findRayIntersection(const glm::vec3& origin, const glm::vec3& direction,
-        float& distance, BoxFace& face, glm::vec3& surfaceNormal, bool precision, bool allowBackface = false);
+        float& distance, BoxFace& face, Triangle& triangle, bool precision, bool allowBackface = false);
 
     void balanceOctree();
 
@@ -72,7 +74,7 @@ public:
     // intersection occurs, the distance and surface normal will be provided.
     // note: this might side-effect internal structures
     bool findRayIntersection(const glm::vec3& origin, const glm::vec3& direction,
-        float& distance, BoxFace& face, glm::vec3& surfaceNormal, bool precision, int& trianglesTouched);
+        float& distance, BoxFace& face, Triangle& triangle, bool precision, int& trianglesTouched);
 
     // Determine if a point is "inside" all the triangles of a convex hull. It is the responsibility of the caller to
     // determine that the triangle set is indeed a convex hull. If the triangles added to this set are not in fact a