limit number of include items for mallets

interface/src/raypick/RayPick.h

@@ -85,9 +85,10 @@ public:
     static glm::vec3 intersectRayWithEntityXYPlane(const QUuid& entityID, const glm::vec3& origin, const glm::vec3& direction);
     static glm::vec2 projectOntoEntityXYPlane(const QUuid& entityID, const glm::vec3& worldPos, bool unNormalized = true);
 
+    static glm::vec2 projectOntoXYPlane(const glm::vec3& worldPos, const glm::vec3& position, const glm::quat& rotation, const glm::vec3& dimensions, const glm::vec3& registrationPoint, bool unNormalized);
+
 private:
     static glm::vec3 intersectRayWithXYPlane(const glm::vec3& origin, const glm::vec3& direction, const glm::vec3& point, const glm::quat& rotation, const glm::vec3& registration);
-    static glm::vec2 projectOntoXYPlane(const glm::vec3& worldPos, const glm::vec3& position, const glm::quat& rotation, const glm::vec3& dimensions, const glm::vec3& registrationPoint, bool unNormalized);
 };
 
 #endif // hifi_RayPick_h

interface/src/raypick/StylusPick.cpp

@@ -137,13 +137,14 @@ PickResultPointer StylusPick::getDefaultResult(const QVariantMap& pickVariant) c
 }
 
 PickResultPointer StylusPick::getEntityIntersection(const StylusTip& pick) {
+    auto entityTree = qApp->getEntities()->getTree();
     StylusPickResult nearestTarget(pick.toVariantMap());
     for (const auto& target : getIncludeItems()) {
         if (target.isNull()) {
             continue;
         }
 
-        auto entity = qApp->getEntities()->getTree()->findEntityByEntityItemID(target);
+        auto entity = entityTree->findEntityByEntityItemID(target);
         if (!entity) {
             continue;
         }
@@ -158,8 +159,11 @@ PickResultPointer StylusPick::getEntityIntersection(const StylusTip& pick) {
         glm::vec3 normal = entityRotation * Vectors::UNIT_Z;
         float distance = glm::dot(pick.position - entityPosition, normal);
         if (distance < nearestTarget.distance) {
+            const auto entityDimensions = entity->getScaledDimensions();
+            const auto entityRegistrationPoint = entity->getRegistrationPoint();
             glm::vec3 intersection = pick.position - (normal * distance);
-            glm::vec2 pos2D = RayPick::projectOntoEntityXYPlane(target, intersection, false);
+            glm::vec2 pos2D = RayPick::projectOntoXYPlane(intersection, entityPosition, entityRotation,
+                                                          entityDimensions, entityRegistrationPoint, false);
             if (pos2D == glm::clamp(pos2D, glm::vec2(0), glm::vec2(1))) {
                 IntersectionType type = IntersectionType::ENTITY;
                 if (getFilter().doesPickLocalEntities()) {

interface/src/ui/Keyboard.cpp

@@ -309,12 +309,22 @@ void Keyboard::setRaised(bool raised) {
             _layerIndex = 0;
             _capsEnabled = false;
             _typedCharacters.clear();
+            addIncludeItemsToMallets();
         });
 
         updateTextDisplay();
     }
 }
 
+void Keyboard::addIncludeItemsToMallets() {
+    if (_layerIndex >= 0 && _layerIndex < (int)_keyboardLayers.size()) {
+        QVector<QUuid> includeItems = _keyboardLayers[_layerIndex].keys().toVector();
+        auto pointerManager = DependencyManager::get<PointerManager>();
+        pointerManager->setIncludeItems(_leftHandStylus, includeItems);
+        pointerManager->setIncludeItems(_rightHandStylus, includeItems);
+    }
+}
+
 void Keyboard::updateTextDisplay() {
     auto myAvatar = DependencyManager::get<AvatarManager>()->getMyAvatar();
     auto entityScriptingInterface = DependencyManager::get<EntityScriptingInterface>();
@@ -463,6 +473,8 @@ void Keyboard::switchToLayer(int layerIndex) {
          properties.setRotation(currentOrientation);
          entityScriptingInterface->editEntity(_anchor.entityID, properties);
 
+         addIncludeItemsToMallets();
+
          startLayerSwitchTimer();
     }
 }
@@ -718,8 +730,6 @@ void Keyboard::loadKeyboardFile(const QString& keyboardFile) {
         clearKeyboardKeys();
         auto requestData = request->getData();
 
-        QVector<QUuid> includeItems;
-
         QJsonParseError parseError;
         QJsonDocument jsonDoc = QJsonDocument::fromJson(requestData, &parseError);
 
@@ -840,7 +850,6 @@ void Keyboard::loadKeyboardFile(const QString& keyboardFile) {
                 key.setKeyString(keyString);
                 key.saveDimensionsAndLocalPosition();
 
-                includeItems.append(key.getID());
                 _itemsToIgnore.insert(key.getID());
                 keyboardLayerKeys.insert(id, key);
             }
@@ -886,9 +895,7 @@ void Keyboard::loadKeyboardFile(const QString& keyboardFile) {
             _itemsToIgnore.insert(_anchor.entityID);
         });
         _layerIndex = 0;
-        auto pointerManager = DependencyManager::get<PointerManager>();
+        addIncludeItemsToMallets();
-        pointerManager->setIncludeItems(_leftHandStylus, includeItems);
-        pointerManager->setIncludeItems(_rightHandStylus, includeItems);
     });
 
     request->send();

interface/src/ui/Keyboard.h

@@ -157,6 +157,7 @@ private:
     bool shouldProcessEntityAndPointerEvent(const PointerEvent& event) const;
     bool shouldProcessPointerEvent(const PointerEvent& event) const;
     bool shouldProcessEntity() const;
+    void addIncludeItemsToMallets();
 
     void startLayerSwitchTimer();
     bool isLayerSwitchTimerFinished() const;