diff --git a/libraries/entities/src/EntityItem.cpp b/libraries/entities/src/EntityItem.cpp
index c8e7ce6c11..48370b02fd 100644
--- a/libraries/entities/src/EntityItem.cpp
+++ b/libraries/entities/src/EntityItem.cpp
@@ -1605,6 +1605,48 @@ void EntityItem::setParentID(const QUuid& value) {
         if (tree && !oldParentID.isNull()) {
             tree->removeFromChildrenOfAvatars(getThisPointer());
         }
+
+        uint32_t oldParentNoBootstrapping = 0;
+        uint32_t newParentNoBootstrapping = 0;
+        if (!value.isNull() && tree) {
+            EntityItemPointer entity = tree->findEntityByEntityItemID(value);
+            if (entity) {
+                newParentNoBootstrapping = entity->getDirtyFlags() & Simulation::NO_BOOTSTRAPPING;
+            }
+        }
+
+        if (!oldParentID.isNull() && tree) {
+            EntityItemPointer entity = tree->findEntityByEntityItemID(oldParentID);
+            if (entity) {
+                oldParentNoBootstrapping = entity->getDirtyFlags() & Simulation::NO_BOOTSTRAPPING;
+            }
+        }
+
+        if (!value.isNull() && (value == Physics::getSessionUUID() || value == AVATAR_SELF_ID)) {
+            newParentNoBootstrapping |= Simulation::NO_BOOTSTRAPPING;
+        }
+
+        if ((bool)(oldParentNoBootstrapping ^ newParentNoBootstrapping)) {
+            if ((bool)(newParentNoBootstrapping & Simulation::NO_BOOTSTRAPPING)) {
+                markDirtyFlags(Simulation::NO_BOOTSTRAPPING);
+                forEachDescendant([&](SpatiallyNestablePointer object) {
+                        if (object->getNestableType() == NestableType::Entity) {
+                            EntityItemPointer entity = std::static_pointer_cast<EntityItem>(object);
+                            entity->markDirtyFlags(Simulation::DIRTY_COLLISION_GROUP | Simulation::NO_BOOTSTRAPPING);
+                        }
+                });
+            } else {
+                clearDirtyFlags(Simulation::NO_BOOTSTRAPPING);
+                forEachDescendant([&](SpatiallyNestablePointer object) {
+                        if (object->getNestableType() == NestableType::Entity) {
+                            EntityItemPointer entity = std::static_pointer_cast<EntityItem>(object);
+                            entity->markDirtyFlags(Simulation::DIRTY_COLLISION_GROUP);
+                            entity->clearDirtyFlags(Simulation::NO_BOOTSTRAPPING);
+                        }
+                });
+            }
+        }
+
         SpatiallyNestable::setParentID(value);
         // children are forced to be kinematic
         // may need to not collide with own avatar
@@ -1834,39 +1876,8 @@ void EntityItem::computeCollisionGroupAndFinalMask(int16_t& group, int16_t& mask
             }
         }
 
-        if (userMask & USER_COLLISION_GROUP_MY_AVATAR) {
-            bool iAmHoldingThis = false;
-            // if this entity is a descendant of MyAvatar, don't collide with MyAvatar.  This avoids the
-            // "bootstrapping" problem where you can shoot yourself across the room by grabbing something
-            // and holding it against your own avatar.
-            if (isChildOfMyAvatar()) {
-                iAmHoldingThis = true;
-            }
-            // also, don't bootstrap our own avatar with a hold action
-            QList<EntityDynamicPointer> holdActions = getActionsOfType(DYNAMIC_TYPE_HOLD);
-            QList<EntityDynamicPointer>::const_iterator i = holdActions.begin();
-            while (i != holdActions.end()) {
-                EntityDynamicPointer action = *i;
-                if (action->isMine()) {
-                    iAmHoldingThis = true;
-                    break;
-                }
-                i++;
-            }
-            QList<EntityDynamicPointer> farGrabActions = getActionsOfType(DYNAMIC_TYPE_FAR_GRAB);
-            i = farGrabActions.begin();
-            while (i != farGrabActions.end()) {
-                EntityDynamicPointer action = *i;
-                if (action->isMine()) {
-                    iAmHoldingThis = true;
-                    break;
-                }
-                i++;
-            }
-
-            if (iAmHoldingThis) {
-                userMask &= ~USER_COLLISION_GROUP_MY_AVATAR;
-            }
+        if ((bool)(_dirtyFlags & Simulation::NO_BOOTSTRAPPING)) {
+            userMask &= ~USER_COLLISION_GROUP_MY_AVATAR;
         }
         mask = Physics::getDefaultCollisionMask(group) & (int16_t)(userMask);
     }
@@ -1961,7 +1972,20 @@ bool EntityItem::addActionInternal(EntitySimulationPointer simulation, EntityDyn
     if (success) {
         _allActionsDataCache = newDataCache;
         _dirtyFlags |= Simulation::DIRTY_PHYSICS_ACTIVATION;
-        _dirtyFlags |= Simulation::DIRTY_COLLISION_GROUP; // may need to not collide with own avatar
+
+        auto actionType = action->getType();
+        if (actionType == DYNAMIC_TYPE_HOLD || actionType == DYNAMIC_TYPE_FAR_GRAB) {
+            if (!(bool)(_dirtyFlags & Simulation::NO_BOOTSTRAPPING)) {
+                _dirtyFlags |= Simulation::NO_BOOTSTRAPPING;
+                _dirtyFlags |= Simulation::DIRTY_COLLISION_GROUP; // may need to not collide with own avatar
+                forEachDescendant([&](SpatiallyNestablePointer child) {
+                    if (child->getNestableType() == NestableType::Entity) {
+                        EntityItemPointer entity = std::static_pointer_cast<EntityItem>(child);
+                        entity->markDirtyFlags(Simulation::NO_BOOTSTRAPPING | Simulation::DIRTY_COLLISION_GROUP);
+                    }
+                });
+            }
+        }
     } else {
         qCDebug(entities) << "EntityItem::addActionInternal -- serializeActions failed";
     }
@@ -2002,6 +2026,29 @@ bool EntityItem::removeAction(EntitySimulationPointer simulation, const QUuid& a
     return success;
 }
 
+bool EntityItem::stillHasGrabActions() const {
+    QList<EntityDynamicPointer> holdActions = getActionsOfType(DYNAMIC_TYPE_HOLD);
+    QList<EntityDynamicPointer>::const_iterator i = holdActions.begin();
+    while (i != holdActions.end()) {
+        EntityDynamicPointer action = *i;
+        if (action->isMine()) {
+            return true;
+        }
+        i++;
+    }
+    QList<EntityDynamicPointer> farGrabActions = getActionsOfType(DYNAMIC_TYPE_FAR_GRAB);
+    i = farGrabActions.begin();
+    while (i != farGrabActions.end()) {
+        EntityDynamicPointer action = *i;
+        if (action->isMine()) {
+            return true;
+        }
+        i++;
+    }
+
+    return false;
+}
+
 bool EntityItem::removeActionInternal(const QUuid& actionID, EntitySimulationPointer simulation) {
     _previouslyDeletedActions.insert(actionID, usecTimestampNow());
     if (_objectActions.contains(actionID)) {
@@ -2015,7 +2062,6 @@ bool EntityItem::removeActionInternal(const QUuid& actionID, EntitySimulationPoi
 
         action->setOwnerEntity(nullptr);
         action->setIsMine(false);
-        _objectActions.remove(actionID);
 
         if (simulation) {
             action->removeFromSimulation(simulation);
@@ -2024,7 +2070,23 @@ bool EntityItem::removeActionInternal(const QUuid& actionID, EntitySimulationPoi
         bool success = true;
         serializeActions(success, _allActionsDataCache);
         _dirtyFlags |= Simulation::DIRTY_PHYSICS_ACTIVATION;
-        _dirtyFlags |= Simulation::DIRTY_COLLISION_GROUP; // may need to not collide with own avatar
+        auto removedActionType = action->getType();
+        if ((removedActionType == DYNAMIC_TYPE_HOLD || removedActionType == DYNAMIC_TYPE_FAR_GRAB) && !stillHasGrabActions()) {
+            _dirtyFlags &= ~Simulation::NO_BOOTSTRAPPING;
+            _dirtyFlags |= Simulation::DIRTY_COLLISION_GROUP; // may need to not collide with own avatar
+            forEachDescendant([&](SpatiallyNestablePointer child) {
+                if (child->getNestableType() == NestableType::Entity) {
+                    EntityItemPointer entity = std::static_pointer_cast<EntityItem>(child);
+                    entity->markDirtyFlags(Simulation::DIRTY_COLLISION_GROUP);
+                    entity->clearDirtyFlags(Simulation::NO_BOOTSTRAPPING);
+                }
+            });
+        } else {
+            // NO-OP: we assume NO_BOOTSTRAPPING bits and collision group are correct
+            // because they should have been set correctly when the action was added
+            // and/or when children were linked
+        }
+        _objectActions.remove(actionID);
         setDynamicDataNeedsTransmit(true);
         return success;
     }
diff --git a/libraries/entities/src/EntityItem.h b/libraries/entities/src/EntityItem.h
index 882b8e6812..4c7f37bd6a 100644
--- a/libraries/entities/src/EntityItem.h
+++ b/libraries/entities/src/EntityItem.h
@@ -470,6 +470,7 @@ protected:
     void setSimulated(bool simulated) { _simulated = simulated; }
 
     const QByteArray getDynamicDataInternal() const;
+    bool stillHasGrabActions() const;
     void setDynamicDataInternal(QByteArray dynamicData);
 
     virtual void dimensionsChanged() override;
diff --git a/libraries/entities/src/SimulationFlags.h b/libraries/entities/src/SimulationFlags.h
index e2b2224b4a..aaa92000e7 100644
--- a/libraries/entities/src/SimulationFlags.h
+++ b/libraries/entities/src/SimulationFlags.h
@@ -27,6 +27,7 @@ namespace Simulation {
     const uint32_t DIRTY_PHYSICS_ACTIVATION = 0x0800; // should activate object in physics engine
     const uint32_t DIRTY_SIMULATOR_ID = 0x1000; // the simulatorID has changed
     const uint32_t DIRTY_SIMULATION_OWNERSHIP_PRIORITY = 0x2000; // our own bid priority has changed
+    const uint32_t NO_BOOTSTRAPPING = 0x4000;
 
     const uint32_t DIRTY_TRANSFORM = DIRTY_POSITION | DIRTY_ROTATION;
     const uint32_t DIRTY_VELOCITIES = DIRTY_LINEAR_VELOCITY | DIRTY_ANGULAR_VELOCITY;
diff --git a/libraries/physics/src/EntityMotionState.cpp b/libraries/physics/src/EntityMotionState.cpp
index 8ebce9f811..7e8b431ceb 100644
--- a/libraries/physics/src/EntityMotionState.cpp
+++ b/libraries/physics/src/EntityMotionState.cpp
@@ -700,7 +700,7 @@ uint32_t EntityMotionState::getIncomingDirtyFlags() {
 void EntityMotionState::clearIncomingDirtyFlags() {
     assert(entityTreeIsLocked());
     if (_body && _entity) {
-        _entity->clearDirtyFlags();
+        _entity->clearDirtyFlags(DIRTY_PHYSICS_FLAGS);
     }
 }