diff --git a/interface/src/ui/overlays/ContextOverlayInterface.cpp b/interface/src/ui/overlays/ContextOverlayInterface.cpp
index 33d492bc7e..9182346352 100644
--- a/interface/src/ui/overlays/ContextOverlayInterface.cpp
+++ b/interface/src/ui/overlays/ContextOverlayInterface.cpp
@@ -52,7 +52,7 @@ void ContextOverlayInterface::createOrDestroyContextOverlay(const EntityItemID&
         _contextOverlay->setPosition(entityProperties.getPosition());
         _contextOverlay->setRotation(entityProperties.getRotation());
         _contextOverlay->setVisible(true);
-    } else {
+    } else if (_currentEntityWithContextOverlay == entityItemID) {
         destroyContextOverlay(entityItemID, event);
     }
 }