diff --git a/libraries/networking/src/PacketReceiver.cpp b/libraries/networking/src/PacketReceiver.cpp
index 973cc82e32..f59be35dc5 100644
--- a/libraries/networking/src/PacketReceiver.cpp
+++ b/libraries/networking/src/PacketReceiver.cpp
@@ -359,6 +359,11 @@ void PacketReceiver::processDatagrams() {
                             << " (" << qPrintable(nameForPacketType(packet->getType())) << ")"
                             << " has been destroyed. Removing from listener map.";
                         it = _packetListenerMap.erase(it);
+                        
+                        // if it exists, remove the listener from _directlyConnectedObjects
+                        _directConnectSetMutex.lock();
+                        _directlyConnectedObjects.remove(listener.first);
+                        _directConnectSetMutex.unlock();
                     }
                     
                 } else {