Bug fix for deadlock between EntitySimulation and EntityItem locks.

This is a classic deadlock between the main thread and the OctreeProcessor network thread.
On the main thread, the EntitySimulation lock is taken before the EntityItem read lock is taken.
On the network thread, the EntityItem write lock is taken before the EntitySimulation lock is taken.

To work around this issue the network thread no longer takes the EntitySimulation lock when calling
PhysicalEntitySimulation::addDynamic().  Instead, a fine grained lock around the EntitySimulation's dynamic lists
is used instead.

libraries/entities/src/EntitySimulation.cpp

@@ -280,24 +280,24 @@ void EntitySimulation::moveSimpleKinematics(const quint64& now) {
 }
 
 void EntitySimulation::addDynamic(EntityDynamicPointer dynamic) {
-    QMutexLocker lock(&_mutex);
+    QMutexLocker lock(&_dynamicsMutex);
     _dynamicsToAdd += dynamic;
 }
 
 void EntitySimulation::removeDynamic(const QUuid dynamicID) {
-    QMutexLocker lock(&_mutex);
+    QMutexLocker lock(&_dynamicsMutex);
     _dynamicsToRemove += dynamicID;
 }
 
 void EntitySimulation::removeDynamics(QList<QUuid> dynamicIDsToRemove) {
-    QMutexLocker lock(&_mutex);
+    QMutexLocker lock(&_dynamicsMutex);
     foreach(QUuid uuid, dynamicIDsToRemove) {
         _dynamicsToRemove.insert(uuid);
     }
 }
 
 void EntitySimulation::applyDynamicChanges() {
-    QMutexLocker lock(&_mutex);
+    QMutexLocker lock(&_dynamicsMutex);
     _dynamicsToAdd.clear();
     _dynamicsToRemove.clear();
 }

libraries/entities/src/EntitySimulation.h

@@ -105,6 +105,7 @@ protected:
     SetOfEntities _simpleKinematicEntities; // entities undergoing non-colliding kinematic motion
     QList<EntityDynamicPointer> _dynamicsToAdd;
     QSet<QUuid> _dynamicsToRemove;
+    QMutex _dynamicsMutex { QMutex::Recursive };
 
 protected:
     SetOfEntities _entitiesToDelete; // entities simulation decided needed to be deleted (EntityTree will actually delete)

libraries/physics/src/PhysicalEntitySimulation.cpp

@@ -348,8 +348,7 @@ void PhysicalEntitySimulation::addDynamic(EntityDynamicPointer dynamic) {
 void PhysicalEntitySimulation::applyDynamicChanges() {
     QList<EntityDynamicPointer> dynamicsFailedToAdd;
     if (_physicsEngine) {
-        // FIXME put fine grain locking into _physicsEngine
-        QMutexLocker lock(&_mutex);
+        QMutexLocker lock(&_dynamicsMutex);
         foreach(QUuid dynamicToRemove, _dynamicsToRemove) {
             _physicsEngine->removeDynamic(dynamicToRemove);
         }
@@ -360,9 +359,10 @@ void PhysicalEntitySimulation::applyDynamicChanges() {
                 }
             }
         }
+        // applyDynamicChanges will clear _dynamicsToRemove and _dynamicsToAdd
+        EntitySimulation::applyDynamicChanges();
     }
-    // applyDynamicChanges will clear _dynamicsToRemove and _dynamicsToAdd
-    EntitySimulation::applyDynamicChanges();
+
     // put back the ones that couldn't yet be added
     foreach (EntityDynamicPointer dynamicFailedToAdd, dynamicsFailedToAdd) {
         addDynamic(dynamicFailedToAdd);