Merge pull request #2305 from ey6es/master

Avatar performance improvements, toggle for Visage (and code to allow it to play nice with Faceshift).  Closes #2300.

interface/interface_en.ts

@@ -14,12 +14,12 @@
         <translation type="unfinished"></translation>
     </message>
     <message>
-        <location filename="src/Application.cpp" line="3569"/>
+        <location filename="src/Application.cpp" line="3573"/>
         <source>Open Script</source>
         <translation type="unfinished"></translation>
     </message>
     <message>
-        <location filename="src/Application.cpp" line="3570"/>
+        <location filename="src/Application.cpp" line="3574"/>
         <source>JavaScript Files (*.js)</source>
         <translation type="unfinished"></translation>
     </message>
@@ -113,18 +113,18 @@
 <context>
     <name>Menu</name>
     <message>
-        <location filename="src/Menu.cpp" line="424"/>
+        <location filename="src/Menu.cpp" line="429"/>
         <source>Open .ini config file</source>
         <translation type="unfinished"></translation>
     </message>
     <message>
-        <location filename="src/Menu.cpp" line="426"/>
+        <location filename="src/Menu.cpp" line="431"/>
-        <location filename="src/Menu.cpp" line="438"/>
+        <location filename="src/Menu.cpp" line="443"/>
         <source>Text files (*.ini)</source>
         <translation type="unfinished"></translation>
     </message>
     <message>
-        <location filename="src/Menu.cpp" line="436"/>
+        <location filename="src/Menu.cpp" line="441"/>
         <source>Save .ini config file</source>
         <translation type="unfinished"></translation>
     </message>

interface/src/Application.cpp

@@ -1534,6 +1534,10 @@ void Application::init() {
     }
     qDebug("Loaded settings");
     
+    // initialize Visage and Faceshift after loading the menu settings
+    _faceshift.init();
+    _visage.init();
+    
     // fire off an immediate domain-server check in now that settings are loaded
     NodeList::getInstance()->sendDomainServerCheckIn();
 

interface/src/Menu.cpp

@@ -272,11 +272,16 @@ Menu::Menu() :
 
     addCheckableActionToQMenuAndActionHash(avatarOptionsMenu, MenuOption::LookAtVectors, 0, false);
     addCheckableActionToQMenuAndActionHash(avatarOptionsMenu,
-                                           MenuOption::FaceshiftTCP,
+                                           MenuOption::Faceshift,
                                            0,
-                                           false,
+                                           true,
                                            appInstance->getFaceshift(),
                                            SLOT(setTCPEnabled(bool)));
+#ifdef HAVE_VISAGE
+    addCheckableActionToQMenuAndActionHash(avatarOptionsMenu, MenuOption::Visage, 0, true,
+        appInstance->getVisage(), SLOT(updateEnabled()));
+#endif
+
     addCheckableActionToQMenuAndActionHash(avatarOptionsMenu, MenuOption::ChatCircling, 0, false);
 
     QMenu* handOptionsMenu = developerMenu->addMenu("Hand Options");

interface/src/Menu.h

@@ -239,7 +239,7 @@ namespace MenuOption {
     const QString DontFadeOnVoxelServerChanges = "Don't Fade In/Out on Voxel Server Changes";
     const QString HeadMouse = "Head Mouse";
     const QString HandsCollideWithSelf = "Collide With Self";
-    const QString FaceshiftTCP = "Faceshift (TCP)";
+    const QString Faceshift = "Faceshift";
     const QString FirstPerson = "First Person";
     const QString FrameTimer = "Show Timer";
     const QString FrustumRenderMode = "Render Mode";
@@ -293,6 +293,7 @@ namespace MenuOption {
     const QString TransmitterDrive = "Transmitter Drive";
     const QString UploaderAvatarHead = "Upload Avatar Head";
     const QString UploaderAvatarSkeleton = "Upload Avatar Skeleton";
+    const QString Visage = "Visage";
     const QString Quit =  "Quit";
     const QString Voxels = "Voxels";
     const QString VoxelMode = "Cycle Voxel Mode";

interface/src/avatar/Avatar.cpp

@@ -56,7 +56,8 @@ Avatar::Avatar() :
     _owningAvatarMixer(),
     _collisionFlags(0),
     _initialized(false),
-    _shouldRenderBillboard(true)
+    _shouldRenderBillboard(true),
+    _modelsDirty(true)
 {
     // we may have been created in the network thread, but we live in the main thread
     moveToThread(Application::getInstance()->thread());
@@ -109,6 +110,11 @@ void Avatar::simulate(float deltaTime) {
         _shouldRenderBillboard = true;
     }
 
+    // simple frustum check
+    float boundingRadius = getBillboardSize();
+    bool inViewFrustum = Application::getInstance()->getViewFrustum()->sphereInFrustum(_position, boundingRadius) !=
+        ViewFrustum::OUTSIDE;
+
     getHand()->simulate(deltaTime, false);
     _skeletonModel.setLODDistance(getLODDistance());
     
@@ -118,8 +124,9 @@ void Avatar::simulate(float deltaTime) {
         _skeletonModel.setJointState(i, data.valid, data.rotation);
     }
     glm::vec3 headPosition = _position;
-    if (!_shouldRenderBillboard) {
+    if (!_shouldRenderBillboard && inViewFrustum) {
-        _skeletonModel.simulate(deltaTime);
+        _skeletonModel.simulate(deltaTime, _modelsDirty);
+        _modelsDirty = false;
         _skeletonModel.getHeadPosition(headPosition);
     }
     Head* head = getHead();
@@ -183,6 +190,12 @@ static TextRenderer* textRenderer(TextRendererType type) {
 }
 
 void Avatar::render(bool forShadowMap) {
+    // simple frustum check
+    float boundingRadius = getBillboardSize();
+    if (Application::getInstance()->getViewFrustum()->sphereInFrustum(_position, boundingRadius) == ViewFrustum::OUTSIDE) {
+        return;
+    }
+
     glm::vec3 toTarget = _position - Application::getInstance()->getAvatar()->getPosition();
     float lengthToTarget = glm::length(toTarget);
    
@@ -336,7 +349,7 @@ void Avatar::renderBillboard() {
     glRotatef(glm::degrees(glm::angle(rotation)), axis.x, axis.y, axis.z);
     
     // compute the size from the billboard camera parameters and scale
-    float size = _scale * BILLBOARD_DISTANCE * tanf(glm::radians(BILLBOARD_FIELD_OF_VIEW / 2.0f));
+    float size = getBillboardSize();
     glScalef(size, size, size);
     
     glColor3f(1.0f, 1.0f, 1.0f);
@@ -361,6 +374,10 @@ void Avatar::renderBillboard() {
     glBindTexture(GL_TEXTURE_2D, 0);
 }
 
+float Avatar::getBillboardSize() const {
+    return _scale * BILLBOARD_DISTANCE * tanf(glm::radians(BILLBOARD_FIELD_OF_VIEW / 2.0f));
+}
+
 void Avatar::renderDisplayName() {
 
     if (_displayName.isEmpty() || _displayNameAlpha == 0.0f) {
@@ -618,6 +635,9 @@ int Avatar::parseData(const QByteArray& packet) {
     const float MOVE_DISTANCE_THRESHOLD = 0.001f;
     _moving = glm::distance(oldPosition, _position) > MOVE_DISTANCE_THRESHOLD;
     
+    // note that we need to update our models
+    _modelsDirty = true;
+    
     return bytesRead;
 }
 

interface/src/avatar/Avatar.h

@@ -187,9 +187,12 @@ private:
     bool _initialized;
     QScopedPointer<Texture> _billboardTexture;
     bool _shouldRenderBillboard;
+    bool _modelsDirty;
 
     void renderBody();
     void renderBillboard();
+    
+    float getBillboardSize() const;
 };
 
 #endif

interface/src/avatar/FaceModel.cpp

@@ -18,9 +18,9 @@ FaceModel::FaceModel(Head* owningHead) :
 {
 }
 
-void FaceModel::simulate(float deltaTime, bool delayLoad) {
+void FaceModel::simulate(float deltaTime) {
+    QVector<JointState> newJointStates = updateGeometry();
     if (!isActive()) {
-        Model::simulate(deltaTime, delayLoad);
         return;
     }
     Avatar* owningAvatar = static_cast<Avatar*>(_owningHead->_owningAvatar);
@@ -36,12 +36,13 @@ void FaceModel::simulate(float deltaTime, bool delayLoad) {
     setRotation(neckRotation);
     const float MODEL_SCALE = 0.0006f;
     setScale(glm::vec3(1.0f, 1.0f, 1.0f) * _owningHead->getScale() * MODEL_SCALE);
+    
     setOffset(-_geometry->getFBXGeometry().neckPivot);
     
     setPupilDilation(_owningHead->getPupilDilation());
     setBlendshapeCoefficients(_owningHead->getBlendshapeCoefficients());
     
-    Model::simulate(deltaTime, delayLoad);
+    Model::simulate(deltaTime, true, newJointStates);
 }
 
 bool FaceModel::render(float alpha) {

interface/src/avatar/FaceModel.h

@@ -21,7 +21,7 @@ public:
 
     FaceModel(Head* owningHead);
 
-    void simulate(float deltaTime, bool delayLoad = false);
+    void simulate(float deltaTime);
     bool render(float alpha);
     
 protected:

interface/src/avatar/SkeletonModel.cpp

@@ -18,13 +18,13 @@ SkeletonModel::SkeletonModel(Avatar* owningAvatar) :
     _owningAvatar(owningAvatar) {
 }
 
-void SkeletonModel::simulate(float deltaTime, bool delayLoad) {
+void SkeletonModel::simulate(float deltaTime, bool fullUpdate) {
     setTranslation(_owningAvatar->getPosition());
     setRotation(_owningAvatar->getOrientation() * glm::angleAxis(PI, glm::vec3(0.0f, 1.0f, 0.0f)));
     const float MODEL_SCALE = 0.0006f;
     setScale(glm::vec3(1.0f, 1.0f, 1.0f) * _owningAvatar->getScale() * MODEL_SCALE);
     
-    Model::simulate(deltaTime, delayLoad);
+    Model::simulate(deltaTime, fullUpdate);
     
     if (!(isActive() && _owningAvatar->isMyAvatar())) {
         return; // only simulate for own avatar

interface/src/avatar/SkeletonModel.h

@@ -22,7 +22,7 @@ public:
 
     SkeletonModel(Avatar* owningAvatar);
     
-    void simulate(float deltaTime, bool delayLoad = false);
+    void simulate(float deltaTime, bool fullUpdate = true);
     bool render(float alpha);
 
     /// \param jointIndex index of hand joint

interface/src/devices/Faceshift.cpp

@@ -21,7 +21,7 @@ using namespace std;
 const quint16 FACESHIFT_PORT = 33433;
 
 Faceshift::Faceshift() :
-    _tcpEnabled(false),
+    _tcpEnabled(true),
     _tcpRetryCount(0),
     _lastTrackingStateReceived(0),
     _eyeGazeLeftPitch(0.0f),
@@ -49,12 +49,22 @@ Faceshift::Faceshift() :
     connect(&_tcpSocket, SIGNAL(connected()), SLOT(noteConnected()));
     connect(&_tcpSocket, SIGNAL(error(QAbstractSocket::SocketError)), SLOT(noteError(QAbstractSocket::SocketError)));
     connect(&_tcpSocket, SIGNAL(readyRead()), SLOT(readFromSocket()));
+    connect(&_tcpSocket, SIGNAL(stateChanged(QAbstractSocket::SocketState)), SIGNAL(connectionStateChanged()));
 
     connect(&_udpSocket, SIGNAL(readyRead()), SLOT(readPendingDatagrams()));
 
     _udpSocket.bind(FACESHIFT_PORT);
 }
 
+void Faceshift::init() {
+    setTCPEnabled(Menu::getInstance()->isOptionChecked(MenuOption::Faceshift));
+}
+
+bool Faceshift::isConnectedOrConnecting() const {
+    return _tcpSocket.state() == QAbstractSocket::ConnectedState ||
+        (_tcpRetryCount == 0 && _tcpSocket.state() != QAbstractSocket::UnconnectedState);
+}
+
 bool Faceshift::isActive() const {
     const quint64 ACTIVE_TIMEOUT_USECS = 1000000;
     return (usecTimestampNow() - _lastTrackingStateReceived) < ACTIVE_TIMEOUT_USECS;

interface/src/devices/Faceshift.h

@@ -27,6 +27,10 @@ public:
 
     Faceshift();
 
+    void init();
+
+    bool isConnectedOrConnecting() const; 
+
     bool isActive() const;
 
     const glm::quat& getHeadRotation() const { return _headRotation; }
@@ -66,6 +70,10 @@ public:
     void updateFakeCoefficients(float leftBlink, float rightBlink, float browUp,
         float jawOpen, std::vector<float>& coefficients) const;
     
+signals:
+
+    void connectionStateChanged();
+
 public slots:
     
     void setTCPEnabled(bool enabled);

interface/src/devices/Visage.cpp

@@ -32,6 +32,7 @@ using namespace VisageSDK;
 const glm::vec3 DEFAULT_HEAD_ORIGIN(0.0f, 0.0f, 0.7f);
 
 Visage::Visage() :
+    _enabled(false),
     _active(false),
     _headOrigin(DEFAULT_HEAD_ORIGIN),
     _estimatedEyePitch(0.0f),
@@ -41,23 +42,15 @@ Visage::Visage() :
     QByteArray licensePath = Application::resourcesPath().toLatin1() + "visage/license.vlc";
     initializeLicenseManager(licensePath.data());
     _tracker = new VisageTracker2(Application::resourcesPath().toLatin1() + "visage/tracker.cfg");
-    if (_tracker->trackFromCam()) {
+    _data = new FaceData();
-        _data = new FaceData();   
-         
-    } else {
-        delete _tracker;
-        _tracker = NULL;
-    }
 #endif
 }
 
 Visage::~Visage() {
 #ifdef HAVE_VISAGE
-    if (_tracker) {
+    _tracker->stop();
-        _tracker->stop();
+    delete _tracker;
-        delete _tracker;
+    delete _data;
-        delete _data;
-    }
 #endif
 }
 
@@ -117,9 +110,14 @@ static const QMultiHash<QByteArray, QPair<int, float> >& getActionUnitNameMap()
 
 const float TRANSLATION_SCALE = 20.0f;
 
+void Visage::init() {
+    connect(Application::getInstance()->getFaceshift(), SIGNAL(connectionStateChanged()), SLOT(updateEnabled()));
+    updateEnabled();
+}
+
 void Visage::update() {
 #ifdef HAVE_VISAGE
-    _active = (_tracker && _tracker->getTrackingData(_data) == TRACK_STAT_OK);
+    _active = (_tracker->getTrackingData(_data) == TRACK_STAT_OK);
     if (!_active) {
         return;
     }
@@ -160,3 +158,22 @@ void Visage::update() {
 void Visage::reset() {
     _headOrigin += _headTranslation / TRANSLATION_SCALE;
 }
+
+void Visage::updateEnabled() {
+    setEnabled(Menu::getInstance()->isOptionChecked(MenuOption::Visage) &&
+        !(Menu::getInstance()->isOptionChecked(MenuOption::Faceshift) &&
+            Application::getInstance()->getFaceshift()->isConnectedOrConnecting()));
+}
+
+void Visage::setEnabled(bool enabled) {
+#ifdef HAVE_VISAGE
+    if (_enabled == enabled) {
+        return;
+    }
+    if ((_enabled = enabled)) {
+        _tracker->trackFromCam();
+    } else {
+        _tracker->stop();
+    }
+#endif
+}

interface/src/devices/Visage.h

@@ -24,11 +24,15 @@ namespace VisageSDK {
 }
 
 /// Handles input from the Visage webcam feature tracking software.
-class Visage {
+class Visage : public QObject {
+    Q_OBJECT
+    
 public:
     
     Visage();
-    ~Visage();
+    virtual ~Visage();
+    
+    void init();
     
     bool isActive() const { return _active; }
     
@@ -42,6 +46,10 @@ public:
     
     void update();
     void reset();
+
+public slots:
+
+    void updateEnabled();
     
 private:
 
@@ -51,6 +59,9 @@ private:
     QMultiHash<int, QPair<int, float> > _actionUnitIndexMap; 
 #endif
     
+    void setEnabled(bool enabled);
+    
+    bool _enabled;
     bool _active;
     glm::quat _headRotation;
     glm::vec3 _headTranslation;

interface/src/renderer/Model.cpp

@@ -156,142 +156,9 @@ void Model::updateShapePositions() {
     }
 }
 
-void Model::simulate(float deltaTime, bool delayLoad) {
+void Model::simulate(float deltaTime, bool fullUpdate) {
-    // update our LOD
+    // update our LOD, then simulate
-    QVector<JointState> newJointStates = updateGeometry(delayLoad);
+    simulate(deltaTime, fullUpdate, updateGeometry());
-    if (!isActive()) {
-        return;
-    }
-    
-    // set up world vertices on first simulate after load
-    const FBXGeometry& geometry = _geometry->getFBXGeometry();
-    if (_jointStates.isEmpty()) {
-        _jointStates = newJointStates.isEmpty() ? createJointStates(geometry) : newJointStates;
-        foreach (const FBXMesh& mesh, geometry.meshes) {
-            MeshState state;
-            state.clusterMatrices.resize(mesh.clusters.size());
-            if (mesh.springiness > 0.0f) {
-                state.worldSpaceVertices.resize(mesh.vertices.size());
-                state.vertexVelocities.resize(mesh.vertices.size());
-                state.worldSpaceNormals.resize(mesh.vertices.size());
-            }
-            _meshStates.append(state);    
-        }
-        foreach (const FBXAttachment& attachment, geometry.attachments) {
-            Model* model = new Model(this);
-            model->init();
-            model->setURL(attachment.url);
-            _attachments.append(model);
-        }
-        _resetStates = true;
-        createCollisionShapes();
-    }
-    
-    // update the world space transforms for all joints
-    for (int i = 0; i < _jointStates.size(); i++) {
-        updateJointState(i);
-    }
-    
-    // update the attachment transforms and simulate them
-    for (int i = 0; i < _attachments.size(); i++) {
-        const FBXAttachment& attachment = geometry.attachments.at(i);
-        Model* model = _attachments.at(i);
-        
-        glm::vec3 jointTranslation = _translation;
-        glm::quat jointRotation = _rotation;
-        getJointPosition(attachment.jointIndex, jointTranslation);
-        getJointRotation(attachment.jointIndex, jointRotation);
-        
-        model->setTranslation(jointTranslation + jointRotation * attachment.translation * _scale);
-        model->setRotation(jointRotation * attachment.rotation);
-        model->setScale(_scale * attachment.scale);
-        
-        model->simulate(deltaTime);
-    }
-    
-    for (int i = 0; i < _meshStates.size(); i++) {
-        MeshState& state = _meshStates[i];
-        const FBXMesh& mesh = geometry.meshes.at(i);
-        for (int j = 0; j < mesh.clusters.size(); j++) {
-            const FBXCluster& cluster = mesh.clusters.at(j);
-            state.clusterMatrices[j] = _jointStates[cluster.jointIndex].transform * cluster.inverseBindMatrix;
-        }
-        int vertexCount = state.worldSpaceVertices.size();
-        if (vertexCount == 0) {
-            continue;
-        }
-        glm::vec3* destVertices = state.worldSpaceVertices.data();
-        glm::vec3* destVelocities = state.vertexVelocities.data();
-        glm::vec3* destNormals = state.worldSpaceNormals.data();
-        
-        const glm::vec3* sourceVertices = mesh.vertices.constData();
-        if (!mesh.blendshapes.isEmpty()) {
-            _blendedVertices.resize(max(_blendedVertices.size(), vertexCount));
-            memcpy(_blendedVertices.data(), mesh.vertices.constData(), vertexCount * sizeof(glm::vec3));
-            
-            // blend in each coefficient
-            for (unsigned int j = 0; j < _blendshapeCoefficients.size(); j++) {
-                float coefficient = _blendshapeCoefficients[j];
-                if (coefficient == 0.0f || j >= (unsigned int)mesh.blendshapes.size() || mesh.blendshapes[j].vertices.isEmpty()) {
-                    continue;
-                }
-                const glm::vec3* vertex = mesh.blendshapes[j].vertices.constData();
-                for (const int* index = mesh.blendshapes[j].indices.constData(),
-                        *end = index + mesh.blendshapes[j].indices.size(); index != end; index++, vertex++) {
-                    _blendedVertices[*index] += *vertex * coefficient;
-                }
-            }
-            sourceVertices = _blendedVertices.constData();
-        }
-        glm::mat4 transform = glm::translate(_translation);
-        if (mesh.clusters.size() > 1) {
-            _blendedVertices.resize(max(_blendedVertices.size(), vertexCount));
-
-            // skin each vertex
-            const glm::vec4* clusterIndices = mesh.clusterIndices.constData();
-            const glm::vec4* clusterWeights = mesh.clusterWeights.constData();
-            for (int j = 0; j < vertexCount; j++) {
-                _blendedVertices[j] =
-                    glm::vec3(state.clusterMatrices[clusterIndices[j][0]] *
-                        glm::vec4(sourceVertices[j], 1.0f)) * clusterWeights[j][0] +
-                    glm::vec3(state.clusterMatrices[clusterIndices[j][1]] *
-                        glm::vec4(sourceVertices[j], 1.0f)) * clusterWeights[j][1] +
-                    glm::vec3(state.clusterMatrices[clusterIndices[j][2]] *
-                        glm::vec4(sourceVertices[j], 1.0f)) * clusterWeights[j][2] +
-                    glm::vec3(state.clusterMatrices[clusterIndices[j][3]] *
-                        glm::vec4(sourceVertices[j], 1.0f)) * clusterWeights[j][3];
-            }
-            sourceVertices = _blendedVertices.constData();
-            
-        } else {
-            transform = state.clusterMatrices[0];
-        }
-        if (_resetStates) {
-            for (int j = 0; j < vertexCount; j++) {
-                destVertices[j] = glm::vec3(transform * glm::vec4(sourceVertices[j], 1.0f));        
-                destVelocities[j] = glm::vec3();
-            }        
-        } else {
-            const float SPRINGINESS_MULTIPLIER = 200.0f;
-            const float DAMPING = 5.0f;
-            for (int j = 0; j < vertexCount; j++) {
-                destVelocities[j] += ((glm::vec3(transform * glm::vec4(sourceVertices[j], 1.0f)) - destVertices[j]) *
-                    mesh.springiness * SPRINGINESS_MULTIPLIER - destVelocities[j] * DAMPING) * deltaTime;
-                destVertices[j] += destVelocities[j] * deltaTime;
-            }
-        }
-        for (int j = 0; j < vertexCount; j++) {
-            destNormals[j] = glm::vec3();
-            
-            const glm::vec3& middle = destVertices[j];
-            for (QVarLengthArray<QPair<int, int>, 4>::const_iterator connection = mesh.vertexConnections.at(j).constBegin(); 
-                    connection != mesh.vertexConnections.at(j).constEnd(); connection++) {
-                destNormals[j] += glm::normalize(glm::cross(destVertices[connection->second] - middle,
-                    destVertices[connection->first] - middle));
-            }
-        }
-    }
-    _resetStates = false;
 }
 
 bool Model::render(float alpha) {
@@ -572,6 +439,186 @@ bool Model::findSphereCollisions(const glm::vec3& sphereCenter, float sphereRadi
     return collided;
 }
 
+QVector<Model::JointState> Model::updateGeometry() {
+    QVector<JointState> newJointStates;
+    if (_nextGeometry) {
+        _nextGeometry = _nextGeometry->getLODOrFallback(_lodDistance, _nextLODHysteresis);
+        _nextGeometry->setLoadPriority(this, -_lodDistance);
+        _nextGeometry->ensureLoading();
+        if (_nextGeometry->isLoaded()) {
+            applyNextGeometry();
+            return newJointStates;
+        }
+    }
+    if (!_geometry) {
+        return newJointStates;
+    }
+    QSharedPointer<NetworkGeometry> geometry = _geometry->getLODOrFallback(_lodDistance, _lodHysteresis);
+    if (_geometry != geometry) {
+        if (!_jointStates.isEmpty()) {
+            // copy the existing joint states
+            const FBXGeometry& oldGeometry = _geometry->getFBXGeometry();
+            const FBXGeometry& newGeometry = geometry->getFBXGeometry();
+            newJointStates = createJointStates(newGeometry);
+            for (QHash<QString, int>::const_iterator it = oldGeometry.jointIndices.constBegin();
+                    it != oldGeometry.jointIndices.constEnd(); it++) {
+                int oldIndex = it.value() - 1;
+                int newIndex = newGeometry.getJointIndex(it.key());
+                if (newIndex != -1) {
+                    newJointStates[newIndex] = _jointStates.at(oldIndex);
+                }
+            }
+        }
+        deleteGeometry();
+        _dilatedTextures.clear();
+        _geometry = geometry;
+    }
+    _geometry->setLoadPriority(this, -_lodDistance);
+    _geometry->ensureLoading();
+    return newJointStates;
+}
+
+void Model::simulate(float deltaTime, bool fullUpdate, const QVector<JointState>& newJointStates) {
+    if (!isActive()) {
+        return;
+    }
+    
+    // set up world vertices on first simulate after load
+    const FBXGeometry& geometry = _geometry->getFBXGeometry();
+    if (_jointStates.isEmpty()) {
+        _jointStates = newJointStates.isEmpty() ? createJointStates(geometry) : newJointStates;
+        foreach (const FBXMesh& mesh, geometry.meshes) {
+            MeshState state;
+            state.clusterMatrices.resize(mesh.clusters.size());
+            if (mesh.springiness > 0.0f) {
+                state.worldSpaceVertices.resize(mesh.vertices.size());
+                state.vertexVelocities.resize(mesh.vertices.size());
+                state.worldSpaceNormals.resize(mesh.vertices.size());
+            }
+            _meshStates.append(state);    
+        }
+        foreach (const FBXAttachment& attachment, geometry.attachments) {
+            Model* model = new Model(this);
+            model->init();
+            model->setURL(attachment.url);
+            _attachments.append(model);
+        }
+        _resetStates = fullUpdate = true;
+        createCollisionShapes();
+    }
+    
+    // exit early if we don't have to perform a full update
+    if (!(fullUpdate || _resetStates)) {
+        return;
+    }
+    
+    // update the world space transforms for all joints
+    for (int i = 0; i < _jointStates.size(); i++) {
+        updateJointState(i);
+    }
+    
+    // update the attachment transforms and simulate them
+    for (int i = 0; i < _attachments.size(); i++) {
+        const FBXAttachment& attachment = geometry.attachments.at(i);
+        Model* model = _attachments.at(i);
+        
+        glm::vec3 jointTranslation = _translation;
+        glm::quat jointRotation = _rotation;
+        getJointPosition(attachment.jointIndex, jointTranslation);
+        getJointRotation(attachment.jointIndex, jointRotation);
+        
+        model->setTranslation(jointTranslation + jointRotation * attachment.translation * _scale);
+        model->setRotation(jointRotation * attachment.rotation);
+        model->setScale(_scale * attachment.scale);
+        
+        model->simulate(deltaTime);
+    }
+    
+    for (int i = 0; i < _meshStates.size(); i++) {
+        MeshState& state = _meshStates[i];
+        const FBXMesh& mesh = geometry.meshes.at(i);
+        for (int j = 0; j < mesh.clusters.size(); j++) {
+            const FBXCluster& cluster = mesh.clusters.at(j);
+            state.clusterMatrices[j] = _jointStates[cluster.jointIndex].transform * cluster.inverseBindMatrix;
+        }
+        int vertexCount = state.worldSpaceVertices.size();
+        if (vertexCount == 0) {
+            continue;
+        }
+        glm::vec3* destVertices = state.worldSpaceVertices.data();
+        glm::vec3* destVelocities = state.vertexVelocities.data();
+        glm::vec3* destNormals = state.worldSpaceNormals.data();
+        
+        const glm::vec3* sourceVertices = mesh.vertices.constData();
+        if (!mesh.blendshapes.isEmpty()) {
+            _blendedVertices.resize(max(_blendedVertices.size(), vertexCount));
+            memcpy(_blendedVertices.data(), mesh.vertices.constData(), vertexCount * sizeof(glm::vec3));
+            
+            // blend in each coefficient
+            for (unsigned int j = 0; j < _blendshapeCoefficients.size(); j++) {
+                float coefficient = _blendshapeCoefficients[j];
+                if (coefficient == 0.0f || j >= (unsigned int)mesh.blendshapes.size() || mesh.blendshapes[j].vertices.isEmpty()) {
+                    continue;
+                }
+                const glm::vec3* vertex = mesh.blendshapes[j].vertices.constData();
+                for (const int* index = mesh.blendshapes[j].indices.constData(),
+                        *end = index + mesh.blendshapes[j].indices.size(); index != end; index++, vertex++) {
+                    _blendedVertices[*index] += *vertex * coefficient;
+                }
+            }
+            sourceVertices = _blendedVertices.constData();
+        }
+        glm::mat4 transform = glm::translate(_translation);
+        if (mesh.clusters.size() > 1) {
+            _blendedVertices.resize(max(_blendedVertices.size(), vertexCount));
+
+            // skin each vertex
+            const glm::vec4* clusterIndices = mesh.clusterIndices.constData();
+            const glm::vec4* clusterWeights = mesh.clusterWeights.constData();
+            for (int j = 0; j < vertexCount; j++) {
+                _blendedVertices[j] =
+                    glm::vec3(state.clusterMatrices[clusterIndices[j][0]] *
+                        glm::vec4(sourceVertices[j], 1.0f)) * clusterWeights[j][0] +
+                    glm::vec3(state.clusterMatrices[clusterIndices[j][1]] *
+                        glm::vec4(sourceVertices[j], 1.0f)) * clusterWeights[j][1] +
+                    glm::vec3(state.clusterMatrices[clusterIndices[j][2]] *
+                        glm::vec4(sourceVertices[j], 1.0f)) * clusterWeights[j][2] +
+                    glm::vec3(state.clusterMatrices[clusterIndices[j][3]] *
+                        glm::vec4(sourceVertices[j], 1.0f)) * clusterWeights[j][3];
+            }
+            sourceVertices = _blendedVertices.constData();
+            
+        } else {
+            transform = state.clusterMatrices[0];
+        }
+        if (_resetStates) {
+            for (int j = 0; j < vertexCount; j++) {
+                destVertices[j] = glm::vec3(transform * glm::vec4(sourceVertices[j], 1.0f));        
+                destVelocities[j] = glm::vec3();
+            }        
+        } else {
+            const float SPRINGINESS_MULTIPLIER = 200.0f;
+            const float DAMPING = 5.0f;
+            for (int j = 0; j < vertexCount; j++) {
+                destVelocities[j] += ((glm::vec3(transform * glm::vec4(sourceVertices[j], 1.0f)) - destVertices[j]) *
+                    mesh.springiness * SPRINGINESS_MULTIPLIER - destVelocities[j] * DAMPING) * deltaTime;
+                destVertices[j] += destVelocities[j] * deltaTime;
+            }
+        }
+        for (int j = 0; j < vertexCount; j++) {
+            destNormals[j] = glm::vec3();
+            
+            const glm::vec3& middle = destVertices[j];
+            for (QVarLengthArray<QPair<int, int>, 4>::const_iterator connection = mesh.vertexConnections.at(j).constBegin(); 
+                    connection != mesh.vertexConnections.at(j).constEnd(); connection++) {
+                destNormals[j] += glm::normalize(glm::cross(destVertices[connection->second] - middle,
+                    destVertices[connection->first] - middle));
+            }
+        }
+    }
+    _resetStates = false;
+}
+
 void Model::updateJointState(int index) {
     _shapesAreDirty = true;
     JointState& state = _jointStates[index];
@@ -868,49 +915,6 @@ void Model::applyCollision(CollisionInfo& collision) {
     }
 }
 
-QVector<Model::JointState> Model::updateGeometry(bool delayLoad) {
-    QVector<JointState> newJointStates;
-    if (_nextGeometry) {
-        _nextGeometry = _nextGeometry->getLODOrFallback(_lodDistance, _nextLODHysteresis, delayLoad);
-        if (!delayLoad) {
-            _nextGeometry->setLoadPriority(this, -_lodDistance);
-            _nextGeometry->ensureLoading();
-        }
-        if (_nextGeometry->isLoaded()) {
-            applyNextGeometry();
-            return newJointStates;
-        }
-    }
-    if (!_geometry) {
-        return newJointStates;
-    }
-    QSharedPointer<NetworkGeometry> geometry = _geometry->getLODOrFallback(_lodDistance, _lodHysteresis, delayLoad);
-    if (_geometry != geometry) {
-        if (!_jointStates.isEmpty()) {
-            // copy the existing joint states
-            const FBXGeometry& oldGeometry = _geometry->getFBXGeometry();
-            const FBXGeometry& newGeometry = geometry->getFBXGeometry();
-            newJointStates = createJointStates(newGeometry);
-            for (QHash<QString, int>::const_iterator it = oldGeometry.jointIndices.constBegin();
-                    it != oldGeometry.jointIndices.constEnd(); it++) {
-                int oldIndex = it.value() - 1;
-                int newIndex = newGeometry.getJointIndex(it.key());
-                if (newIndex != -1) {
-                    newJointStates[newIndex] = _jointStates.at(oldIndex);
-                }
-            }
-        }
-        deleteGeometry();
-        _dilatedTextures.clear();
-        _geometry = geometry;
-    }
-    if (!delayLoad) {
-        _geometry->setLoadPriority(this, -_lodDistance);
-        _geometry->ensureLoading();
-    }
-    return newJointStates;
-}
-
 void Model::applyNextGeometry() {
     // delete our local geometry and custom textures
     deleteGeometry();

interface/src/renderer/Model.h

@@ -57,7 +57,7 @@ public:
     void clearShapes();
     void createCollisionShapes();
     void updateShapePositions();
-    void simulate(float deltaTime, bool delayLoad = false);
+    void simulate(float deltaTime, bool fullUpdate = true);
     bool render(float alpha);
     
     /// Sets the URL of the model to render.
@@ -226,6 +226,9 @@ protected:
     
     QVector<MeshState> _meshStates;
     
+    QVector<JointState> updateGeometry();
+    void simulate(float deltaTime, bool fullUpdate, const QVector<JointState>& newJointStates);
+    
     /// Updates the state of the joint at the specified index.
     virtual void updateJointState(int index);
     
@@ -256,7 +259,6 @@ protected:
     
 private:
     
-    QVector<JointState> updateGeometry(bool delayLoad);
     void applyNextGeometry();
     void deleteGeometry();
     void renderMeshes(float alpha, bool translucent);