Merge pull request #1028 from ey6es/springy

Support for "springy" models that spring towards target positions.

interface/src/Application.cpp

@@ -332,7 +332,15 @@ void Application::paintGL() {
 
     if (_myCamera.getMode() == CAMERA_MODE_MIRROR) {
         _myCamera.setTightness     (100.0f); 
-        _myCamera.setTargetPosition(_myAvatar.getUprightHeadPosition());
+        glm::vec3 targetPosition = _myAvatar.getUprightHeadPosition();
+        if (_myAvatar.getHead().getBlendFace().isActive()) {
+            // make sure we're aligned to the blend face eyes
+            glm::vec3 leftEyePosition, rightEyePosition;
+            if (_myAvatar.getHead().getBlendFace().getEyePositions(leftEyePosition, rightEyePosition, true)) {
+                targetPosition = (leftEyePosition + rightEyePosition) * 0.5f;
+            }
+        }
+        _myCamera.setTargetPosition(targetPosition);
         _myCamera.setTargetRotation(_myAvatar.getWorldAlignedOrientation() * glm::quat(glm::vec3(0.0f, PIf, 0.0f)));
         
     } else if (OculusManager::isConnected()) {

interface/src/avatar/BlendFace.cpp

@@ -8,6 +8,8 @@
 
 #include <QNetworkReply>
 
+#include <glm/gtx/transform.hpp>
+
 #include "Application.h"
 #include "BlendFace.h"
 #include "Head.h"
@@ -41,21 +43,117 @@ void BlendFace::init() {
     }
 }
 
+void BlendFace::reset() {
+    _resetStates = true;
+}
+
 const glm::vec3 MODEL_TRANSLATION(0.0f, -120.0f, 40.0f); // temporary fudge factor
 const float MODEL_SCALE = 0.0006f;
 
-bool BlendFace::render(float alpha) {
+void BlendFace::simulate(float deltaTime) {
     if (!isActive()) {
+        return;
+    }
+    
+    // set up world vertices on first simulate after load
+    const FBXGeometry& geometry = _geometry->getFBXGeometry();
+    if (_meshStates.isEmpty()) {
+        QVector<glm::vec3> vertices;
+        foreach (const FBXMesh& mesh, geometry.meshes) {
+            MeshState state;
+            if (mesh.springiness > 0.0f) {
+                state.worldSpaceVertices.resize(mesh.vertices.size());
+                state.vertexVelocities.resize(mesh.vertices.size());
+                state.worldSpaceNormals.resize(mesh.vertices.size());
+            }
+            _meshStates.append(state);    
+        }
+        _resetStates = true;
+    }
+    
+    glm::quat orientation = _owningHead->getOrientation();
+    glm::vec3 scale = glm::vec3(-1.0f, 1.0f, -1.0f) * _owningHead->getScale() * MODEL_SCALE;
+    glm::vec3 offset = MODEL_TRANSLATION - _geometry->getFBXGeometry().neckPivot;
+    glm::mat4 baseTransform = glm::translate(_owningHead->getPosition()) * glm::mat4_cast(orientation) *
+        glm::scale(scale) * glm::translate(offset);
+            
+    for (int i = 0; i < _meshStates.size(); i++) {
+        MeshState& state = _meshStates[i];
+        int vertexCount = state.worldSpaceVertices.size();
+        if (vertexCount == 0) {
+            continue;
+        }
+        glm::vec3* destVertices = state.worldSpaceVertices.data();
+        glm::vec3* destVelocities = state.vertexVelocities.data();
+        glm::vec3* destNormals = state.worldSpaceNormals.data();
+        const FBXMesh& mesh = geometry.meshes.at(i);
+        const glm::vec3* sourceVertices = mesh.vertices.constData();
+        if (!mesh.blendshapes.isEmpty()) {
+            _blendedVertices.resize(max(_blendedVertices.size(), vertexCount));
+            memcpy(_blendedVertices.data(), mesh.vertices.constData(), vertexCount * sizeof(glm::vec3));
+            
+            // blend in each coefficient
+            const vector<float>& coefficients = _owningHead->getBlendshapeCoefficients();
+            for (int j = 0; j < coefficients.size(); j++) {
+                float coefficient = coefficients[j];
+                if (coefficient == 0.0f || j >= mesh.blendshapes.size() || mesh.blendshapes[j].vertices.isEmpty()) {
+                    continue;
+                }
+                const glm::vec3* vertex = mesh.blendshapes[j].vertices.constData();
+                for (const int* index = mesh.blendshapes[j].indices.constData(),
+                        *end = index + mesh.blendshapes[j].indices.size(); index != end; index++, vertex++) {
+                    _blendedVertices[*index] += *vertex * coefficient;
+                }
+            }
+            
+            sourceVertices = _blendedVertices.constData();
+        }
+        glm::mat4 transform = baseTransform;
+        if (mesh.isEye) {
+            transform = transform * glm::translate(mesh.pivot) * glm::mat4_cast(glm::inverse(orientation) *
+                _owningHead->getEyeRotation(orientation * ((mesh.pivot + offset) * scale) + _owningHead->getPosition())) *
+                glm::translate(-mesh.pivot);
+        }
+        if (_resetStates) {
+            for (int j = 0; j < vertexCount; j++) {
+                destVertices[j] = glm::vec3(transform * glm::vec4(sourceVertices[j], 1.0f));        
+                destVelocities[j] = glm::vec3();
+            }        
+        } else {
+            const float SPRINGINESS_MULTIPLIER = 200.0f;
+            const float DAMPING = 5.0f;
+            for (int j = 0; j < vertexCount; j++) {
+                destVelocities[j] += ((glm::vec3(transform * glm::vec4(sourceVertices[j], 1.0f)) - destVertices[j]) *
+                    mesh.springiness * SPRINGINESS_MULTIPLIER - destVelocities[j] * DAMPING) * deltaTime;
+                destVertices[j] += destVelocities[j] * deltaTime;
+            }
+        }
+        for (int j = 0; j < vertexCount; j++) {
+            destNormals[j] = glm::vec3();
+            
+            const glm::vec3& middle = destVertices[j];
+            for (QVarLengthArray<QPair<int, int>, 4>::const_iterator connection = mesh.vertexConnections.at(j).constBegin(); 
+                    connection != mesh.vertexConnections.at(j).constEnd(); connection++) {
+                destNormals[j] += glm::normalize(glm::cross(destVertices[connection->second] - middle,
+                    destVertices[connection->first] - middle));
+            }
+        }
+    }
+    _resetStates = false;
+}
+
+bool BlendFace::render(float alpha) {
+    if (_meshStates.isEmpty()) {
         return false;
     }
 
-    // set up blended buffer ids on first render after load
+    // set up blended buffer ids on first render after load/simulate
     const FBXGeometry& geometry = _geometry->getFBXGeometry();
     const QVector<NetworkMesh>& networkMeshes = _geometry->getMeshes();
     if (_blendedVertexBufferIDs.isEmpty()) {
         foreach (const FBXMesh& mesh, geometry.meshes) {
             GLuint id = 0;
-            if (!mesh.blendshapes.isEmpty()) {
+            if (!mesh.blendshapes.isEmpty() || mesh.springiness > 0.0f) {
                 glGenBuffers(1, &id);
                 glBindBuffer(GL_ARRAY_BUFFER, id);
                 glBufferData(GL_ARRAY_BUFFER, (mesh.vertices.size() + mesh.normals.size()) * sizeof(glm::vec3),
@@ -69,6 +167,9 @@ bool BlendFace::render(float alpha) {
         _dilatedTextures.resize(geometry.meshes.size());
     }
 
+    glm::mat4 viewMatrix;
+    glGetFloatv(GL_MODELVIEW_MATRIX, (GLfloat*)&viewMatrix);
+
     glPushMatrix();
     glTranslatef(_owningHead->getPosition().x, _owningHead->getPosition().y, _owningHead->getPosition().z);
     glm::quat orientation = _owningHead->getOrientation();
@@ -130,39 +231,49 @@ bool BlendFace::render(float alpha) {
         glBindTexture(GL_TEXTURE_2D, texture == NULL ? 0 : texture->getID());
         
         glBindBuffer(GL_ARRAY_BUFFER, networkMesh.vertexBufferID);
-        if (mesh.blendshapes.isEmpty()) {
+        if (mesh.blendshapes.isEmpty() && mesh.springiness == 0.0f) {
             glTexCoordPointer(2, GL_FLOAT, 0, (void*)(vertexCount * 2 * sizeof(glm::vec3)));    
         
         } else {
             glTexCoordPointer(2, GL_FLOAT, 0, 0);
-        
-            _blendedVertices.resize(max(_blendedVertices.size(), vertexCount));
-            _blendedNormals.resize(_blendedVertices.size());
-            memcpy(_blendedVertices.data(), mesh.vertices.constData(), vertexCount * sizeof(glm::vec3));
-            memcpy(_blendedNormals.data(), mesh.normals.constData(), vertexCount * sizeof(glm::vec3));
-            
-            // blend in each coefficient
-            const vector<float>& coefficients = _owningHead->getBlendshapeCoefficients();
-            for (int j = 0; j < coefficients.size(); j++) {
-                float coefficient = coefficients[j];
-                if (coefficient == 0.0f || j >= mesh.blendshapes.size() || mesh.blendshapes[j].vertices.isEmpty()) {
-                    continue;
-                }
-                const float NORMAL_COEFFICIENT_SCALE = 0.01f;
-                float normalCoefficient = coefficient * NORMAL_COEFFICIENT_SCALE;
-                const glm::vec3* vertex = mesh.blendshapes[j].vertices.constData();
-                const glm::vec3* normal = mesh.blendshapes[j].normals.constData();
-                for (const int* index = mesh.blendshapes[j].indices.constData(),
-                        *end = index + mesh.blendshapes[j].indices.size(); index != end; index++, vertex++, normal++) {
-                    _blendedVertices[*index] += *vertex * coefficient;
-                    _blendedNormals[*index] += *normal * normalCoefficient;
-                }
-            }
-    
             glBindBuffer(GL_ARRAY_BUFFER, _blendedVertexBufferIDs.at(i));
-            glBufferSubData(GL_ARRAY_BUFFER, 0, vertexCount * sizeof(glm::vec3), _blendedVertices.constData());
-            glBufferSubData(GL_ARRAY_BUFFER, vertexCount * sizeof(glm::vec3),
-                vertexCount * sizeof(glm::vec3), _blendedNormals.constData());
+            
+            const MeshState& state = _meshStates.at(i);
+            if (!state.worldSpaceVertices.isEmpty()) {
+                glLoadMatrixf((const GLfloat*)&viewMatrix);
+            
+                glBufferSubData(GL_ARRAY_BUFFER, 0, vertexCount * sizeof(glm::vec3), state.worldSpaceVertices.constData());
+                glBufferSubData(GL_ARRAY_BUFFER, vertexCount * sizeof(glm::vec3),
+                    vertexCount * sizeof(glm::vec3), state.worldSpaceNormals.constData());
+                
+            } else {
+                _blendedVertices.resize(max(_blendedVertices.size(), vertexCount));
+                _blendedNormals.resize(_blendedVertices.size());
+                memcpy(_blendedVertices.data(), mesh.vertices.constData(), vertexCount * sizeof(glm::vec3));
+                memcpy(_blendedNormals.data(), mesh.normals.constData(), vertexCount * sizeof(glm::vec3));
+                
+                // blend in each coefficient
+                const vector<float>& coefficients = _owningHead->getBlendshapeCoefficients();
+                for (int j = 0; j < coefficients.size(); j++) {
+                    float coefficient = coefficients[j];
+                    if (coefficient == 0.0f || j >= mesh.blendshapes.size() || mesh.blendshapes[j].vertices.isEmpty()) {
+                        continue;
+                    }
+                    const float NORMAL_COEFFICIENT_SCALE = 0.01f;
+                    float normalCoefficient = coefficient * NORMAL_COEFFICIENT_SCALE;
+                    const glm::vec3* vertex = mesh.blendshapes[j].vertices.constData();
+                    const glm::vec3* normal = mesh.blendshapes[j].normals.constData();
+                    for (const int* index = mesh.blendshapes[j].indices.constData(),
+                            *end = index + mesh.blendshapes[j].indices.size(); index != end; index++, vertex++, normal++) {
+                        _blendedVertices[*index] += *vertex * coefficient;
+                        _blendedNormals[*index] += *normal * normalCoefficient;
+                    }
+                }
+        
+                glBufferSubData(GL_ARRAY_BUFFER, 0, vertexCount * sizeof(glm::vec3), _blendedVertices.constData());
+                glBufferSubData(GL_ARRAY_BUFFER, vertexCount * sizeof(glm::vec3),
+                    vertexCount * sizeof(glm::vec3), _blendedNormals.constData());
+            }
         }
         glVertexPointer(3, GL_FLOAT, 0, 0);
         glNormalPointer(GL_FLOAT, 0, (void*)(vertexCount * sizeof(glm::vec3)));
@@ -199,12 +310,16 @@ bool BlendFace::render(float alpha) {
     return true;
 }
 
-void BlendFace::getEyePositions(glm::vec3& firstEyePosition, glm::vec3& secondEyePosition) const {
+bool BlendFace::getEyePositions(glm::vec3& firstEyePosition, glm::vec3& secondEyePosition, bool upright) const {
     if (!isActive()) {
-        return;
+        return false;
     }
-    
+    glm::vec3 translation = _owningHead->getPosition();
     glm::quat orientation = _owningHead->getOrientation();
+    if (upright) {
+        translation = static_cast<MyAvatar*>(_owningHead->_owningAvatar)->getUprightHeadPosition();
+        orientation = static_cast<Avatar*>(_owningHead->_owningAvatar)->getWorldAlignedOrientation();
+    }
     glm::vec3 scale(-_owningHead->getScale() * MODEL_SCALE, _owningHead->getScale() * MODEL_SCALE,
         -_owningHead->getScale() * MODEL_SCALE);
     bool foundFirst = false;
@@ -212,16 +327,16 @@ void BlendFace::getEyePositions(glm::vec3& firstEyePosition, glm::vec3& secondEy
     const FBXGeometry& geometry = _geometry->getFBXGeometry();
     foreach (const FBXMesh& mesh, geometry.meshes) {
         if (mesh.isEye) {
-            glm::vec3 position = orientation * ((mesh.pivot + MODEL_TRANSLATION - geometry.neckPivot) * scale) +
-                _owningHead->getPosition();
+            glm::vec3 position = orientation * ((mesh.pivot + MODEL_TRANSLATION - geometry.neckPivot) * scale) + translation;
             if (foundFirst) {
                 secondEyePosition = position;
-                return;
+                return true;
             }
             firstEyePosition = position;
             foundFirst = true;
         }
     }
+    return false;
 }
 
 void BlendFace::setModelURL(const QUrl& url) {
@@ -243,4 +358,5 @@ void BlendFace::deleteGeometry() {
         glDeleteBuffers(1, &id);
     }
     _blendedVertexBufferIDs.clear();
+    _meshStates.clear();
 }

interface/src/avatar/BlendFace.h

@@ -33,12 +33,17 @@ public:
     bool isActive() const { return _geometry && _geometry->isLoaded(); }
     
     void init();
+    void reset();
+    void simulate(float deltaTime);
     bool render(float alpha);
     
     Q_INVOKABLE void setModelURL(const QUrl& url);
     const QUrl& getModelURL() const { return _modelURL; }
 
-    void getEyePositions(glm::vec3& firstEyePosition, glm::vec3& secondEyePosition) const;
+    /// Retrieve the positions of up to two eye meshes.
+    /// \param upright if true, retrieve the locations of the eyes in the upright position
+    /// \return whether or not both eye meshes were found
+    bool getEyePositions(glm::vec3& firstEyePosition, glm::vec3& secondEyePosition, bool upright = false) const;
     
 private:
     
@@ -50,8 +55,17 @@ private:
     
     QSharedPointer<NetworkGeometry> _geometry;
     
+    class MeshState {
+    public:
+        QVector<glm::vec3> worldSpaceVertices;
+        QVector<glm::vec3> vertexVelocities;
+        QVector<glm::vec3> worldSpaceNormals;
+    };
+    
+    QVector<MeshState> _meshStates;
     QVector<GLuint> _blendedVertexBufferIDs;
     QVector<QSharedPointer<Texture> > _dilatedTextures;
+    bool _resetStates;
     
     QVector<glm::vec3> _blendedVertices;
     QVector<glm::vec3> _blendedNormals;

interface/src/avatar/Head.cpp

@@ -117,6 +117,8 @@ void Head::reset() {
     if (USING_PHYSICAL_MOHAWK) {
         resetHairPhysics();
     }
+    
+    _blendFace.reset();
 }
 
 void Head::resetHairPhysics() {
@@ -235,6 +237,8 @@ void Head::simulate(float deltaTime, bool isMine) {
     if (USING_PHYSICAL_MOHAWK) {
         updateHairPhysics(deltaTime);
     }
+    
+    _blendFace.simulate(deltaTime);
 }
 
 void Head::calculateGeometry() {
@@ -300,15 +304,14 @@ void Head::render(float alpha, bool isMine) {
             renderEyeBrows();
         }
     }
+    
+    if (_blendFace.isActive()) {
+        // the blend face may have custom eye meshes
+        _blendFace.getEyePositions(_leftEyePosition, _rightEyePosition);
+    }
         
     if (_renderLookatVectors) {
-        glm::vec3 firstEyePosition = _leftEyePosition;
-        glm::vec3 secondEyePosition = _rightEyePosition;
-        if (_blendFace.isActive()) {
-            // the blend face may have custom eye meshes
-            _blendFace.getEyePositions(firstEyePosition, secondEyePosition);
-        }
-        renderLookatVectors(firstEyePosition, secondEyePosition, _lookAtPosition);
+        renderLookatVectors(_leftEyePosition, _rightEyePosition, _lookAtPosition);
     }
 }
 

interface/src/avatar/Head.h

@@ -158,6 +158,7 @@ private:
     void resetHairPhysics();
     void updateHairPhysics(float deltaTime);
 
+    friend class BlendFace;
     friend class PerlinFace;
 };
 

interface/src/renderer/FBXReader.cpp

@@ -300,6 +300,7 @@ const char* FACESHIFT_BLENDSHAPES[] = {
 
 class Transform {
 public:
+    QByteArray name;
     bool inheritScale;
     glm::mat4 withScale;
     glm::mat4 withoutScale;
@@ -535,7 +536,7 @@ FBXGeometry extractFBXGeometry(const FBXNode& node, const QVariantHash& mapping)
                     glm::vec3 preRotation, rotation, postRotation;
                     glm::vec3 scale = glm::vec3(1.0f, 1.0f, 1.0f);
                     glm::vec3 scalePivot, rotationPivot;
-                    Transform transform = { true };
+                    Transform transform = { name, true };
                     foreach (const FBXNode& subobject, object.children) {
                         if (subobject.name == "Properties70") {
                             foreach (const FBXNode& property, subobject.children) {
@@ -683,14 +684,15 @@ FBXGeometry extractFBXGeometry(const FBXNode& node, const QVariantHash& mapping)
             mapping.value("ry").toFloat(), mapping.value("rz").toFloat())))) *
         glm::scale(offsetScale, offsetScale, offsetScale);
     
-    // as a temporary hack, put the mesh with the most blendshapes on top; assume it to be the face
     FBXGeometry geometry;
-    int mostBlendshapes = 0;
+    QVariantHash springs = mapping.value("spring").toHash();
+    QVariant defaultSpring = springs.value("default");
     for (QHash<qint64, FBXMesh>::iterator it = meshes.begin(); it != meshes.end(); it++) {
         FBXMesh& mesh = it.value();
         
         // accumulate local transforms
         qint64 modelID = parentMap.value(it.key());
+        mesh.springiness = springs.value(localTransforms.value(modelID).name, defaultSpring).toFloat();
         glm::mat4 modelTransform = getGlobalTransform(parentMap, localTransforms, modelID);
         
         // look for textures, material properties
@@ -735,13 +737,47 @@ FBXGeometry extractFBXGeometry(const FBXNode& node, const QVariantHash& mapping)
             }
         }
         
-        if (mesh.blendshapes.size() > mostBlendshapes) {
-            geometry.meshes.prepend(mesh);    
-            mostBlendshapes = mesh.blendshapes.size();
+        // extract spring edges, connections if springy
+        if (mesh.springiness > 0.0f) {
+            QSet<QPair<int, int> > edges;
             
-        } else {
-            geometry.meshes.append(mesh);
+            mesh.vertexConnections.resize(mesh.vertices.size());
+            for (int i = 0; i < mesh.quadIndices.size(); i += 4) {
+                int index0 = mesh.quadIndices.at(i);
+                int index1 = mesh.quadIndices.at(i + 1);
+                int index2 = mesh.quadIndices.at(i + 2);
+                int index3 = mesh.quadIndices.at(i + 3);
+                
+                edges.insert(QPair<int, int>(qMin(index0, index1), qMax(index0, index1)));
+                edges.insert(QPair<int, int>(qMin(index1, index2), qMax(index1, index2)));
+                edges.insert(QPair<int, int>(qMin(index2, index3), qMax(index2, index3)));
+                edges.insert(QPair<int, int>(qMin(index3, index0), qMax(index3, index0)));
+           
+                mesh.vertexConnections[index0].append(QPair<int, int>(index3, index1));
+                mesh.vertexConnections[index1].append(QPair<int, int>(index0, index2));
+                mesh.vertexConnections[index2].append(QPair<int, int>(index1, index3));
+                mesh.vertexConnections[index3].append(QPair<int, int>(index2, index0));
+            }
+            for (int i = 0; i < mesh.triangleIndices.size(); i += 3) {
+                int index0 = mesh.triangleIndices.at(i);
+                int index1 = mesh.triangleIndices.at(i + 1);
+                int index2 = mesh.triangleIndices.at(i + 2);
+                
+                edges.insert(QPair<int, int>(qMin(index0, index1), qMax(index0, index1)));
+                edges.insert(QPair<int, int>(qMin(index1, index2), qMax(index1, index2)));
+                edges.insert(QPair<int, int>(qMin(index2, index0), qMax(index2, index0)));
+                
+                mesh.vertexConnections[index0].append(QPair<int, int>(index2, index1));
+                mesh.vertexConnections[index1].append(QPair<int, int>(index0, index2));
+                mesh.vertexConnections[index2].append(QPair<int, int>(index1, index0));
+            }
+            
+            for (QSet<QPair<int, int> >::const_iterator edge = edges.constBegin(); edge != edges.constEnd(); edge++) {
+                mesh.springEdges.append(*edge);
+            }
         }
+        
+        geometry.meshes.append(mesh);
     }
     
     // extract translation component for neck pivot

interface/src/renderer/FBXReader.h

@@ -9,6 +9,7 @@
 #ifndef __interface__FBXReader__
 #define __interface__FBXReader__
 
+#include <QVarLengthArray>
 #include <QVariant>
 #include <QVector>
 
@@ -59,6 +60,10 @@ public:
     QByteArray normalFilename;
     
     QVector<FBXBlendshape> blendshapes;
+    
+    float springiness;
+    QVector<QPair<int, int> > springEdges;
+    QVector<QVarLengthArray<QPair<int, int>, 4> > vertexConnections;
 };
 
 /// A set of meshes extracted from an FBX document.

interface/src/renderer/GeometryCache.cpp

@@ -346,7 +346,7 @@ void NetworkGeometry::maybeReadModelWithMapping() {
         glGenBuffers(1, &networkMesh.vertexBufferID);
         glBindBuffer(GL_ARRAY_BUFFER, networkMesh.vertexBufferID);
             
-        if (mesh.blendshapes.isEmpty()) {
+        if (mesh.blendshapes.isEmpty() && mesh.springiness == 0.0f) {
             glBufferData(GL_ARRAY_BUFFER, (mesh.vertices.size() + mesh.normals.size()) * sizeof(glm::vec3) +
                 mesh.texCoords.size() * sizeof(glm::vec2), NULL, GL_STATIC_DRAW);
             glBufferSubData(GL_ARRAY_BUFFER, 0, mesh.vertices.size() * sizeof(glm::vec3), mesh.vertices.constData());