diff --git a/interface/src/Application.cpp b/interface/src/Application.cpp index 71677cc918..297194003e 100644 --- a/interface/src/Application.cpp +++ b/interface/src/Application.cpp @@ -333,7 +333,15 @@ void Application::paintGL() { if (_myCamera.getMode() == CAMERA_MODE_MIRROR) { _myCamera.setTightness (100.0f); - _myCamera.setTargetPosition(_myAvatar.getUprightHeadPosition()); + glm::vec3 targetPosition = _myAvatar.getUprightHeadPosition(); + if (_myAvatar.getHead().getBlendFace().isActive()) { + // make sure we're aligned to the blend face eyes + glm::vec3 leftEyePosition, rightEyePosition; + if (_myAvatar.getHead().getBlendFace().getEyePositions(leftEyePosition, rightEyePosition, true)) { + targetPosition = (leftEyePosition + rightEyePosition) * 0.5f; + } + } + _myCamera.setTargetPosition(targetPosition); _myCamera.setTargetRotation(_myAvatar.getWorldAlignedOrientation() * glm::quat(glm::vec3(0.0f, PIf, 0.0f))); } else if (OculusManager::isConnected()) { diff --git a/interface/src/avatar/BlendFace.cpp b/interface/src/avatar/BlendFace.cpp index 146e5062d5..280dd6a5cc 100644 --- a/interface/src/avatar/BlendFace.cpp +++ b/interface/src/avatar/BlendFace.cpp @@ -8,6 +8,8 @@ #include +#include + #include "Application.h" #include "BlendFace.h" #include "Head.h" @@ -41,21 +43,117 @@ void BlendFace::init() { } } +void BlendFace::reset() { + _resetStates = true; +} + const glm::vec3 MODEL_TRANSLATION(0.0f, -120.0f, 40.0f); // temporary fudge factor const float MODEL_SCALE = 0.0006f; -bool BlendFace::render(float alpha) { +void BlendFace::simulate(float deltaTime) { if (!isActive()) { + return; + } + + // set up world vertices on first simulate after load + const FBXGeometry& geometry = _geometry->getFBXGeometry(); + if (_meshStates.isEmpty()) { + QVector vertices; + foreach (const FBXMesh& mesh, geometry.meshes) { + MeshState state; + if (mesh.springiness > 0.0f) { + state.worldSpaceVertices.resize(mesh.vertices.size()); + state.vertexVelocities.resize(mesh.vertices.size()); + state.worldSpaceNormals.resize(mesh.vertices.size()); + } + _meshStates.append(state); + } + _resetStates = true; + } + + glm::quat orientation = _owningHead->getOrientation(); + glm::vec3 scale = glm::vec3(-1.0f, 1.0f, -1.0f) * _owningHead->getScale() * MODEL_SCALE; + glm::vec3 offset = MODEL_TRANSLATION - _geometry->getFBXGeometry().neckPivot; + glm::mat4 baseTransform = glm::translate(_owningHead->getPosition()) * glm::mat4_cast(orientation) * + glm::scale(scale) * glm::translate(offset); + + for (int i = 0; i < _meshStates.size(); i++) { + MeshState& state = _meshStates[i]; + int vertexCount = state.worldSpaceVertices.size(); + if (vertexCount == 0) { + continue; + } + glm::vec3* destVertices = state.worldSpaceVertices.data(); + glm::vec3* destVelocities = state.vertexVelocities.data(); + glm::vec3* destNormals = state.worldSpaceNormals.data(); + const FBXMesh& mesh = geometry.meshes.at(i); + const glm::vec3* sourceVertices = mesh.vertices.constData(); + if (!mesh.blendshapes.isEmpty()) { + _blendedVertices.resize(max(_blendedVertices.size(), vertexCount)); + memcpy(_blendedVertices.data(), mesh.vertices.constData(), vertexCount * sizeof(glm::vec3)); + + // blend in each coefficient + const vector& coefficients = _owningHead->getBlendshapeCoefficients(); + for (int j = 0; j < coefficients.size(); j++) { + float coefficient = coefficients[j]; + if (coefficient == 0.0f || j >= mesh.blendshapes.size() || mesh.blendshapes[j].vertices.isEmpty()) { + continue; + } + const glm::vec3* vertex = mesh.blendshapes[j].vertices.constData(); + for (const int* index = mesh.blendshapes[j].indices.constData(), + *end = index + mesh.blendshapes[j].indices.size(); index != end; index++, vertex++) { + _blendedVertices[*index] += *vertex * coefficient; + } + } + + sourceVertices = _blendedVertices.constData(); + } + glm::mat4 transform = baseTransform; + if (mesh.isEye) { + transform = transform * glm::translate(mesh.pivot) * glm::mat4_cast(glm::inverse(orientation) * + _owningHead->getEyeRotation(orientation * ((mesh.pivot + offset) * scale) + _owningHead->getPosition())) * + glm::translate(-mesh.pivot); + } + if (_resetStates) { + for (int j = 0; j < vertexCount; j++) { + destVertices[j] = glm::vec3(transform * glm::vec4(sourceVertices[j], 1.0f)); + destVelocities[j] = glm::vec3(); + } + } else { + const float SPRINGINESS_MULTIPLIER = 200.0f; + const float DAMPING = 5.0f; + for (int j = 0; j < vertexCount; j++) { + destVelocities[j] += ((glm::vec3(transform * glm::vec4(sourceVertices[j], 1.0f)) - destVertices[j]) * + mesh.springiness * SPRINGINESS_MULTIPLIER - destVelocities[j] * DAMPING) * deltaTime; + destVertices[j] += destVelocities[j] * deltaTime; + } + } + for (int j = 0; j < vertexCount; j++) { + destNormals[j] = glm::vec3(); + + const glm::vec3& middle = destVertices[j]; + for (QVarLengthArray, 4>::const_iterator connection = mesh.vertexConnections.at(j).constBegin(); + connection != mesh.vertexConnections.at(j).constEnd(); connection++) { + destNormals[j] += glm::normalize(glm::cross(destVertices[connection->second] - middle, + destVertices[connection->first] - middle)); + } + } + } + _resetStates = false; +} + +bool BlendFace::render(float alpha) { + if (_meshStates.isEmpty()) { return false; } - // set up blended buffer ids on first render after load + // set up blended buffer ids on first render after load/simulate const FBXGeometry& geometry = _geometry->getFBXGeometry(); const QVector& networkMeshes = _geometry->getMeshes(); if (_blendedVertexBufferIDs.isEmpty()) { foreach (const FBXMesh& mesh, geometry.meshes) { GLuint id = 0; - if (!mesh.blendshapes.isEmpty()) { + if (!mesh.blendshapes.isEmpty() || mesh.springiness > 0.0f) { glGenBuffers(1, &id); glBindBuffer(GL_ARRAY_BUFFER, id); glBufferData(GL_ARRAY_BUFFER, (mesh.vertices.size() + mesh.normals.size()) * sizeof(glm::vec3), @@ -69,6 +167,9 @@ bool BlendFace::render(float alpha) { _dilatedTextures.resize(geometry.meshes.size()); } + glm::mat4 viewMatrix; + glGetFloatv(GL_MODELVIEW_MATRIX, (GLfloat*)&viewMatrix); + glPushMatrix(); glTranslatef(_owningHead->getPosition().x, _owningHead->getPosition().y, _owningHead->getPosition().z); glm::quat orientation = _owningHead->getOrientation(); @@ -130,39 +231,49 @@ bool BlendFace::render(float alpha) { glBindTexture(GL_TEXTURE_2D, texture == NULL ? 0 : texture->getID()); glBindBuffer(GL_ARRAY_BUFFER, networkMesh.vertexBufferID); - if (mesh.blendshapes.isEmpty()) { + if (mesh.blendshapes.isEmpty() && mesh.springiness == 0.0f) { glTexCoordPointer(2, GL_FLOAT, 0, (void*)(vertexCount * 2 * sizeof(glm::vec3))); } else { glTexCoordPointer(2, GL_FLOAT, 0, 0); - - _blendedVertices.resize(max(_blendedVertices.size(), vertexCount)); - _blendedNormals.resize(_blendedVertices.size()); - memcpy(_blendedVertices.data(), mesh.vertices.constData(), vertexCount * sizeof(glm::vec3)); - memcpy(_blendedNormals.data(), mesh.normals.constData(), vertexCount * sizeof(glm::vec3)); - - // blend in each coefficient - const vector& coefficients = _owningHead->getBlendshapeCoefficients(); - for (int j = 0; j < coefficients.size(); j++) { - float coefficient = coefficients[j]; - if (coefficient == 0.0f || j >= mesh.blendshapes.size() || mesh.blendshapes[j].vertices.isEmpty()) { - continue; - } - const float NORMAL_COEFFICIENT_SCALE = 0.01f; - float normalCoefficient = coefficient * NORMAL_COEFFICIENT_SCALE; - const glm::vec3* vertex = mesh.blendshapes[j].vertices.constData(); - const glm::vec3* normal = mesh.blendshapes[j].normals.constData(); - for (const int* index = mesh.blendshapes[j].indices.constData(), - *end = index + mesh.blendshapes[j].indices.size(); index != end; index++, vertex++, normal++) { - _blendedVertices[*index] += *vertex * coefficient; - _blendedNormals[*index] += *normal * normalCoefficient; - } - } - glBindBuffer(GL_ARRAY_BUFFER, _blendedVertexBufferIDs.at(i)); - glBufferSubData(GL_ARRAY_BUFFER, 0, vertexCount * sizeof(glm::vec3), _blendedVertices.constData()); - glBufferSubData(GL_ARRAY_BUFFER, vertexCount * sizeof(glm::vec3), - vertexCount * sizeof(glm::vec3), _blendedNormals.constData()); + + const MeshState& state = _meshStates.at(i); + if (!state.worldSpaceVertices.isEmpty()) { + glLoadMatrixf((const GLfloat*)&viewMatrix); + + glBufferSubData(GL_ARRAY_BUFFER, 0, vertexCount * sizeof(glm::vec3), state.worldSpaceVertices.constData()); + glBufferSubData(GL_ARRAY_BUFFER, vertexCount * sizeof(glm::vec3), + vertexCount * sizeof(glm::vec3), state.worldSpaceNormals.constData()); + + } else { + _blendedVertices.resize(max(_blendedVertices.size(), vertexCount)); + _blendedNormals.resize(_blendedVertices.size()); + memcpy(_blendedVertices.data(), mesh.vertices.constData(), vertexCount * sizeof(glm::vec3)); + memcpy(_blendedNormals.data(), mesh.normals.constData(), vertexCount * sizeof(glm::vec3)); + + // blend in each coefficient + const vector& coefficients = _owningHead->getBlendshapeCoefficients(); + for (int j = 0; j < coefficients.size(); j++) { + float coefficient = coefficients[j]; + if (coefficient == 0.0f || j >= mesh.blendshapes.size() || mesh.blendshapes[j].vertices.isEmpty()) { + continue; + } + const float NORMAL_COEFFICIENT_SCALE = 0.01f; + float normalCoefficient = coefficient * NORMAL_COEFFICIENT_SCALE; + const glm::vec3* vertex = mesh.blendshapes[j].vertices.constData(); + const glm::vec3* normal = mesh.blendshapes[j].normals.constData(); + for (const int* index = mesh.blendshapes[j].indices.constData(), + *end = index + mesh.blendshapes[j].indices.size(); index != end; index++, vertex++, normal++) { + _blendedVertices[*index] += *vertex * coefficient; + _blendedNormals[*index] += *normal * normalCoefficient; + } + } + + glBufferSubData(GL_ARRAY_BUFFER, 0, vertexCount * sizeof(glm::vec3), _blendedVertices.constData()); + glBufferSubData(GL_ARRAY_BUFFER, vertexCount * sizeof(glm::vec3), + vertexCount * sizeof(glm::vec3), _blendedNormals.constData()); + } } glVertexPointer(3, GL_FLOAT, 0, 0); glNormalPointer(GL_FLOAT, 0, (void*)(vertexCount * sizeof(glm::vec3))); @@ -199,12 +310,16 @@ bool BlendFace::render(float alpha) { return true; } -void BlendFace::getEyePositions(glm::vec3& firstEyePosition, glm::vec3& secondEyePosition) const { +bool BlendFace::getEyePositions(glm::vec3& firstEyePosition, glm::vec3& secondEyePosition, bool upright) const { if (!isActive()) { - return; + return false; } - + glm::vec3 translation = _owningHead->getPosition(); glm::quat orientation = _owningHead->getOrientation(); + if (upright) { + translation = static_cast(_owningHead->_owningAvatar)->getUprightHeadPosition(); + orientation = static_cast(_owningHead->_owningAvatar)->getWorldAlignedOrientation(); + } glm::vec3 scale(-_owningHead->getScale() * MODEL_SCALE, _owningHead->getScale() * MODEL_SCALE, -_owningHead->getScale() * MODEL_SCALE); bool foundFirst = false; @@ -212,16 +327,16 @@ void BlendFace::getEyePositions(glm::vec3& firstEyePosition, glm::vec3& secondEy const FBXGeometry& geometry = _geometry->getFBXGeometry(); foreach (const FBXMesh& mesh, geometry.meshes) { if (mesh.isEye) { - glm::vec3 position = orientation * ((mesh.pivot + MODEL_TRANSLATION - geometry.neckPivot) * scale) + - _owningHead->getPosition(); + glm::vec3 position = orientation * ((mesh.pivot + MODEL_TRANSLATION - geometry.neckPivot) * scale) + translation; if (foundFirst) { secondEyePosition = position; - return; + return true; } firstEyePosition = position; foundFirst = true; } } + return false; } void BlendFace::setModelURL(const QUrl& url) { @@ -243,4 +358,5 @@ void BlendFace::deleteGeometry() { glDeleteBuffers(1, &id); } _blendedVertexBufferIDs.clear(); + _meshStates.clear(); } diff --git a/interface/src/avatar/BlendFace.h b/interface/src/avatar/BlendFace.h index 9307dc5c89..ea2d2eb597 100644 --- a/interface/src/avatar/BlendFace.h +++ b/interface/src/avatar/BlendFace.h @@ -33,12 +33,17 @@ public: bool isActive() const { return _geometry && _geometry->isLoaded(); } void init(); + void reset(); + void simulate(float deltaTime); bool render(float alpha); Q_INVOKABLE void setModelURL(const QUrl& url); const QUrl& getModelURL() const { return _modelURL; } - void getEyePositions(glm::vec3& firstEyePosition, glm::vec3& secondEyePosition) const; + /// Retrieve the positions of up to two eye meshes. + /// \param upright if true, retrieve the locations of the eyes in the upright position + /// \return whether or not both eye meshes were found + bool getEyePositions(glm::vec3& firstEyePosition, glm::vec3& secondEyePosition, bool upright = false) const; private: @@ -50,8 +55,17 @@ private: QSharedPointer _geometry; + class MeshState { + public: + QVector worldSpaceVertices; + QVector vertexVelocities; + QVector worldSpaceNormals; + }; + + QVector _meshStates; QVector _blendedVertexBufferIDs; QVector > _dilatedTextures; + bool _resetStates; QVector _blendedVertices; QVector _blendedNormals; diff --git a/interface/src/avatar/Head.cpp b/interface/src/avatar/Head.cpp index e54d2fb18c..635edf29d4 100644 --- a/interface/src/avatar/Head.cpp +++ b/interface/src/avatar/Head.cpp @@ -117,6 +117,8 @@ void Head::reset() { if (USING_PHYSICAL_MOHAWK) { resetHairPhysics(); } + + _blendFace.reset(); } void Head::resetHairPhysics() { @@ -235,6 +237,8 @@ void Head::simulate(float deltaTime, bool isMine) { if (USING_PHYSICAL_MOHAWK) { updateHairPhysics(deltaTime); } + + _blendFace.simulate(deltaTime); } void Head::calculateGeometry() { @@ -300,15 +304,14 @@ void Head::render(float alpha, bool isMine) { renderEyeBrows(); } } + + if (_blendFace.isActive()) { + // the blend face may have custom eye meshes + _blendFace.getEyePositions(_leftEyePosition, _rightEyePosition); + } if (_renderLookatVectors) { - glm::vec3 firstEyePosition = _leftEyePosition; - glm::vec3 secondEyePosition = _rightEyePosition; - if (_blendFace.isActive()) { - // the blend face may have custom eye meshes - _blendFace.getEyePositions(firstEyePosition, secondEyePosition); - } - renderLookatVectors(firstEyePosition, secondEyePosition, _lookAtPosition); + renderLookatVectors(_leftEyePosition, _rightEyePosition, _lookAtPosition); } } diff --git a/interface/src/avatar/Head.h b/interface/src/avatar/Head.h index 38574fa430..7c5491ae39 100644 --- a/interface/src/avatar/Head.h +++ b/interface/src/avatar/Head.h @@ -158,6 +158,7 @@ private: void resetHairPhysics(); void updateHairPhysics(float deltaTime); + friend class BlendFace; friend class PerlinFace; }; diff --git a/interface/src/renderer/FBXReader.cpp b/interface/src/renderer/FBXReader.cpp index c8647459d7..8091687488 100644 --- a/interface/src/renderer/FBXReader.cpp +++ b/interface/src/renderer/FBXReader.cpp @@ -300,6 +300,7 @@ const char* FACESHIFT_BLENDSHAPES[] = { class Transform { public: + QByteArray name; bool inheritScale; glm::mat4 withScale; glm::mat4 withoutScale; @@ -535,7 +536,7 @@ FBXGeometry extractFBXGeometry(const FBXNode& node, const QVariantHash& mapping) glm::vec3 preRotation, rotation, postRotation; glm::vec3 scale = glm::vec3(1.0f, 1.0f, 1.0f); glm::vec3 scalePivot, rotationPivot; - Transform transform = { true }; + Transform transform = { name, true }; foreach (const FBXNode& subobject, object.children) { if (subobject.name == "Properties70") { foreach (const FBXNode& property, subobject.children) { @@ -683,14 +684,15 @@ FBXGeometry extractFBXGeometry(const FBXNode& node, const QVariantHash& mapping) mapping.value("ry").toFloat(), mapping.value("rz").toFloat())))) * glm::scale(offsetScale, offsetScale, offsetScale); - // as a temporary hack, put the mesh with the most blendshapes on top; assume it to be the face FBXGeometry geometry; - int mostBlendshapes = 0; + QVariantHash springs = mapping.value("spring").toHash(); + QVariant defaultSpring = springs.value("default"); for (QHash::iterator it = meshes.begin(); it != meshes.end(); it++) { FBXMesh& mesh = it.value(); // accumulate local transforms qint64 modelID = parentMap.value(it.key()); + mesh.springiness = springs.value(localTransforms.value(modelID).name, defaultSpring).toFloat(); glm::mat4 modelTransform = getGlobalTransform(parentMap, localTransforms, modelID); // look for textures, material properties @@ -735,13 +737,47 @@ FBXGeometry extractFBXGeometry(const FBXNode& node, const QVariantHash& mapping) } } - if (mesh.blendshapes.size() > mostBlendshapes) { - geometry.meshes.prepend(mesh); - mostBlendshapes = mesh.blendshapes.size(); + // extract spring edges, connections if springy + if (mesh.springiness > 0.0f) { + QSet > edges; - } else { - geometry.meshes.append(mesh); + mesh.vertexConnections.resize(mesh.vertices.size()); + for (int i = 0; i < mesh.quadIndices.size(); i += 4) { + int index0 = mesh.quadIndices.at(i); + int index1 = mesh.quadIndices.at(i + 1); + int index2 = mesh.quadIndices.at(i + 2); + int index3 = mesh.quadIndices.at(i + 3); + + edges.insert(QPair(qMin(index0, index1), qMax(index0, index1))); + edges.insert(QPair(qMin(index1, index2), qMax(index1, index2))); + edges.insert(QPair(qMin(index2, index3), qMax(index2, index3))); + edges.insert(QPair(qMin(index3, index0), qMax(index3, index0))); + + mesh.vertexConnections[index0].append(QPair(index3, index1)); + mesh.vertexConnections[index1].append(QPair(index0, index2)); + mesh.vertexConnections[index2].append(QPair(index1, index3)); + mesh.vertexConnections[index3].append(QPair(index2, index0)); + } + for (int i = 0; i < mesh.triangleIndices.size(); i += 3) { + int index0 = mesh.triangleIndices.at(i); + int index1 = mesh.triangleIndices.at(i + 1); + int index2 = mesh.triangleIndices.at(i + 2); + + edges.insert(QPair(qMin(index0, index1), qMax(index0, index1))); + edges.insert(QPair(qMin(index1, index2), qMax(index1, index2))); + edges.insert(QPair(qMin(index2, index0), qMax(index2, index0))); + + mesh.vertexConnections[index0].append(QPair(index2, index1)); + mesh.vertexConnections[index1].append(QPair(index0, index2)); + mesh.vertexConnections[index2].append(QPair(index1, index0)); + } + + for (QSet >::const_iterator edge = edges.constBegin(); edge != edges.constEnd(); edge++) { + mesh.springEdges.append(*edge); + } } + + geometry.meshes.append(mesh); } // extract translation component for neck pivot diff --git a/interface/src/renderer/FBXReader.h b/interface/src/renderer/FBXReader.h index 310dea265f..5c2ffd9ee0 100644 --- a/interface/src/renderer/FBXReader.h +++ b/interface/src/renderer/FBXReader.h @@ -9,6 +9,7 @@ #ifndef __interface__FBXReader__ #define __interface__FBXReader__ +#include #include #include @@ -59,6 +60,10 @@ public: QByteArray normalFilename; QVector blendshapes; + + float springiness; + QVector > springEdges; + QVector, 4> > vertexConnections; }; /// A set of meshes extracted from an FBX document. diff --git a/interface/src/renderer/GeometryCache.cpp b/interface/src/renderer/GeometryCache.cpp index c63ec7eed4..aa717175a0 100644 --- a/interface/src/renderer/GeometryCache.cpp +++ b/interface/src/renderer/GeometryCache.cpp @@ -346,7 +346,7 @@ void NetworkGeometry::maybeReadModelWithMapping() { glGenBuffers(1, &networkMesh.vertexBufferID); glBindBuffer(GL_ARRAY_BUFFER, networkMesh.vertexBufferID); - if (mesh.blendshapes.isEmpty()) { + if (mesh.blendshapes.isEmpty() && mesh.springiness == 0.0f) { glBufferData(GL_ARRAY_BUFFER, (mesh.vertices.size() + mesh.normals.size()) * sizeof(glm::vec3) + mesh.texCoords.size() * sizeof(glm::vec2), NULL, GL_STATIC_DRAW); glBufferSubData(GL_ARRAY_BUFFER, 0, mesh.vertices.size() * sizeof(glm::vec3), mesh.vertices.constData());