From db30e729d4e15ae199c45b5163851b260c224cd5 Mon Sep 17 00:00:00 2001 From: Andrzej Kapolka Date: Mon, 14 Oct 2013 16:46:17 -0700 Subject: [PATCH] More work towards skinning. --- interface/src/avatar/BlendFace.cpp | 124 +++++++++++++---------- interface/src/avatar/BlendFace.h | 2 + interface/src/renderer/FBXReader.cpp | 58 ++++++----- interface/src/renderer/FBXReader.h | 6 ++ interface/src/renderer/GeometryCache.cpp | 25 ++++- 5 files changed, 134 insertions(+), 81 deletions(-) diff --git a/interface/src/avatar/BlendFace.cpp b/interface/src/avatar/BlendFace.cpp index cd2d3aa59f..1380703fe6 100644 --- a/interface/src/avatar/BlendFace.cpp +++ b/interface/src/avatar/BlendFace.cpp @@ -61,10 +61,12 @@ void BlendFace::simulate(float deltaTime) { QVector vertices; foreach (const FBXJoint& joint, geometry.joints) { JointState state; + state.rotation = joint.rotation; _jointStates.append(state); } foreach (const FBXMesh& mesh, geometry.meshes) { MeshState state; + state.jointMatrices.resize(mesh.clusters.size()); if (mesh.springiness > 0.0f) { state.worldSpaceVertices.resize(mesh.vertices.size()); state.vertexVelocities.resize(mesh.vertices.size()); @@ -75,14 +77,44 @@ void BlendFace::simulate(float deltaTime) { _resetStates = true; } - glm::quat orientation = _owningHead->getOrientation(); + glm::quat orientation = static_cast(_owningHead->_owningAvatar)->getOrientation(); glm::vec3 scale = glm::vec3(-1.0f, 1.0f, -1.0f) * _owningHead->getScale() * MODEL_SCALE; glm::vec3 offset = MODEL_TRANSLATION - _geometry->getFBXGeometry().neckPivot; glm::mat4 baseTransform = glm::translate(_owningHead->getPosition()) * glm::mat4_cast(orientation) * glm::scale(scale) * glm::translate(offset); + + // apply the neck rotation + if (geometry.neckJointIndex != -1) { + _jointStates[geometry.neckJointIndex].rotation = glm::quat(glm::radians(glm::vec3( + _owningHead->getPitch(), _owningHead->getYaw(), _owningHead->getRoll()))); + } + + // update the world space transforms for all joints + for (int i = 0; i < _jointStates.size(); i++) { + JointState& state = _jointStates[i]; + const FBXJoint& joint = geometry.joints.at(i); + if (joint.parentIndex == -1) { + state.transform = baseTransform * geometry.offset * joint.preRotation * + glm::mat4_cast(state.rotation) * joint.postRotation; + } else { + state.transform = _jointStates[joint.parentIndex].transform * joint.preRotation; + if (i == geometry.leftEyeJointIndex || i == geometry.rightEyeJointIndex) { + // extract the translation component of the matrix + state.rotation = _owningHead->getEyeRotation(glm::vec3( + state.transform[3][0], state.transform[3][1], state.transform[3][2])); + } + state.transform = state.transform * glm::mat4_cast(state.rotation) * joint.postRotation; + } + } + for (int i = 0; i < _meshStates.size(); i++) { MeshState& state = _meshStates[i]; + const FBXMesh& mesh = geometry.meshes.at(i); + for (int j = 0; j < mesh.clusters.size(); j++) { + const FBXCluster& cluster = mesh.clusters.at(j); + state.jointMatrices[j] = _jointStates[cluster.jointIndex].transform * cluster.inverseBindMatrix; + } int vertexCount = state.worldSpaceVertices.size(); if (vertexCount == 0) { continue; @@ -90,7 +122,7 @@ void BlendFace::simulate(float deltaTime) { glm::vec3* destVertices = state.worldSpaceVertices.data(); glm::vec3* destVelocities = state.vertexVelocities.data(); glm::vec3* destNormals = state.worldSpaceNormals.data(); - const FBXMesh& mesh = geometry.meshes.at(i); + const glm::vec3* sourceVertices = mesh.vertices.constData(); if (!mesh.blendshapes.isEmpty()) { _blendedVertices.resize(max(_blendedVertices.size(), vertexCount)); @@ -112,11 +144,29 @@ void BlendFace::simulate(float deltaTime) { sourceVertices = _blendedVertices.constData(); } - glm::mat4 transform = baseTransform; - if (mesh.isEye) { - transform = transform * glm::translate(mesh.pivot) * glm::mat4_cast(glm::inverse(orientation) * - _owningHead->getEyeRotation(orientation * ((mesh.pivot + offset) * scale) + _owningHead->getPosition())) * - glm::translate(-mesh.pivot); + glm::mat4 transform; + if (mesh.clusters.size() > 1) { + _blendedVertices.resize(max(_blendedVertices.size(), vertexCount)); + + // skin each vertex + const glm::vec4* clusterIndices = mesh.clusterIndices.constData(); + const glm::vec4* clusterWeights = mesh.clusterWeights.constData(); + for (int j = 0; j < vertexCount; j++) { + _blendedVertices[j] = + glm::vec3(state.jointMatrices[clusterIndices[j][0]] * + glm::vec4(sourceVertices[j], 1.0f)) * clusterWeights[j][0] + + glm::vec3(state.jointMatrices[clusterIndices[j][1]] * + glm::vec4(sourceVertices[j], 1.0f)) * clusterWeights[j][1] + + glm::vec3(state.jointMatrices[clusterIndices[j][2]] * + glm::vec4(sourceVertices[j], 1.0f)) * clusterWeights[j][2] + + glm::vec3(state.jointMatrices[clusterIndices[j][3]] * + glm::vec4(sourceVertices[j], 1.0f)) * clusterWeights[j][3]; + } + + sourceVertices = _blendedVertices.constData(); + + } else { + transform = state.jointMatrices[0]; } if (_resetStates) { for (int j = 0; j < vertexCount; j++) { @@ -171,21 +221,11 @@ bool BlendFace::render(float alpha) { _dilatedTextures.resize(geometry.meshes.size()); } - glm::mat4 viewMatrix; - glGetFloatv(GL_MODELVIEW_MATRIX, (GLfloat*)&viewMatrix); - - glPushMatrix(); - glTranslatef(_owningHead->getPosition().x, _owningHead->getPosition().y, _owningHead->getPosition().z); glm::quat orientation = _owningHead->getOrientation(); - glm::vec3 axis = glm::axis(orientation); - glRotatef(glm::angle(orientation), axis.x, axis.y, axis.z); glm::vec3 scale(-_owningHead->getScale() * MODEL_SCALE, _owningHead->getScale() * MODEL_SCALE, -_owningHead->getScale() * MODEL_SCALE); - glScalef(scale.x, scale.y, scale.z); - glm::vec3 offset = MODEL_TRANSLATION - geometry.neckPivot; - glTranslatef(offset.x, offset.y, offset.z); - + glEnableClientState(GL_VERTEX_ARRAY); glEnableClientState(GL_NORMAL_ARRAY); glEnableClientState(GL_TEXTURE_COORD_ARRAY); @@ -210,13 +250,6 @@ bool BlendFace::render(float alpha) { // apply eye rotation if appropriate Texture* texture = networkMesh.diffuseTexture.data(); if (mesh.isEye) { - glTranslatef(mesh.pivot.x, mesh.pivot.y, mesh.pivot.z); - glm::quat rotation = glm::inverse(orientation) * _owningHead->getEyeRotation(orientation * - ((mesh.pivot + offset) * scale) + _owningHead->getPosition()); - glm::vec3 rotationAxis = glm::axis(rotation); - glRotatef(glm::angle(rotation), -rotationAxis.x, rotationAxis.y, -rotationAxis.z); - glTranslatef(-mesh.pivot.x, -mesh.pivot.y, -mesh.pivot.z); - _eyeProgram.bind(); if (texture != NULL) { @@ -233,7 +266,10 @@ bool BlendFace::render(float alpha) { glMaterialfv(GL_FRONT, GL_SPECULAR, (const float*)&specular); glMaterialf(GL_FRONT, GL_SHININESS, mesh.shininess); - glMultMatrixf((const GLfloat*)&mesh.transform); + const MeshState& state = _meshStates.at(i); + if (state.worldSpaceVertices.isEmpty()) { + glMultMatrixf((const GLfloat*)&state.jointMatrices[0]); + } glBindTexture(GL_TEXTURE_2D, texture == NULL ? 0 : texture->getID()); @@ -245,10 +281,7 @@ bool BlendFace::render(float alpha) { glTexCoordPointer(2, GL_FLOAT, 0, 0); glBindBuffer(GL_ARRAY_BUFFER, _blendedVertexBufferIDs.at(i)); - const MeshState& state = _meshStates.at(i); if (!state.worldSpaceVertices.isEmpty()) { - glLoadMatrixf((const GLfloat*)&viewMatrix); - glBufferSubData(GL_ARRAY_BUFFER, 0, vertexCount * sizeof(glm::vec3), state.worldSpaceVertices.constData()); glBufferSubData(GL_ARRAY_BUFFER, vertexCount * sizeof(glm::vec3), vertexCount * sizeof(glm::vec3), state.worldSpaceNormals.constData()); @@ -309,8 +342,6 @@ bool BlendFace::render(float alpha) { glBindBuffer(GL_ELEMENT_ARRAY_BUFFER, 0); glBindTexture(GL_TEXTURE_2D, 0); - glPopMatrix(); - // restore all the default material settings Application::getInstance()->setupWorldLight(*Application::getInstance()->getCamera()); @@ -318,32 +349,19 @@ bool BlendFace::render(float alpha) { } bool BlendFace::getEyePositions(glm::vec3& firstEyePosition, glm::vec3& secondEyePosition, bool upright) const { - if (!isActive()) { + if (!isActive() || _jointStates.isEmpty()) { return false; } - glm::vec3 translation = _owningHead->getPosition(); - glm::quat orientation = _owningHead->getOrientation(); - if (upright) { - translation = static_cast(_owningHead->_owningAvatar)->getUprightHeadPosition(); - orientation = static_cast(_owningHead->_owningAvatar)->getWorldAlignedOrientation(); - } - glm::vec3 scale(-_owningHead->getScale() * MODEL_SCALE, _owningHead->getScale() * MODEL_SCALE, - -_owningHead->getScale() * MODEL_SCALE); - bool foundFirst = false; - const FBXGeometry& geometry = _geometry->getFBXGeometry(); - foreach (const FBXMesh& mesh, geometry.meshes) { - if (mesh.isEye) { - glm::vec3 position = orientation * ((mesh.pivot + MODEL_TRANSLATION - geometry.neckPivot) * scale) + translation; - if (foundFirst) { - secondEyePosition = position; - return true; - } - firstEyePosition = position; - foundFirst = true; - } + if (geometry.leftEyeJointIndex != -1) { + const glm::mat4& transform = _jointStates[geometry.leftEyeJointIndex].transform; + firstEyePosition = glm::vec3(transform[3][0], transform[3][1], transform[3][2]); } - return false; + if (geometry.rightEyeJointIndex != -1) { + const glm::mat4& transform = _jointStates[geometry.rightEyeJointIndex].transform; + secondEyePosition = glm::vec3(transform[3][0], transform[3][1], transform[3][2]); + } + return geometry.leftEyeJointIndex != -1 && geometry.rightEyeJointIndex != -1; } glm::vec4 BlendFace::computeAverageColor() const { diff --git a/interface/src/avatar/BlendFace.h b/interface/src/avatar/BlendFace.h index 62b04ca19e..df944558b1 100644 --- a/interface/src/avatar/BlendFace.h +++ b/interface/src/avatar/BlendFace.h @@ -61,12 +61,14 @@ private: class JointState { public: glm::quat rotation; + glm::mat4 transform; }; QVector _jointStates; class MeshState { public: + QVector jointMatrices; QVector worldSpaceVertices; QVector vertexVelocities; QVector worldSpaceNormals; diff --git a/interface/src/renderer/FBXReader.cpp b/interface/src/renderer/FBXReader.cpp index c55872dd1a..c20a965a24 100644 --- a/interface/src/renderer/FBXReader.cpp +++ b/interface/src/renderer/FBXReader.cpp @@ -301,20 +301,19 @@ const char* FACESHIFT_BLENDSHAPES[] = { class Model { public: QByteArray name; - bool inheritScale; - glm::mat4 withScale; - glm::mat4 withoutScale; + + glm::mat4 preRotation; + glm::quat rotation; + glm::mat4 postRotation; + + int parentIndex; }; -glm::mat4 getGlobalTransform(const QMultiHash& parentMap, const QHash& models, - qint64 nodeID, bool forceScale = true) { - +glm::mat4 getGlobalTransform(const QMultiHash& parentMap, const QHash& models, qint64 nodeID) { glm::mat4 globalTransform; - bool useScale = true; while (nodeID != 0) { const Model& model = models.value(nodeID); - globalTransform = (useScale ? model.withScale : model.withoutScale) * globalTransform; - useScale = (useScale && model.inheritScale) || forceScale; + globalTransform = model.preRotation * glm::mat4_cast(model.rotation) * model.postRotation * globalTransform; QList parentIDs = parentMap.values(nodeID); nodeID = 0; @@ -361,12 +360,15 @@ public: glm::mat4 transformLink; }; -void appendModelIDs(qint64 parentID, const QMultiHash& childMap, QHash& models, QVector& modelIDs) { +void appendModelIDs(qint64 parentID, const QMultiHash& childMap, + QHash& models, QVector& modelIDs) { if (parentID != 0) { modelIDs.append(parentID); } + int parentIndex = modelIDs.size() - 1; foreach (qint64 childID, childMap.values(parentID)) { if (models.contains(childID)) { + models[childID].parentIndex = parentIndex; appendModelIDs(childID, childMap, models, modelIDs); } } @@ -554,7 +556,7 @@ FBXGeometry extractFBXGeometry(const FBXNode& node, const QVariantHash& mapping) glm::vec3 preRotation, rotation, postRotation; glm::vec3 scale = glm::vec3(1.0f, 1.0f, 1.0f); glm::vec3 scalePivot, rotationPivot; - Model model = { name, true }; + Model model = { name }; foreach (const FBXNode& subobject, object.children) { if (subobject.name == "Properties70") { foreach (const FBXNode& property, subobject.children) { @@ -592,22 +594,18 @@ FBXGeometry extractFBXGeometry(const FBXNode& node, const QVariantHash& mapping) } else if (property.properties.at(0) == "Lcl Scaling") { scale = glm::vec3(property.properties.at(4).value(), property.properties.at(5).value(), - property.properties.at(6).value()); - - } else if (property.properties.at(0) == "InheritType") { - model.inheritScale = property.properties.at(4) != 2; + property.properties.at(6).value()); } } } } } // see FBX documentation, http://download.autodesk.com/us/fbx/20112/FBX_SDK_HELP/index.html - model.withoutScale = glm::translate(translation) * glm::translate(rotationPivot) * - glm::mat4_cast(glm::quat(glm::radians(preRotation))) * - glm::mat4_cast(glm::quat(glm::radians(rotation))) * - glm::mat4_cast(glm::quat(glm::radians(postRotation))) * glm::translate(-rotationPivot); - model.withScale = model.withoutScale * glm::translate(scalePivot) * glm::scale(scale) * - glm::translate(-scalePivot); + model.preRotation = glm::translate(translation) * glm::translate(rotationPivot) * + glm::mat4_cast(glm::quat(glm::radians(preRotation))); + model.rotation = glm::quat(glm::radians(rotation)); + model.postRotation = glm::mat4_cast(glm::quat(glm::radians(postRotation))) * glm::translate(-rotationPivot) * + glm::translate(scalePivot) * glm::scale(scale) * glm::translate(-scalePivot); models.insert(object.properties.at(0).value(), model); } else if (object.name == "Texture") { @@ -718,11 +716,20 @@ FBXGeometry extractFBXGeometry(const FBXNode& node, const QVariantHash& mapping) // convert the models to joints foreach (qint64 modelID, modelIDs) { + const Model& model = models[modelID]; FBXJoint joint; - joint.parentIndex = modelIDs.indexOf(parentMap.value(modelID)); + joint.parentIndex = model.parentIndex; + joint.preRotation = model.preRotation; + joint.rotation = model.rotation; + joint.postRotation = model.postRotation; geometry.joints.append(joint); } + // find our special joints + geometry.leftEyeJointIndex = modelIDs.indexOf(jointEyeLeftID); + geometry.rightEyeJointIndex = modelIDs.indexOf(jointEyeRightID); + geometry.neckJointIndex = modelIDs.indexOf(jointNeckID); + QVariantHash springs = mapping.value("spring").toHash(); QVariant defaultSpring = springs.value("default"); for (QHash::iterator it = meshes.begin(); it != meshes.end(); it++) { @@ -754,7 +761,7 @@ FBXGeometry extractFBXGeometry(const FBXNode& node, const QVariantHash& mapping) // find the clusters with which the mesh is associated mesh.isEye = false; - QVector clusterIDs; + QVector clusterIDs; foreach (qint64 childID, childMap.values(it.key())) { foreach (qint64 clusterID, childMap.values(childID)) { if (!clusters.contains(clusterID)) { @@ -778,7 +785,7 @@ FBXGeometry extractFBXGeometry(const FBXNode& node, const QVariantHash& mapping) mesh.transform = jointTransform * glm::inverse(cluster.transformLink) * modelTransform; // extract translation component for pivot - glm::mat4 jointTransformScaled = geometry.offset * getGlobalTransform(parentMap, models, jointID, true); + glm::mat4 jointTransformScaled = geometry.offset * getGlobalTransform(parentMap, models, jointID); mesh.pivot = glm::vec3(jointTransformScaled[3][0], jointTransformScaled[3][1], jointTransformScaled[3][2]); } } @@ -798,6 +805,7 @@ FBXGeometry extractFBXGeometry(const FBXNode& node, const QVariantHash& mapping) qint64 clusterID = clusterIDs.at(i); const Cluster& cluster = clusters[clusterID]; qint64 jointID = childMap.value(clusterID); + for (int j = 0; j < cluster.indices.size(); j++) { int index = cluster.indices.at(j); glm::vec4& weights = mesh.clusterWeights[index]; @@ -858,7 +866,7 @@ FBXGeometry extractFBXGeometry(const FBXNode& node, const QVariantHash& mapping) } // extract translation component for neck pivot - glm::mat4 neckTransform = geometry.offset * getGlobalTransform(parentMap, models, jointNeckID, true); + glm::mat4 neckTransform = geometry.offset * getGlobalTransform(parentMap, models, jointNeckID); geometry.neckPivot = glm::vec3(neckTransform[3][0], neckTransform[3][1], neckTransform[3][2]); return geometry; diff --git a/interface/src/renderer/FBXReader.h b/interface/src/renderer/FBXReader.h index 3d8c3d0fa2..72033813fa 100644 --- a/interface/src/renderer/FBXReader.h +++ b/interface/src/renderer/FBXReader.h @@ -42,7 +42,9 @@ class FBXJoint { public: int parentIndex; + glm::mat4 preRotation; glm::quat rotation; + glm::mat4 postRotation; }; /// A single binding to a joint in an FBX document. @@ -96,6 +98,10 @@ public: glm::mat4 offset; + int leftEyeJointIndex; + int rightEyeJointIndex; + int neckJointIndex; + glm::vec3 neckPivot; }; diff --git a/interface/src/renderer/GeometryCache.cpp b/interface/src/renderer/GeometryCache.cpp index 926f226d6c..408d159198 100644 --- a/interface/src/renderer/GeometryCache.cpp +++ b/interface/src/renderer/GeometryCache.cpp @@ -363,18 +363,37 @@ void NetworkGeometry::maybeReadModelWithMapping() { glGenBuffers(1, &networkMesh.vertexBufferID); glBindBuffer(GL_ARRAY_BUFFER, networkMesh.vertexBufferID); + // if we don't need to do any blending or springing, then the positions/normals can be static if (mesh.blendshapes.isEmpty() && mesh.springiness == 0.0f) { glBufferData(GL_ARRAY_BUFFER, (mesh.vertices.size() + mesh.normals.size()) * sizeof(glm::vec3) + - mesh.texCoords.size() * sizeof(glm::vec2), NULL, GL_STATIC_DRAW); + mesh.texCoords.size() * sizeof(glm::vec2) + (mesh.clusterIndices.size() + + mesh.clusterWeights.size()) * sizeof(glm::vec4), NULL, GL_STATIC_DRAW); glBufferSubData(GL_ARRAY_BUFFER, 0, mesh.vertices.size() * sizeof(glm::vec3), mesh.vertices.constData()); glBufferSubData(GL_ARRAY_BUFFER, mesh.vertices.size() * sizeof(glm::vec3), mesh.normals.size() * sizeof(glm::vec3), mesh.normals.constData()); glBufferSubData(GL_ARRAY_BUFFER, (mesh.vertices.size() + mesh.normals.size()) * sizeof(glm::vec3), mesh.texCoords.size() * sizeof(glm::vec2), mesh.texCoords.constData()); + glBufferSubData(GL_ARRAY_BUFFER, (mesh.vertices.size() + mesh.normals.size()) * sizeof(glm::vec3) + + mesh.texCoords.size() * sizeof(glm::vec2), mesh.clusterIndices.size() * sizeof(glm::vec4), + mesh.clusterIndices.constData()); + glBufferSubData(GL_ARRAY_BUFFER, (mesh.vertices.size() + mesh.normals.size()) * sizeof(glm::vec3) + + mesh.texCoords.size() * sizeof(glm::vec2) + mesh.clusterIndices.size() * sizeof(glm::vec4), + mesh.clusterWeights.size() * sizeof(glm::vec4), mesh.clusterWeights.constData()); + + // if there's no springiness, then the cluster indices/weights can be static + } else if (mesh.springiness == 0.0f) { + glBufferData(GL_ARRAY_BUFFER, mesh.texCoords.size() * sizeof(glm::vec2) + (mesh.clusterIndices.size() + + mesh.clusterWeights.size()) * sizeof(glm::vec4), NULL, GL_STATIC_DRAW); + glBufferSubData(GL_ARRAY_BUFFER, 0, mesh.texCoords.size() * sizeof(glm::vec2), mesh.texCoords.constData()); + glBufferSubData(GL_ARRAY_BUFFER, mesh.texCoords.size() * sizeof(glm::vec2), + mesh.clusterIndices.size() * sizeof(glm::vec4), mesh.clusterIndices.constData()); + glBufferSubData(GL_ARRAY_BUFFER, mesh.texCoords.size() * sizeof(glm::vec2) + + mesh.clusterIndices.size() * sizeof(glm::vec4), mesh.clusterWeights.size() * sizeof(glm::vec4), + mesh.clusterWeights.constData()); } else { - glBufferData(GL_ARRAY_BUFFER, mesh.texCoords.size() * sizeof(glm::vec2), - mesh.texCoords.constData(), GL_STATIC_DRAW); + glBufferData(GL_ARRAY_BUFFER, mesh.texCoords.size() * sizeof(glm::vec2), NULL, GL_STATIC_DRAW); + glBufferSubData(GL_ARRAY_BUFFER, 0, mesh.texCoords.size() * sizeof(glm::vec2), mesh.texCoords.constData()); } glBindBuffer(GL_ARRAY_BUFFER, 0);