From f8d26f207d958c20356422f60bd0d65816533211 Mon Sep 17 00:00:00 2001 From: Andrzej Kapolka Date: Fri, 27 Sep 2013 17:57:18 -0700 Subject: [PATCH 1/6] Include the local transforms from the FBX file when rendering meshes. --- interface/src/avatar/BlendFace.cpp | 6 +++- interface/src/renderer/FBXReader.cpp | 42 ++++++++++++++++++++++++++-- interface/src/renderer/FBXReader.h | 1 + 3 files changed, 46 insertions(+), 3 deletions(-) diff --git a/interface/src/avatar/BlendFace.cpp b/interface/src/avatar/BlendFace.cpp index e531052726..3d221bc120 100644 --- a/interface/src/avatar/BlendFace.cpp +++ b/interface/src/avatar/BlendFace.cpp @@ -77,6 +77,9 @@ bool BlendFace::render(float alpha) { const FBXMesh& mesh = _geometry.meshes.at(i); int vertexCount = mesh.vertices.size(); + glPushMatrix(); + glMultMatrixf((const GLfloat*)&mesh.transform); + // apply eye rotation if appropriate if (mesh.isEye) { glPushMatrix(); @@ -144,8 +147,9 @@ bool BlendFace::render(float alpha) { glBindTexture(GL_TEXTURE_2D, 0); glDisable(GL_TEXTURE_2D); glDisableClientState(GL_TEXTURE_COORD_ARRAY); - glPopMatrix(); } + + glPopMatrix(); } glDisable(GL_NORMALIZE); diff --git a/interface/src/renderer/FBXReader.cpp b/interface/src/renderer/FBXReader.cpp index a9e8688388..74ca5b2fa3 100644 --- a/interface/src/renderer/FBXReader.cpp +++ b/interface/src/renderer/FBXReader.cpp @@ -12,6 +12,9 @@ #include #include +#include +#include + #include "FBXReader.h" using namespace std; @@ -282,9 +285,10 @@ public: FBXGeometry extractFBXGeometry(const FBXNode& node) { QHash meshes; QVector blendshapes; - QHash parentMap; + QMultiHash parentMap; QMultiHash childMap; QHash pivots; + QHash localTransforms; qint64 jointEyeLeftID = 0; qint64 jointEyeRightID = 0; @@ -403,13 +407,42 @@ FBXGeometry extractFBXGeometry(const FBXNode& node) { blendshapes.append(extracted); } - } else if (object.name == "Model" && object.properties.at(2) == "LimbNode") { + } else if (object.name == "Model") { if (object.properties.at(1).toByteArray().startsWith("jointEyeLeft")) { jointEyeLeftID = object.properties.at(0).value(); } else if (object.properties.at(1).toByteArray().startsWith("jointEyeRight")) { jointEyeRightID = object.properties.at(0).value(); } + glm::vec3 translation; + glm::vec3 rotation; + glm::vec3 scale = glm::vec3(1.0f, 1.0f, 1.0f); + foreach (const FBXNode& subobject, object.children) { + if (subobject.name == "Properties70") { + foreach (const FBXNode& property, subobject.children) { + if (property.name == "P") { + if (property.properties.at(0) == "Lcl Translation") { + translation = glm::vec3(property.properties.at(4).value(), + property.properties.at(5).value(), + property.properties.at(6).value()); + + } else if (property.properties.at(0) == "Lcl Rotation") { + rotation = glm::vec3(property.properties.at(4).value(), + property.properties.at(5).value(), + property.properties.at(6).value()); + + } else if (property.properties.at(0) == "Lcl Scaling") { + scale = glm::vec3(property.properties.at(4).value(), + property.properties.at(5).value(), + property.properties.at(6).value()); + } + } + } + } + } + localTransforms.insert(object.properties.at(0).value(), + glm::translate(translation) * glm::mat4_cast(glm::quat(glm::radians(rotation))) * glm::scale(scale)); + } else if (object.name == "Deformer" && object.properties.at(2) == "Cluster") { foreach (const FBXNode& subobject, object.children) { if (subobject.name == "TransformLink") { @@ -446,6 +479,11 @@ FBXGeometry extractFBXGeometry(const FBXNode& node) { for (QHash::iterator it = meshes.begin(); it != meshes.end(); it++) { FBXMesh& mesh = it.value(); + // accumulate local transforms + for (qint64 parentID = parentMap.value(it.key()); parentID != 0; parentID = parentMap.value(parentID)) { + mesh.transform = localTransforms.value(parentID) * mesh.transform; + } + // look for a limb pivot mesh.isEye = false; foreach (qint64 childID, childMap.values(it.key())) { diff --git a/interface/src/renderer/FBXReader.h b/interface/src/renderer/FBXReader.h index c212e0a850..05a0662e05 100644 --- a/interface/src/renderer/FBXReader.h +++ b/interface/src/renderer/FBXReader.h @@ -49,6 +49,7 @@ public: QVector texCoords; glm::vec3 pivot; + glm::mat4 transform; bool isEye; From 1592aac1776c7f60436897671f63a57e5c55de25 Mon Sep 17 00:00:00 2001 From: Andrzej Kapolka Date: Mon, 30 Sep 2013 11:24:41 -0700 Subject: [PATCH 2/6] Getting closer to the correct set of transforms. --- interface/src/avatar/BlendFace.cpp | 1 - interface/src/renderer/FBXReader.cpp | 74 +++++++++++++++++++++++----- 2 files changed, 63 insertions(+), 12 deletions(-) diff --git a/interface/src/avatar/BlendFace.cpp b/interface/src/avatar/BlendFace.cpp index 3d221bc120..3bb3237637 100644 --- a/interface/src/avatar/BlendFace.cpp +++ b/interface/src/avatar/BlendFace.cpp @@ -82,7 +82,6 @@ bool BlendFace::render(float alpha) { // apply eye rotation if appropriate if (mesh.isEye) { - glPushMatrix(); glTranslatef(mesh.pivot.x, mesh.pivot.y, mesh.pivot.z); glm::quat rotation = glm::inverse(orientation) * _owningHead->getEyeRotation(orientation * (mesh.pivot * scale + MODEL_TRANSLATION) + _owningHead->getPosition()); diff --git a/interface/src/renderer/FBXReader.cpp b/interface/src/renderer/FBXReader.cpp index 74ca5b2fa3..27c3eac259 100644 --- a/interface/src/renderer/FBXReader.cpp +++ b/interface/src/renderer/FBXReader.cpp @@ -210,6 +210,13 @@ QVector createVec2Vector(const QVector& doubleVector) { return values; } +glm::mat4 createMat4(const QVector& doubleVector) { + return glm::mat4(doubleVector.at(0), doubleVector.at(1), doubleVector.at(2), doubleVector.at(3), + doubleVector.at(4), doubleVector.at(5), doubleVector.at(6), doubleVector.at(7), + doubleVector.at(8), doubleVector.at(9), doubleVector.at(10), doubleVector.at(11), + doubleVector.at(12), doubleVector.at(13), doubleVector.at(14), doubleVector.at(15)); +} + const char* FACESHIFT_BLENDSHAPES[] = { "EyeBlink_L", "EyeBlink_R", @@ -289,6 +296,7 @@ FBXGeometry extractFBXGeometry(const FBXNode& node) { QMultiHash childMap; QHash pivots; QHash localTransforms; + QHash transformLinkMatrices; qint64 jointEyeLeftID = 0; qint64 jointEyeRightID = 0; @@ -404,19 +412,21 @@ FBXGeometry extractFBXGeometry(const FBXNode& node) { QByteArray name = object.properties.at(1).toByteArray(); static QHash blendshapeMap = createBlendshapeMap(); extracted.index = blendshapeMap.value(name.left(name.indexOf('\0'))); - + blendshapes.append(extracted); } } else if (object.name == "Model") { - if (object.properties.at(1).toByteArray().startsWith("jointEyeLeft")) { + QByteArray name = object.properties.at(1).toByteArray(); + if (name.startsWith("jointEyeLeft") || name.startsWith("EyeL")) { jointEyeLeftID = object.properties.at(0).value(); - } else if (object.properties.at(1).toByteArray().startsWith("jointEyeRight")) { + } else if (name.startsWith("jointEyeRight") || name.startsWith("EyeR")) { jointEyeRightID = object.properties.at(0).value(); } glm::vec3 translation; - glm::vec3 rotation; + glm::vec3 preRotation, rotation, postRotation; glm::vec3 scale = glm::vec3(1.0f, 1.0f, 1.0f); + glm::vec3 scalePivot, rotationPivot; foreach (const FBXNode& subobject, object.children) { if (subobject.name == "Properties70") { foreach (const FBXNode& property, subobject.children) { @@ -425,12 +435,32 @@ FBXGeometry extractFBXGeometry(const FBXNode& node) { translation = glm::vec3(property.properties.at(4).value(), property.properties.at(5).value(), property.properties.at(6).value()); - + + } else if (property.properties.at(0) == "RotationPivot") { + rotationPivot = glm::vec3(property.properties.at(4).value(), + property.properties.at(5).value(), + property.properties.at(6).value()); + + } else if (property.properties.at(0) == "PreRotation") { + preRotation = glm::vec3(property.properties.at(4).value(), + property.properties.at(5).value(), + property.properties.at(6).value()); + } else if (property.properties.at(0) == "Lcl Rotation") { rotation = glm::vec3(property.properties.at(4).value(), property.properties.at(5).value(), property.properties.at(6).value()); + + } else if (property.properties.at(0) == "PostRotation") { + postRotation = glm::vec3(property.properties.at(4).value(), + property.properties.at(5).value(), + property.properties.at(6).value()); + } else if (property.properties.at(0) == "ScalingPivot") { + scalePivot = glm::vec3(property.properties.at(4).value(), + property.properties.at(5).value(), + property.properties.at(6).value()); + } else if (property.properties.at(0) == "Lcl Scaling") { scale = glm::vec3(property.properties.at(4).value(), property.properties.at(5).value(), @@ -440,13 +470,19 @@ FBXGeometry extractFBXGeometry(const FBXNode& node) { } } } + // see FBX documentation, http://download.autodesk.com/us/fbx/20112/FBX_SDK_HELP/index.html localTransforms.insert(object.properties.at(0).value(), - glm::translate(translation) * glm::mat4_cast(glm::quat(glm::radians(rotation))) * glm::scale(scale)); + glm::translate(translation) * glm::translate(rotationPivot) * + glm::mat4_cast(glm::quat(glm::radians(preRotation))) * + glm::mat4_cast(glm::quat(glm::radians(rotation))) * + glm::mat4_cast(glm::quat(glm::radians(postRotation))) * glm::translate(-rotationPivot) * + glm::translate(scalePivot) * glm::scale(scale) * glm::translate(-scalePivot)); } else if (object.name == "Deformer" && object.properties.at(2) == "Cluster") { foreach (const FBXNode& subobject, object.children) { if (subobject.name == "TransformLink") { QVector values = subobject.properties.at(0).value >(); + transformLinkMatrices.insert(object.properties.at(0).value(), createMat4(values)); pivots.insert(object.properties.at(0).value(), glm::vec3(values.at(12), values.at(13), values.at(14))); // matrix translation component } @@ -488,11 +524,27 @@ FBXGeometry extractFBXGeometry(const FBXNode& node) { mesh.isEye = false; foreach (qint64 childID, childMap.values(it.key())) { qint64 clusterID = childMap.value(childID); - if (pivots.contains(clusterID)) { - mesh.pivot = pivots.value(clusterID); - qint64 jointID = childMap.value(clusterID); - if (jointID == jointEyeLeftID || jointID == jointEyeRightID) { - mesh.isEye = true; + if (!pivots.contains(clusterID)) { + continue; + } + mesh.pivot = pivots.value(clusterID); + qint64 jointID = childMap.value(clusterID); + if (jointID == jointEyeLeftID || jointID == jointEyeRightID) { + mesh.isEye = true; + } + + mesh.transform = glm::inverse(transformLinkMatrices.value(clusterID)) * mesh.transform; + + while (jointID != 0) { + mesh.transform = localTransforms.value(jointID) * mesh.transform; + + QList parentIDs = parentMap.values(jointID); + jointID = 0; + foreach (qint64 parentID, parentIDs) { + if (localTransforms.contains(parentID)) { + jointID = parentID; + break; + } } } } From d4203ff3d3040f78397060fc82774a85057c196c Mon Sep 17 00:00:00 2001 From: Andrzej Kapolka Date: Mon, 30 Sep 2013 12:01:27 -0700 Subject: [PATCH 3/6] Pivot fix. --- interface/src/avatar/BlendFace.cpp | 3 ++- interface/src/renderer/FBXReader.cpp | 12 ++++++------ 2 files changed, 8 insertions(+), 7 deletions(-) diff --git a/interface/src/avatar/BlendFace.cpp b/interface/src/avatar/BlendFace.cpp index 3bb3237637..c0d39fa360 100644 --- a/interface/src/avatar/BlendFace.cpp +++ b/interface/src/avatar/BlendFace.cpp @@ -78,7 +78,6 @@ bool BlendFace::render(float alpha) { int vertexCount = mesh.vertices.size(); glPushMatrix(); - glMultMatrixf((const GLfloat*)&mesh.transform); // apply eye rotation if appropriate if (mesh.isEye) { @@ -100,6 +99,8 @@ bool BlendFace::render(float alpha) { _eyeProgram.bind(); } + glMultMatrixf((const GLfloat*)&mesh.transform); + // all meshes after the first are white if (i == 1) { glColor4f(1.0f, 1.0f, 1.0f, alpha); diff --git a/interface/src/renderer/FBXReader.cpp b/interface/src/renderer/FBXReader.cpp index 27c3eac259..3677a386ef 100644 --- a/interface/src/renderer/FBXReader.cpp +++ b/interface/src/renderer/FBXReader.cpp @@ -294,7 +294,6 @@ FBXGeometry extractFBXGeometry(const FBXNode& node) { QVector blendshapes; QMultiHash parentMap; QMultiHash childMap; - QHash pivots; QHash localTransforms; QHash transformLinkMatrices; qint64 jointEyeLeftID = 0; @@ -483,8 +482,6 @@ FBXGeometry extractFBXGeometry(const FBXNode& node) { if (subobject.name == "TransformLink") { QVector values = subobject.properties.at(0).value >(); transformLinkMatrices.insert(object.properties.at(0).value(), createMat4(values)); - pivots.insert(object.properties.at(0).value(), - glm::vec3(values.at(12), values.at(13), values.at(14))); // matrix translation component } } } @@ -524,10 +521,9 @@ FBXGeometry extractFBXGeometry(const FBXNode& node) { mesh.isEye = false; foreach (qint64 childID, childMap.values(it.key())) { qint64 clusterID = childMap.value(childID); - if (!pivots.contains(clusterID)) { + if (!transformLinkMatrices.contains(clusterID)) { continue; } - mesh.pivot = pivots.value(clusterID); qint64 jointID = childMap.value(clusterID); if (jointID == jointEyeLeftID || jointID == jointEyeRightID) { mesh.isEye = true; @@ -535,8 +531,9 @@ FBXGeometry extractFBXGeometry(const FBXNode& node) { mesh.transform = glm::inverse(transformLinkMatrices.value(clusterID)) * mesh.transform; + glm::mat4 jointTransform; while (jointID != 0) { - mesh.transform = localTransforms.value(jointID) * mesh.transform; + jointTransform = localTransforms.value(jointID) * jointTransform; QList parentIDs = parentMap.values(jointID); jointID = 0; @@ -547,6 +544,9 @@ FBXGeometry extractFBXGeometry(const FBXNode& node) { } } } + + mesh.transform = jointTransform * mesh.transform; + mesh.pivot = glm::vec3(jointTransform[3][0], jointTransform[3][1], jointTransform[3][2]); } if (mesh.blendshapes.size() > mostBlendshapes) { From d4c4cbc9813c287f5d520925d2f888e2cd15ff65 Mon Sep 17 00:00:00 2001 From: Andrzej Kapolka Date: Mon, 30 Sep 2013 13:15:59 -0700 Subject: [PATCH 4/6] Get the neck joint and position blend face relative to that. --- interface/src/avatar/BlendFace.cpp | 5 ++- interface/src/renderer/FBXReader.cpp | 49 +++++++++++++++++++--------- interface/src/renderer/FBXReader.h | 2 ++ 3 files changed, 39 insertions(+), 17 deletions(-) diff --git a/interface/src/avatar/BlendFace.cpp b/interface/src/avatar/BlendFace.cpp index c0d39fa360..c7e1f82bf8 100644 --- a/interface/src/avatar/BlendFace.cpp +++ b/interface/src/avatar/BlendFace.cpp @@ -43,7 +43,7 @@ void BlendFace::init() { } } -const glm::vec3 MODEL_TRANSLATION(0.0f, -0.025f, -0.025f); // temporary fudge factor +const glm::vec3 MODEL_TRANSLATION(0.0f, -0.07f, -0.025f); // temporary fudge factor const float MODEL_SCALE = 0.0006f; bool BlendFace::render(float alpha) { @@ -61,6 +61,9 @@ bool BlendFace::render(float alpha) { -_owningHead->getScale() * MODEL_SCALE); glScalef(scale.x, scale.y, scale.z); + + glTranslatef(-_geometry.neckPivot.x, -_geometry.neckPivot.y, -_geometry.neckPivot.z); + glEnableClientState(GL_VERTEX_ARRAY); glEnableClientState(GL_NORMAL_ARRAY); diff --git a/interface/src/renderer/FBXReader.cpp b/interface/src/renderer/FBXReader.cpp index 3677a386ef..bc77a1c8b4 100644 --- a/interface/src/renderer/FBXReader.cpp +++ b/interface/src/renderer/FBXReader.cpp @@ -282,6 +282,26 @@ QHash createBlendshapeMap() { } } +glm::mat4 getGlobalTransform( + const QMultiHash& parentMap, const QHash& localTransforms, qint64 nodeID) { + + glm::mat4 globalTransform; + while (nodeID != 0) { + globalTransform = localTransforms.value(nodeID) * globalTransform; + + QList parentIDs = parentMap.values(nodeID); + nodeID = 0; + foreach (qint64 parentID, parentIDs) { + if (localTransforms.contains(parentID)) { + nodeID = parentID; + break; + } + } + } + + return globalTransform; +} + class ExtractedBlendshape { public: qint64 id; @@ -298,6 +318,7 @@ FBXGeometry extractFBXGeometry(const FBXNode& node) { QHash transformLinkMatrices; qint64 jointEyeLeftID = 0; qint64 jointEyeRightID = 0; + qint64 jointNeckID = 0; foreach (const FBXNode& child, node.children) { if (child.name == "Objects") { @@ -421,6 +442,9 @@ FBXGeometry extractFBXGeometry(const FBXNode& node) { } else if (name.startsWith("jointEyeRight") || name.startsWith("EyeR")) { jointEyeRightID = object.properties.at(0).value(); + + } else if (name.startsWith("jointNeck") || name.startsWith("NeckRot")) { + jointNeckID = object.properties.at(0).value(); } glm::vec3 translation; glm::vec3 preRotation, rotation, postRotation; @@ -529,23 +553,12 @@ FBXGeometry extractFBXGeometry(const FBXNode& node) { mesh.isEye = true; } - mesh.transform = glm::inverse(transformLinkMatrices.value(clusterID)) * mesh.transform; + // see http://stackoverflow.com/questions/13566608/loading-skinning-information-from-fbx for a discussion + // of skinning information in FBX + glm::mat4 jointTransform = getGlobalTransform(parentMap, localTransforms, jointID); + mesh.transform = jointTransform * glm::inverse(transformLinkMatrices.value(clusterID)) * mesh.transform; - glm::mat4 jointTransform; - while (jointID != 0) { - jointTransform = localTransforms.value(jointID) * jointTransform; - - QList parentIDs = parentMap.values(jointID); - jointID = 0; - foreach (qint64 parentID, parentIDs) { - if (localTransforms.contains(parentID)) { - jointID = parentID; - break; - } - } - } - - mesh.transform = jointTransform * mesh.transform; + // extract translation component for pivot mesh.pivot = glm::vec3(jointTransform[3][0], jointTransform[3][1], jointTransform[3][2]); } @@ -558,6 +571,10 @@ FBXGeometry extractFBXGeometry(const FBXNode& node) { } } + // extract translation component for neck pivot + glm::mat4 neckTransform = getGlobalTransform(parentMap, localTransforms, jointNeckID); + geometry.neckPivot = glm::vec3(neckTransform[3][0], neckTransform[3][1], neckTransform[3][2]); + return geometry; } diff --git a/interface/src/renderer/FBXReader.h b/interface/src/renderer/FBXReader.h index 05a0662e05..38148ced0b 100644 --- a/interface/src/renderer/FBXReader.h +++ b/interface/src/renderer/FBXReader.h @@ -61,6 +61,8 @@ class FBXGeometry { public: QVector meshes; + + glm::vec3 neckPivot; }; /// Parses the input from the supplied data as an FBX file. From 40d195595f763f4f5115bfee2476f7b1b8c8b35e Mon Sep 17 00:00:00 2001 From: Andrzej Kapolka Date: Mon, 30 Sep 2013 13:20:28 -0700 Subject: [PATCH 5/6] Include the neck pivot when we get the eye locations. --- interface/src/avatar/BlendFace.cpp | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/interface/src/avatar/BlendFace.cpp b/interface/src/avatar/BlendFace.cpp index c7e1f82bf8..b7d0e5d55b 100644 --- a/interface/src/avatar/BlendFace.cpp +++ b/interface/src/avatar/BlendFace.cpp @@ -178,7 +178,8 @@ void BlendFace::getEyePositions(glm::vec3& firstEyePosition, glm::vec3& secondEy foreach (const FBXMesh& mesh, _geometry.meshes) { if (mesh.isEye) { - glm::vec3 position = orientation * (mesh.pivot * scale + MODEL_TRANSLATION) + _owningHead->getPosition(); + glm::vec3 position = orientation * ((mesh.pivot - _geometry.neckPivot) * scale + MODEL_TRANSLATION) + + _owningHead->getPosition(); if (foundFirst) { secondEyePosition = position; return; From 71d68796f34cd5c65c42358eee8dbd768bbaa73e Mon Sep 17 00:00:00 2001 From: Andrzej Kapolka Date: Mon, 30 Sep 2013 13:27:10 -0700 Subject: [PATCH 6/6] Remove whitespace. --- interface/src/avatar/BlendFace.cpp | 1 - 1 file changed, 1 deletion(-) diff --git a/interface/src/avatar/BlendFace.cpp b/interface/src/avatar/BlendFace.cpp index b7d0e5d55b..0bab8c66b3 100644 --- a/interface/src/avatar/BlendFace.cpp +++ b/interface/src/avatar/BlendFace.cpp @@ -61,7 +61,6 @@ bool BlendFace::render(float alpha) { -_owningHead->getScale() * MODEL_SCALE); glScalef(scale.x, scale.y, scale.z); - glTranslatef(-_geometry.neckPivot.x, -_geometry.neckPivot.y, -_geometry.neckPivot.z); glEnableClientState(GL_VERTEX_ARRAY);