From ad6720240fe31f888f6b11a46e5e2c3c68923dd8 Mon Sep 17 00:00:00 2001 From: sabrina-shanman Date: Fri, 18 Oct 2019 11:12:08 -0700 Subject: [PATCH 1/2] Introduce hfm::Mesh.clusterWeightsPerVertex --- libraries/fbx/src/FBXSerializer.cpp | 1 + libraries/hfm/src/hfm/HFM.h | 1 + libraries/hfm/src/hfm/HFMModelMath.cpp | 1 + libraries/hfm/src/hfm/HFMModelMath.h | 7 ++++--- .../model-baker/src/model-baker/BuildGraphicsMeshTask.cpp | 2 +- .../src/model-baker/CollectShapeVerticesTask.cpp | 2 +- 6 files changed, 9 insertions(+), 5 deletions(-) diff --git a/libraries/fbx/src/FBXSerializer.cpp b/libraries/fbx/src/FBXSerializer.cpp index 4b3311c95a..2c03c3c3ae 100644 --- a/libraries/fbx/src/FBXSerializer.cpp +++ b/libraries/fbx/src/FBXSerializer.cpp @@ -1671,6 +1671,7 @@ HFMModel* FBXSerializer::extractHFMModel(const hifi::VariantHash& mapping, const } mesh.clusterIndices = std::move(reweightedDeformers.indices); mesh.clusterWeights = std::move(reweightedDeformers.weights); + mesh.clusterWeightsPerVertex = reweightedDeformers.weightsPerVertex; } // Store the model's dynamic transform, and put its ID in the shapes diff --git a/libraries/hfm/src/hfm/HFM.h b/libraries/hfm/src/hfm/HFM.h index b2d8147ac6..3d7f33383d 100644 --- a/libraries/hfm/src/hfm/HFM.h +++ b/libraries/hfm/src/hfm/HFM.h @@ -250,6 +250,7 @@ public: // Skinning cluster attributes std::vector clusterIndices; std::vector clusterWeights; + uint16_t clusterWeightsPerVertex { 0 }; // Blendshape attributes QVector blendshapes; diff --git a/libraries/hfm/src/hfm/HFMModelMath.cpp b/libraries/hfm/src/hfm/HFMModelMath.cpp index 09083ab4cc..93687b08b0 100644 --- a/libraries/hfm/src/hfm/HFMModelMath.cpp +++ b/libraries/hfm/src/hfm/HFMModelMath.cpp @@ -75,6 +75,7 @@ ReweightedDeformers getReweightedDeformers(const size_t numMeshVertices, const s size_t numClusterIndices = numMeshVertices * weightsPerVertex; reweightedDeformers.indices.resize(numClusterIndices, (uint16_t)(skinClusters.size() - 1)); reweightedDeformers.weights.resize(numClusterIndices, 0); + reweightedDeformers.weightsPerVertex = weightsPerVertex; std::vector weightAccumulators; weightAccumulators.resize(numClusterIndices, 0.0f); diff --git a/libraries/hfm/src/hfm/HFMModelMath.h b/libraries/hfm/src/hfm/HFMModelMath.h index 9420c96f08..b80adad3d0 100644 --- a/libraries/hfm/src/hfm/HFMModelMath.h +++ b/libraries/hfm/src/hfm/HFMModelMath.h @@ -25,16 +25,17 @@ void calculateExtentsForShape(hfm::Shape& shape, const std::vector& m void calculateExtentsForModel(Extents& modelExtents, const std::vector& shapes); -const uint16_t NUM_SKINNING_WEIGHTS_PER_VERTEX = 4; - class ReweightedDeformers { public: std::vector indices; std::vector weights; + uint16_t weightsPerVertex { 0 }; bool trimmedToMatch { false }; }; -ReweightedDeformers getReweightedDeformers(const size_t numMeshVertices, const std::vector skinClusters, const uint16_t weightsPerVertex = NUM_SKINNING_WEIGHTS_PER_VERTEX); +const uint16_t DEFAULT_SKINNING_WEIGHTS_PER_VERTEX = 4; + +ReweightedDeformers getReweightedDeformers(const size_t numMeshVertices, const std::vector skinClusters, const uint16_t weightsPerVertex = DEFAULT_SKINNING_WEIGHTS_PER_VERTEX); }; #endif // #define hifi_hfm_ModelMath_h diff --git a/libraries/model-baker/src/model-baker/BuildGraphicsMeshTask.cpp b/libraries/model-baker/src/model-baker/BuildGraphicsMeshTask.cpp index 6af0f9edf7..66429ed2c4 100644 --- a/libraries/model-baker/src/model-baker/BuildGraphicsMeshTask.cpp +++ b/libraries/model-baker/src/model-baker/BuildGraphicsMeshTask.cpp @@ -92,7 +92,7 @@ void buildGraphicsMesh(const hfm::Mesh& hfmMesh, graphics::MeshPointer& graphics const auto clusterWeightElement = gpu::Element(gpu::VEC4, gpu::NUINT16, gpu::XYZW); // Record cluster sizes - const size_t numVertClusters = hfmMesh.clusterIndices.size() / hfm::NUM_SKINNING_WEIGHTS_PER_VERTEX; + const size_t numVertClusters = hfmMesh.clusterWeightsPerVertex == 0 ? 0 : hfmMesh.clusterIndices.size() / hfmMesh.clusterWeightsPerVertex; const size_t clusterIndicesSize = numVertClusters * clusterIndiceElement.getSize(); const size_t clusterWeightsSize = numVertClusters * clusterWeightElement.getSize(); diff --git a/libraries/model-baker/src/model-baker/CollectShapeVerticesTask.cpp b/libraries/model-baker/src/model-baker/CollectShapeVerticesTask.cpp index 5ede25a42c..13bc75ced9 100644 --- a/libraries/model-baker/src/model-baker/CollectShapeVerticesTask.cpp +++ b/libraries/model-baker/src/model-baker/CollectShapeVerticesTask.cpp @@ -62,7 +62,7 @@ void CollectShapeVerticesTask::run(const baker::BakeContextPointer& context, con const auto& vertices = mesh.vertices; const glm::mat4 meshToJoint = cluster.inverseBindMatrix; - const uint16_t weightsPerVertex = hfm::NUM_SKINNING_WEIGHTS_PER_VERTEX; + const uint16_t weightsPerVertex = mesh.clusterWeightsPerVertex; if (weightsPerVertex == 0) { for (int vertexIndex = 0; vertexIndex < (int)vertices.size(); ++vertexIndex) { const glm::mat4 vertexTransform = meshToJoint * glm::translate(vertices[vertexIndex]); From ca164375f1b8d41ae5c41b21b500585602a82ffe Mon Sep 17 00:00:00 2001 From: sabrina-shanman Date: Fri, 18 Oct 2019 11:56:43 -0700 Subject: [PATCH 2/2] New skinning for GLTF --- libraries/animation/src/AnimSkeleton.cpp | 8 +- libraries/fbx/src/GLTFSerializer.cpp | 176 ++++++++++++----------- libraries/fbx/src/GLTFSerializer.h | 2 +- 3 files changed, 97 insertions(+), 89 deletions(-) diff --git a/libraries/animation/src/AnimSkeleton.cpp b/libraries/animation/src/AnimSkeleton.cpp index bae1fb5b69..0a7881abd8 100644 --- a/libraries/animation/src/AnimSkeleton.cpp +++ b/libraries/animation/src/AnimSkeleton.cpp @@ -20,13 +20,7 @@ AnimSkeleton::AnimSkeleton(const HFMModel& hfmModel) { _geometryOffset = hfmModel.offset; - // convert to std::vector of joints - std::vector joints; - joints.reserve(hfmModel.joints.size()); - for (auto& joint : hfmModel.joints) { - joints.push_back(joint); - } - buildSkeletonFromJoints(joints, hfmModel.jointRotationOffsets); + buildSkeletonFromJoints(hfmModel.joints, hfmModel.jointRotationOffsets); // we make a copy of the inverseBindMatrices in order to prevent mutating the model bind pose // when we are dealing with a joint offset in the model diff --git a/libraries/fbx/src/GLTFSerializer.cpp b/libraries/fbx/src/GLTFSerializer.cpp index da48c3d2e3..ea31f74312 100755 --- a/libraries/fbx/src/GLTFSerializer.cpp +++ b/libraries/fbx/src/GLTFSerializer.cpp @@ -1014,6 +1014,9 @@ bool GLTFSerializer::buildGeometry(HFMModel& hfmModel, const hifi::VariantHash& const auto& parentJoint = hfmModel.joints[(size_t)joint.parentIndex]; joint.transform = parentJoint.transform * joint.transform; joint.globalTransform = joint.globalTransform * parentJoint.globalTransform; + } else { + joint.transform = hfmModel.offset * joint.transform; + joint.globalTransform = hfmModel.offset * joint.globalTransform; } joint.name = node.name; @@ -1034,6 +1037,9 @@ bool GLTFSerializer::buildGeometry(HFMModel& hfmModel, const hifi::VariantHash& std::vector globalBindTransforms; jointInverseBindTransforms.resize(numNodes); globalBindTransforms.resize(numNodes); + // Lookup between the GLTF mesh and the skin + std::vector gltfMeshToSkin; + gltfMeshToSkin.resize(_file.meshes.size(), -1); hfmModel.hasSkeletonJoints = !_file.skins.isEmpty(); if (hfmModel.hasSkeletonJoints) { @@ -1042,7 +1048,7 @@ bool GLTFSerializer::buildGeometry(HFMModel& hfmModel, const hifi::VariantHash& for (int jointIndex = 0; jointIndex < numNodes; ++jointIndex) { int nodeIndex = jointIndex; - auto joint = hfmModel.joints[jointIndex]; + auto& joint = hfmModel.joints[jointIndex]; for (int s = 0; s < _file.skins.size(); ++s) { const auto& skin = _file.skins[s]; @@ -1068,7 +1074,41 @@ bool GLTFSerializer::buildGeometry(HFMModel& hfmModel, const hifi::VariantHash& glm::vec3 bindTranslation = extractTranslation(hfmModel.offset * glm::inverse(jointInverseBindTransforms[jointIndex])); hfmModel.bindExtents.addPoint(bindTranslation); } - hfmModel.joints[jointIndex] = joint; + } + + std::vector skinToRootJoint; + skinToRootJoint.resize(_file.skins.size(), 0); + for (int jointIndex = 0; jointIndex < numNodes; ++jointIndex) { + const auto& node = _file.nodes[jointIndex]; + if (node.skin != -1) { + skinToRootJoint[node.skin] = jointIndex; + if (node.mesh != -1) { + gltfMeshToSkin[node.mesh] = node.skin; + } + } + } + + for (int skinIndex = 0; skinIndex < _file.skins.size(); ++skinIndex) { + const auto& skin = _file.skins[skinIndex]; + hfmModel.skinDeformers.emplace_back(); + auto& skinDeformer = hfmModel.skinDeformers.back(); + + // Add the nodes being referred to for skinning + for (int skinJointIndex : skin.joints) { + hfm::Cluster cluster; + cluster.jointIndex = skinJointIndex; + cluster.inverseBindMatrix = jointInverseBindTransforms[skinJointIndex]; + cluster.inverseBindTransform = Transform(cluster.inverseBindMatrix); + skinDeformer.clusters.push_back(cluster); + } + + // Always append a cluster referring to the root joint at the end + int rootJointIndex = skinToRootJoint[skinIndex]; + hfm::Cluster root; + root.jointIndex = rootJointIndex; + root.inverseBindMatrix = jointInverseBindTransforms[root.jointIndex]; + root.inverseBindTransform = Transform(root.inverseBindMatrix); + skinDeformer.clusters.push_back(root); } } @@ -1095,30 +1135,6 @@ bool GLTFSerializer::buildGeometry(HFMModel& hfmModel, const hifi::VariantHash& templateShapePerPrimPerGLTFMesh.emplace_back(); std::vector& templateShapePerPrim = templateShapePerPrimPerGLTFMesh.back(); - // TODO: Rewrite GLTF skinning definition - if (!hfmModel.hasSkeletonJoints) { - HFMCluster cluster; -#if 0 - cluster.jointIndex = nodeIndex; -#endif - cluster.inverseBindMatrix = glm::mat4(); - cluster.inverseBindTransform = Transform(cluster.inverseBindMatrix); - meshPtr->clusters.append(cluster); - } else { // skinned model - for (int j = 0; j < numNodes; ++j) { - HFMCluster cluster; - cluster.jointIndex = j; - cluster.inverseBindMatrix = jointInverseBindTransforms[j]; - cluster.inverseBindTransform = Transform(cluster.inverseBindMatrix); - meshPtr->clusters.append(cluster); - } - } - HFMCluster root; - root.jointIndex = 0; - root.inverseBindMatrix = jointInverseBindTransforms[root.jointIndex]; - root.inverseBindTransform = Transform(root.inverseBindMatrix); - meshPtr->clusters.append(root); - QSet primitiveAttributes; if (!gltfMesh.primitives.empty()) { for (const auto& attribute : gltfMesh.primitives[0].attributes.values.keys()) { @@ -1333,35 +1349,7 @@ bool GLTFSerializer::buildGeometry(HFMModel& hfmModel, const hifi::VariantHash& } } - if (joints.size() == partVerticesCount * jointStride) { - for (int n = 0; n < joints.size(); n += jointStride) { - clusterJoints.push_back(joints[n]); - if (jointStride > 1) { - clusterJoints.push_back(joints[n + 1]); - if (jointStride > 2) { - clusterJoints.push_back(joints[n + 2]); - if (jointStride > 3) { - clusterJoints.push_back(joints[n + 3]); - } else { - clusterJoints.push_back(0); - } - } else { - clusterJoints.push_back(0); - clusterJoints.push_back(0); - } - } else { - clusterJoints.push_back(0); - clusterJoints.push_back(0); - clusterJoints.push_back(0); - } - } - } else if (primitiveAttributes.contains("JOINTS_0")) { - for (int i = 0; i < partVerticesCount; ++i) { - for (int j = 0; j < 4; ++j) { - clusterJoints.push_back(0); - } - } - } + const int WEIGHTS_PER_VERTEX = 4; if (weights.size() == partVerticesCount * weightStride) { for (int n = 0; n < weights.size(); n += weightStride) { @@ -1388,40 +1376,65 @@ bool GLTFSerializer::buildGeometry(HFMModel& hfmModel, const hifi::VariantHash& } else if (primitiveAttributes.contains("WEIGHTS_0")) { for (int i = 0; i < partVerticesCount; ++i) { clusterWeights.push_back(1.0f); - for (int j = 0; j < 4; ++j) { + for (int j = 0; j < WEIGHTS_PER_VERTEX; ++j) { clusterWeights.push_back(0.0f); } } } - // Build weights (adapted from FBXSerializer.cpp) - if (hfmModel.hasSkeletonJoints) { - const int WEIGHTS_PER_VERTEX = 4; - const float ALMOST_HALF = 0.499f; - int numVertices = mesh.vertices.size() - prevMeshVerticesCount; - - // Append new cluster indices and weights for this mesh part - size_t prevMeshClusterWeightCount = mesh.clusterWeights.size(); - for (int i = 0; i < numVertices * WEIGHTS_PER_VERTEX; ++i) { - mesh.clusterIndices.push_back(mesh.clusters.size() - 1); - mesh.clusterWeights.push_back(0); + // Compress floating point weights to uint16_t for graphics runtime + // TODO: If the GLTF skinning weights are already in integer format, we should just copy the data + if (!clusterWeights.empty()) { + size_t numWeights = 4 * (mesh.vertices.size() - (uint32_t)prevMeshVerticesCount); + size_t newWeightsStart = mesh.clusterWeights.size(); + size_t newWeightsEnd = newWeightsStart + numWeights; + mesh.clusterWeights.reserve(newWeightsEnd); + for (int weightIndex = 0; weightIndex < clusterWeights.size(); ++weightIndex) { + // Per the GLTF specification + uint16_t weight = std::round(clusterWeights[weightIndex] * 65535.0); + mesh.clusterWeights.push_back(weight); } + } - // normalize and compress to 16-bits - for (int i = 0; i < numVertices; ++i) { - int j = i * WEIGHTS_PER_VERTEX; - - float totalWeight = 0.0f; - for (int k = j; k < j + WEIGHTS_PER_VERTEX; ++k) { - totalWeight += clusterWeights[k]; - } - if (totalWeight > 0.0f) { - float weightScalingFactor = (float)(UINT16_MAX) / totalWeight; - for (int k = j; k < j + WEIGHTS_PER_VERTEX; ++k) { - mesh.clusterWeights[prevMeshClusterWeightCount + k] = (uint16_t)(weightScalingFactor * clusterWeights[k] + ALMOST_HALF); + if (joints.size() == partVerticesCount * jointStride) { + for (int n = 0; n < joints.size(); n += jointStride) { + mesh.clusterIndices.push_back(joints[n]); + if (jointStride > 1) { + mesh.clusterIndices.push_back(joints[n + 1]); + if (jointStride > 2) { + mesh.clusterIndices.push_back(joints[n + 2]); + if (jointStride > 3) { + mesh.clusterIndices.push_back(joints[n + 3]); + } else { + mesh.clusterIndices.push_back(0); + } + } else { + mesh.clusterIndices.push_back(0); + mesh.clusterIndices.push_back(0); } } else { - mesh.clusterWeights[prevMeshClusterWeightCount + j] = (uint16_t)((float)(UINT16_MAX) + ALMOST_HALF); + mesh.clusterIndices.push_back(0); + mesh.clusterIndices.push_back(0); + mesh.clusterIndices.push_back(0); + } + } + } else if (primitiveAttributes.contains("JOINTS_0")) { + for (int i = 0; i < partVerticesCount; ++i) { + for (int j = 0; j < 4; ++j) { + mesh.clusterIndices.push_back(0); + } + } + } + + if (!mesh.clusterIndices.empty()) { + int skinIndex = gltfMeshToSkin[gltfMeshIndex]; + if (skinIndex != -1) { + const auto& deformer = hfmModel.skinDeformers[(size_t)skinIndex]; + std::vector oldToNew; + oldToNew.resize(_file.nodes.size()); + for (uint16_t clusterIndex = 0; clusterIndex < deformer.clusters.size() - 1; ++clusterIndex) { + const auto& cluster = deformer.clusters[clusterIndex]; + oldToNew[(size_t)cluster.jointIndex] = clusterIndex; } } } @@ -1523,8 +1536,9 @@ bool GLTFSerializer::buildGeometry(HFMModel& hfmModel, const hifi::VariantHash& const auto& templateShape = templateShapePerPrim[primIndex]; hfmModel.shapes.push_back(templateShape); auto& hfmShape = hfmModel.shapes.back(); - // Everything else is already defined (mesh, meshPart, material), so just define the new transform + // Everything else is already defined (mesh, meshPart, material), so just define the new transform and deformer if present hfmShape.joint = nodeIndex; + hfmShape.skinDeformer = node.skin != -1 ? node.skin : hfm::UNDEFINED_KEY; } } diff --git a/libraries/fbx/src/GLTFSerializer.h b/libraries/fbx/src/GLTFSerializer.h index 78dc9b9a37..edecde6985 100755 --- a/libraries/fbx/src/GLTFSerializer.h +++ b/libraries/fbx/src/GLTFSerializer.h @@ -46,7 +46,7 @@ struct GLTFNode { QVector scale; QVector matrix; glm::mat4 transform; - int skin; + int skin { -1 }; QVector skeletons; QString jointName; QMap defined;