diff --git a/libraries/fbx/src/FBXSerializer.cpp b/libraries/fbx/src/FBXSerializer.cpp index 4ce12d30e4..e6b4a62b51 100644 --- a/libraries/fbx/src/FBXSerializer.cpp +++ b/libraries/fbx/src/FBXSerializer.cpp @@ -1480,7 +1480,7 @@ HFMModel* FBXSerializer::extractHFMModel(const hifi::VariantHash& mapping, const // of skinning information in FBX QString jointID = _connectionChildMap.value(clusterID); hfmCluster.jointIndex = modelIDs.indexOf(jointID); - if (hfmCluster.jointIndex == HFMCluster::INVALID_JOINT_INDEX) { + if (hfmCluster.jointIndex == hfm::Cluster::INVALID_JOINT_INDEX) { qCDebug(modelformat) << "Joint not in model list: " << jointID; hfmCluster.jointIndex = 0; } @@ -1514,7 +1514,7 @@ HFMModel* FBXSerializer::extractHFMModel(const hifi::VariantHash& mapping, const { HFMCluster cluster; cluster.jointIndex = modelIDs.indexOf(modelID); - if (cluster.jointIndex == HFMCluster::INVALID_JOINT_INDEX) { + if (cluster.jointIndex == hfm::Cluster::INVALID_JOINT_INDEX) { qCDebug(modelformat) << "Model not in model list: " << modelID; cluster.jointIndex = 0; } diff --git a/libraries/hfm/src/hfm/HFM.cpp b/libraries/hfm/src/hfm/HFM.cpp index 3a61ebb6b9..f68af2b1ce 100644 --- a/libraries/hfm/src/hfm/HFM.cpp +++ b/libraries/hfm/src/hfm/HFM.cpp @@ -175,7 +175,7 @@ void HFMModel::computeKdops() { // NOTE: points are in joint-frame ShapeVertices& points = shapeVertices.at(i); - glm::quat rotOffset = jointRotationOffsets.contains((int)i) ? glm::inverse(jointRotationOffsets[i]) : quat(); + glm::quat rotOffset = jointRotationOffsets.contains((int)i) ? glm::inverse(jointRotationOffsets[(int)i]) : quat(); if (points.size() > 0) { // compute average point glm::vec3 avgPoint = glm::vec3(0.0f); diff --git a/libraries/hfm/src/hfm/HFM.h b/libraries/hfm/src/hfm/HFM.h index 1a3a5ecdb2..27add7cb98 100644 --- a/libraries/hfm/src/hfm/HFM.h +++ b/libraries/hfm/src/hfm/HFM.h @@ -66,6 +66,8 @@ static const int DRACO_ATTRIBUTE_ORIGINAL_INDEX = DRACO_BEGIN_CUSTOM_HIFI_ATTRIB // High Fidelity Model namespace namespace hfm { +static const uint32_t UNDEFINED_KEY = (uint32_t)-1; + /// A single blendshape. class Blendshape { public: @@ -122,9 +124,9 @@ public: /// A single binding to a joint. class Cluster { public: - static const uint32_t INVALID_JOINT_INDEX{ (uint32_t)-1 }; - uint32_t jointIndex{ INVALID_JOINT_INDEX }; - glm::mat4 inverseBindMatrix{ glm::mat4{ 1.0 } }; + static const uint32_t INVALID_JOINT_INDEX { (uint32_t)-1 }; + uint32_t jointIndex { INVALID_JOINT_INDEX }; + glm::mat4 inverseBindMatrix; Transform inverseBindTransform; }; @@ -303,7 +305,7 @@ public: class DynamicTransform { public: - std::vector deformers; + std::vector deformers; std::vector clusters; // affect the deformer of the same index std::vector blendshapes; // There are also the meshExtents and modelTransform, which for now are left in hfm::Mesh @@ -312,8 +314,6 @@ public: // The lightweight model part description. class Shape { public: - const static uint32_t UNDEFINED_KEY { (uint32_t)-1 }; - uint32_t mesh { UNDEFINED_KEY }; uint32_t meshPart { UNDEFINED_KEY }; uint32_t material { UNDEFINED_KEY }; diff --git a/libraries/model-baker/src/model-baker/Baker.cpp b/libraries/model-baker/src/model-baker/Baker.cpp index 3dab7f7241..1a68d3508d 100644 --- a/libraries/model-baker/src/model-baker/Baker.cpp +++ b/libraries/model-baker/src/model-baker/Baker.cpp @@ -27,7 +27,7 @@ namespace baker { class GetModelPartsTask { public: using Input = hfm::Model::Pointer; - using Output = VaryingSet5, hifi::URL, baker::MeshIndicesToModelNames, baker::BlendshapesPerMesh, std::vector>; + using Output = VaryingSet8, hifi::URL, baker::MeshIndicesToModelNames, baker::BlendshapesPerMesh, std::vector, std::vector, std::vector, std::vector>; using JobModel = Job::ModelIO; void run(const BakeContextPointer& context, const Input& input, Output& output) { @@ -41,6 +41,9 @@ namespace baker { blendshapesPerMesh.push_back(hfmModelIn->meshes[i].blendshapes.toStdVector()); } output.edit4() = hfmModelIn->joints; + output.edit5() = hfmModelIn->shapes; + output.edit6() = hfmModelIn->dynamicTransforms; + output.edit7() = hfmModelIn->deformers; } }; @@ -134,6 +137,9 @@ namespace baker { const auto meshIndicesToModelNames = modelPartsIn.getN(2); const auto blendshapesPerMeshIn = modelPartsIn.getN(3); const auto jointsIn = modelPartsIn.getN(4); + const auto shapesIn = modelPartsIn.getN(5); + const auto dynamicTransformsIn = modelPartsIn.getN(6); + const auto deformersIn = modelPartsIn.getN(7); // Calculate normals and tangents for meshes and blendshapes if they do not exist // Note: Normals are never calculated here for OBJ models. OBJ files optionally define normals on a per-face basis, so for consistency normals are calculated beforehand in OBJSerializer. @@ -146,7 +152,7 @@ namespace baker { const auto tangentsPerBlendshapePerMesh = model.addJob("CalculateBlendshapeTangents", calculateBlendshapeTangentsInputs); // Build the graphics::MeshPointer for each hfm::Mesh - const auto buildGraphicsMeshInputs = BuildGraphicsMeshTask::Input(meshesIn, url, meshIndicesToModelNames, normalsPerMesh, tangentsPerMesh).asVarying(); + const auto buildGraphicsMeshInputs = BuildGraphicsMeshTask::Input(meshesIn, url, meshIndicesToModelNames, normalsPerMesh, tangentsPerMesh, shapesIn, dynamicTransformsIn, deformersIn).asVarying(); const auto graphicsMeshes = model.addJob("BuildGraphicsMesh", buildGraphicsMeshInputs); // Prepare joint information diff --git a/libraries/model-baker/src/model-baker/BuildGraphicsMeshTask.cpp b/libraries/model-baker/src/model-baker/BuildGraphicsMeshTask.cpp index 2467da7656..ea05b81d1f 100644 --- a/libraries/model-baker/src/model-baker/BuildGraphicsMeshTask.cpp +++ b/libraries/model-baker/src/model-baker/BuildGraphicsMeshTask.cpp @@ -27,7 +27,83 @@ glm::vec3 normalizeDirForPacking(const glm::vec3& dir) { return dir; } -void buildGraphicsMesh(const hfm::Mesh& hfmMesh, graphics::MeshPointer& graphicsMeshPointer, const baker::MeshNormals& meshNormals, const baker::MeshTangents& meshTangentsIn) { +class ReweightedDeformers { +public: + std::vector indices; + std::vector weights; + bool trimmedToMatch { false }; +}; + +ReweightedDeformers getReweightedDeformers(size_t numMeshVertices, const hfm::DynamicTransform* dynamicTransform, const std::vector deformers, const uint16_t weightsPerVertex) { + size_t numClusterIndices = numMeshVertices * weightsPerVertex; + ReweightedDeformers reweightedDeformers; + // TODO: Consider having a rootCluster property in the DynamicTransform rather than appending the root to the end of the cluster list. + reweightedDeformers.indices.resize(numClusterIndices, (uint16_t)(deformers.size() - 1)); + reweightedDeformers.weights.resize(numClusterIndices, 0); + + std::vector weightAccumulators; + weightAccumulators.resize(numClusterIndices, 0.0f); + for (uint16_t i = 0; i < (uint16_t)deformers.size(); ++i) { + const hfm::Deformer& deformer = *deformers[i]; + + if (deformer.indices.size() != deformer.weights.size()) { + reweightedDeformers.trimmedToMatch = true; + } + size_t numIndicesOrWeights = std::min(deformer.indices.size(), deformer.weights.size()); + for (size_t j = 0; j < numIndicesOrWeights; ++j) { + uint32_t index = deformer.indices[j]; + float weight = deformer.weights[j]; + + // look for an unused slot in the weights vector + uint32_t weightIndex = index * weightsPerVertex; + uint32_t lowestIndex = -1; + float lowestWeight = FLT_MAX; + uint16_t k = 0; + for (; k < weightsPerVertex; k++) { + if (weightAccumulators[weightIndex + k] == 0.0f) { + reweightedDeformers.indices[weightIndex + k] = i; + weightAccumulators[weightIndex + k] = weight; + break; + } + if (weightAccumulators[weightIndex + k] < lowestWeight) { + lowestIndex = k; + lowestWeight = weightAccumulators[weightIndex + k]; + } + } + if (k == weightsPerVertex && weight > lowestWeight) { + // no space for an additional weight; we must replace the lowest + weightAccumulators[weightIndex + lowestIndex] = weight; + reweightedDeformers.indices[weightIndex + lowestIndex] = i; + } + } + } + + // now that we've accumulated the most relevant weights for each vertex + // normalize and compress to 16-bits + for (size_t i = 0; i < numMeshVertices; ++i) { + size_t j = i * weightsPerVertex; + + // normalize weights into uint16_t + float totalWeight = 0.0f; + for (size_t k = j; k < j + weightsPerVertex; ++k) { + totalWeight += weightAccumulators[k]; + } + + const float ALMOST_HALF = 0.499f; + if (totalWeight > 0.0f) { + float weightScalingFactor = (float)(UINT16_MAX) / totalWeight; + for (size_t k = j; k < j + weightsPerVertex; ++k) { + reweightedDeformers.weights[k] = (uint16_t)(weightScalingFactor * weightAccumulators[k] + ALMOST_HALF); + } + } else { + reweightedDeformers.weights[j] = (uint16_t)((float)(UINT16_MAX) + ALMOST_HALF); + } + } + + return reweightedDeformers; +} + +void buildGraphicsMesh(const hfm::Mesh& hfmMesh, graphics::MeshPointer& graphicsMeshPointer, const baker::MeshNormals& meshNormals, const baker::MeshTangents& meshTangentsIn, const hfm::DynamicTransform* dynamicTransform, const std::vector meshDeformers) { auto graphicsMesh = std::make_shared(); // Fill tangents with a dummy value to force tangents to be present if there are normals @@ -86,25 +162,31 @@ void buildGraphicsMesh(const hfm::Mesh& hfmMesh, graphics::MeshPointer& graphics // Support for 4 skinning clusters: // 4 Indices are uint8 ideally, uint16 if more than 256. - const auto clusterIndiceElement = (hfmMesh.clusters.size() < UINT8_MAX ? gpu::Element(gpu::VEC4, gpu::UINT8, gpu::XYZW) : gpu::Element(gpu::VEC4, gpu::UINT16, gpu::XYZW)); + const auto clusterIndiceElement = ((meshDeformers.size() < (size_t)UINT8_MAX) ? gpu::Element(gpu::VEC4, gpu::UINT8, gpu::XYZW) : gpu::Element(gpu::VEC4, gpu::UINT16, gpu::XYZW)); // 4 Weights are normalized 16bits const auto clusterWeightElement = gpu::Element(gpu::VEC4, gpu::NUINT16, gpu::XYZW); + // Calculate a more condensed view of all the deformer weights + const uint16_t NUM_CLUSTERS_PER_VERT = 4; + ReweightedDeformers reweightedDeformers = getReweightedDeformers(hfmMesh.vertices.size(), dynamicTransform, meshDeformers, NUM_CLUSTERS_PER_VERT); // Cluster indices and weights must be the same sizes - const int NUM_CLUSTERS_PER_VERT = 4; - const int numVertClusters = (hfmMesh.clusterIndices.size() == hfmMesh.clusterWeights.size() ? hfmMesh.clusterIndices.size() / NUM_CLUSTERS_PER_VERT : 0); - const int clusterIndicesSize = numVertClusters * clusterIndiceElement.getSize(); - const int clusterWeightsSize = numVertClusters * clusterWeightElement.getSize(); + if (reweightedDeformers.trimmedToMatch) { + HIFI_FCDEBUG_ID(model_baker(), repeatMessageID, "BuildGraphicsMeshTask -- The number of indices and weights for a deformer had different sizes and have been trimmed to match"); + } + // Record cluster sizes + const size_t numVertClusters = reweightedDeformers.indices.size() / NUM_CLUSTERS_PER_VERT; + const size_t clusterIndicesSize = numVertClusters * clusterIndiceElement.getSize(); + const size_t clusterWeightsSize = numVertClusters * clusterWeightElement.getSize(); // Decide on where to put what seequencially in a big buffer: - const int positionsOffset = 0; - const int normalsAndTangentsOffset = positionsOffset + positionsSize; - const int colorsOffset = normalsAndTangentsOffset + normalsAndTangentsSize; - const int texCoordsOffset = colorsOffset + colorsSize; - const int texCoords1Offset = texCoordsOffset + texCoordsSize; - const int clusterIndicesOffset = texCoords1Offset + texCoords1Size; - const int clusterWeightsOffset = clusterIndicesOffset + clusterIndicesSize; - const int totalVertsSize = clusterWeightsOffset + clusterWeightsSize; + const size_t positionsOffset = 0; + const size_t normalsAndTangentsOffset = positionsOffset + positionsSize; + const size_t colorsOffset = normalsAndTangentsOffset + normalsAndTangentsSize; + const size_t texCoordsOffset = colorsOffset + colorsSize; + const size_t texCoords1Offset = texCoordsOffset + texCoordsSize; + const size_t clusterIndicesOffset = texCoords1Offset + texCoords1Size; + const size_t clusterWeightsOffset = clusterIndicesOffset + clusterIndicesSize; + const size_t totalVertsSize = clusterWeightsOffset + clusterWeightsSize; // Copy all vertex data in a single buffer auto vertBuffer = std::make_shared(); @@ -181,22 +263,22 @@ void buildGraphicsMesh(const hfm::Mesh& hfmMesh, graphics::MeshPointer& graphics // Clusters data if (clusterIndicesSize > 0) { - if (hfmMesh.clusters.size() < UINT8_MAX) { + if (meshDeformers.size() < UINT8_MAX) { // yay! we can fit the clusterIndices within 8-bits - int32_t numIndices = hfmMesh.clusterIndices.size(); - QVector clusterIndices; - clusterIndices.resize(numIndices); + int32_t numIndices = (int32_t)reweightedDeformers.indices.size(); + std::vector packedDeformerIndices; + packedDeformerIndices.resize(numIndices); for (int32_t i = 0; i < numIndices; ++i) { assert(hfmMesh.clusterIndices[i] <= UINT8_MAX); - clusterIndices[i] = (uint8_t)(hfmMesh.clusterIndices[i]); + packedDeformerIndices[i] = (uint8_t)(reweightedDeformers.indices[i]); } - vertBuffer->setSubData(clusterIndicesOffset, clusterIndicesSize, (const gpu::Byte*) clusterIndices.constData()); + vertBuffer->setSubData(clusterIndicesOffset, clusterIndicesSize, (const gpu::Byte*) packedDeformerIndices.data()); } else { - vertBuffer->setSubData(clusterIndicesOffset, clusterIndicesSize, (const gpu::Byte*) hfmMesh.clusterIndices.constData()); + vertBuffer->setSubData(clusterIndicesOffset, clusterIndicesSize, (const gpu::Byte*) reweightedDeformers.indices.data()); } } if (clusterWeightsSize > 0) { - vertBuffer->setSubData(clusterWeightsOffset, clusterWeightsSize, (const gpu::Byte*) hfmMesh.clusterWeights.constData()); + vertBuffer->setSubData(clusterWeightsOffset, clusterWeightsSize, (const gpu::Byte*) reweightedDeformers.weights.data()); } @@ -206,7 +288,7 @@ void buildGraphicsMesh(const hfm::Mesh& hfmMesh, graphics::MeshPointer& graphics auto vertexBufferStream = std::make_shared(); gpu::BufferPointer attribBuffer; - int totalAttribBufferSize = totalVertsSize; + size_t totalAttribBufferSize = totalVertsSize; gpu::uint8 posChannel = 0; gpu::uint8 tangentChannel = posChannel; gpu::uint8 attribChannel = posChannel; @@ -377,6 +459,18 @@ void BuildGraphicsMeshTask::run(const baker::BakeContextPointer& context, const const auto& meshIndicesToModelNames = input.get2(); const auto& normalsPerMesh = input.get3(); const auto& tangentsPerMesh = input.get4(); + const auto& shapes = input.get5(); + const auto& dynamicTransforms = input.get6(); + const auto& deformers = input.get7(); + + // Currently, there is only (at most) one dynamicTransform per mesh + // An undefined shape.dynamicTransform has the value hfm::UNDEFINED_KEY + std::vector dynamicTransformPerMesh; + dynamicTransformPerMesh.resize(meshes.size(), hfm::UNDEFINED_KEY); + for (const auto& shape : shapes) { + uint32_t dynamicTransformIndex = shape.dynamicTransform; + dynamicTransformPerMesh[shape.mesh] = dynamicTransformIndex; + } auto& graphicsMeshes = output; @@ -384,9 +478,20 @@ void BuildGraphicsMeshTask::run(const baker::BakeContextPointer& context, const for (int i = 0; i < n; i++) { graphicsMeshes.emplace_back(); auto& graphicsMesh = graphicsMeshes[i]; - + + auto dynamicTransformIndex = dynamicTransformPerMesh[i]; + const hfm::DynamicTransform* dynamicTransform = nullptr; + std::vector meshDeformers; + if (dynamicTransformIndex != hfm::UNDEFINED_KEY) { + dynamicTransform = &dynamicTransforms[dynamicTransformIndex]; + for (const auto& deformerIndex : dynamicTransform->deformers) { + const auto& deformer = deformers[deformerIndex]; + meshDeformers.push_back(&deformer); + } + } + // Try to create the graphics::Mesh - buildGraphicsMesh(meshes[i], graphicsMesh, baker::safeGet(normalsPerMesh, i), baker::safeGet(tangentsPerMesh, i)); + buildGraphicsMesh(meshes[i], graphicsMesh, baker::safeGet(normalsPerMesh, i), baker::safeGet(tangentsPerMesh, i), dynamicTransform, meshDeformers); // Choose a name for the mesh if (graphicsMesh) { diff --git a/libraries/model-baker/src/model-baker/BuildGraphicsMeshTask.h b/libraries/model-baker/src/model-baker/BuildGraphicsMeshTask.h index bb4136c086..be1e4350be 100644 --- a/libraries/model-baker/src/model-baker/BuildGraphicsMeshTask.h +++ b/libraries/model-baker/src/model-baker/BuildGraphicsMeshTask.h @@ -20,7 +20,7 @@ class BuildGraphicsMeshTask { public: - using Input = baker::VaryingSet5, hifi::URL, baker::MeshIndicesToModelNames, baker::NormalsPerMesh, baker::TangentsPerMesh>; + using Input = baker::VaryingSet8, hifi::URL, baker::MeshIndicesToModelNames, baker::NormalsPerMesh, baker::TangentsPerMesh, std::vector, std::vector, std::vector>; using Output = std::vector; using JobModel = baker::Job::ModelIO;