From de8223fee808c2ae1efec5c93b1da945b10d14ed Mon Sep 17 00:00:00 2001 From: sabrina-shanman Date: Mon, 16 Sep 2019 17:44:43 -0700 Subject: [PATCH] Change BuildGraphicsMeshTask to use new deformers --- .../model-baker/src/model-baker/Baker.cpp | 10 +- .../src/model-baker/BuildGraphicsMeshTask.cpp | 134 ++++++++++++++++-- .../src/model-baker/BuildGraphicsMeshTask.h | 2 +- 3 files changed, 129 insertions(+), 17 deletions(-) diff --git a/libraries/model-baker/src/model-baker/Baker.cpp b/libraries/model-baker/src/model-baker/Baker.cpp index 3dab7f7241..1a68d3508d 100644 --- a/libraries/model-baker/src/model-baker/Baker.cpp +++ b/libraries/model-baker/src/model-baker/Baker.cpp @@ -27,7 +27,7 @@ namespace baker { class GetModelPartsTask { public: using Input = hfm::Model::Pointer; - using Output = VaryingSet5, hifi::URL, baker::MeshIndicesToModelNames, baker::BlendshapesPerMesh, std::vector>; + using Output = VaryingSet8, hifi::URL, baker::MeshIndicesToModelNames, baker::BlendshapesPerMesh, std::vector, std::vector, std::vector, std::vector>; using JobModel = Job::ModelIO; void run(const BakeContextPointer& context, const Input& input, Output& output) { @@ -41,6 +41,9 @@ namespace baker { blendshapesPerMesh.push_back(hfmModelIn->meshes[i].blendshapes.toStdVector()); } output.edit4() = hfmModelIn->joints; + output.edit5() = hfmModelIn->shapes; + output.edit6() = hfmModelIn->dynamicTransforms; + output.edit7() = hfmModelIn->deformers; } }; @@ -134,6 +137,9 @@ namespace baker { const auto meshIndicesToModelNames = modelPartsIn.getN(2); const auto blendshapesPerMeshIn = modelPartsIn.getN(3); const auto jointsIn = modelPartsIn.getN(4); + const auto shapesIn = modelPartsIn.getN(5); + const auto dynamicTransformsIn = modelPartsIn.getN(6); + const auto deformersIn = modelPartsIn.getN(7); // Calculate normals and tangents for meshes and blendshapes if they do not exist // Note: Normals are never calculated here for OBJ models. OBJ files optionally define normals on a per-face basis, so for consistency normals are calculated beforehand in OBJSerializer. @@ -146,7 +152,7 @@ namespace baker { const auto tangentsPerBlendshapePerMesh = model.addJob("CalculateBlendshapeTangents", calculateBlendshapeTangentsInputs); // Build the graphics::MeshPointer for each hfm::Mesh - const auto buildGraphicsMeshInputs = BuildGraphicsMeshTask::Input(meshesIn, url, meshIndicesToModelNames, normalsPerMesh, tangentsPerMesh).asVarying(); + const auto buildGraphicsMeshInputs = BuildGraphicsMeshTask::Input(meshesIn, url, meshIndicesToModelNames, normalsPerMesh, tangentsPerMesh, shapesIn, dynamicTransformsIn, deformersIn).asVarying(); const auto graphicsMeshes = model.addJob("BuildGraphicsMesh", buildGraphicsMeshInputs); // Prepare joint information diff --git a/libraries/model-baker/src/model-baker/BuildGraphicsMeshTask.cpp b/libraries/model-baker/src/model-baker/BuildGraphicsMeshTask.cpp index 2467da7656..54e0e0ee2e 100644 --- a/libraries/model-baker/src/model-baker/BuildGraphicsMeshTask.cpp +++ b/libraries/model-baker/src/model-baker/BuildGraphicsMeshTask.cpp @@ -27,7 +27,84 @@ glm::vec3 normalizeDirForPacking(const glm::vec3& dir) { return dir; } -void buildGraphicsMesh(const hfm::Mesh& hfmMesh, graphics::MeshPointer& graphicsMeshPointer, const baker::MeshNormals& meshNormals, const baker::MeshTangents& meshTangentsIn) { +class ReweightedDeformers { +public: + std::vector indices; + std::vector weights; + bool trimmedToMatch { false }; +}; + +ReweightedDeformers getReweightedDeformers(size_t numMeshVertices, const hfm::DynamicTransform* dynamicTransform, const std::vector deformers, const uint16_t weightsPerVertex) { + size_t numClusterIndices = numMeshVertices * weightsPerVertex; + ReweightedDeformers reweightedDeformers; + // TODO: Consider having a rootCluster property in the DynamicTransform rather than appending the root to the end of the cluster list. + reweightedDeformers.indices.resize(numClusterIndices, deformers.size() - 1); + reweightedDeformers.weights.resize(numClusterIndices, 0); + + std::vector weightAccumulators; + weightAccumulators.resize(numClusterIndices, 0.0f); + for (size_t i = 0; i < deformers.size(); ++i) { + const hfm::Deformer& deformer = *deformers[i]; + const hfm::Cluster& cluster = dynamicTransform->clusters[i]; + + if (deformer.indices.size() != deformer.weights.size()) { + reweightedDeformers.trimmedToMatch = true; + } + size_t numIndicesOrWeights = std::min(deformer.indices.size(), deformer.weights.size()); + for (size_t j = 0; j < numIndicesOrWeights; ++j) { + uint32_t index = deformer.indices[j]; + float weight = deformer.weights[j]; + + // look for an unused slot in the weights vector + uint32_t weightIndex = index * weightsPerVertex; + uint32_t lowestIndex = -1; + float lowestWeight = FLT_MAX; + uint16_t k = 0; + for (; k < weightsPerVertex; k++) { + if (weightAccumulators[weightIndex + k] == 0.0f) { + reweightedDeformers.indices[weightIndex + k] = i; + weightAccumulators[weightIndex + k] = weight; + break; + } + if (weightAccumulators[weightIndex + k] < lowestWeight) { + lowestIndex = k; + lowestWeight = weightAccumulators[weightIndex + k]; + } + } + if (k == weightsPerVertex && weight > lowestWeight) { + // no space for an additional weight; we must replace the lowest + weightAccumulators[weightIndex + lowestIndex] = weight; + reweightedDeformers.indices[weightIndex + lowestIndex] = i; + } + } + } + + // now that we've accumulated the most relevant weights for each vertex + // normalize and compress to 16-bits + for (size_t i = 0; i < numMeshVertices; ++i) { + size_t j = i * weightsPerVertex; + + // normalize weights into uint16_t + float totalWeight = 0.0f; + for (size_t k = j; k < j + weightsPerVertex; ++k) { + totalWeight += weightAccumulators[k]; + } + + const float ALMOST_HALF = 0.499f; + if (totalWeight > 0.0f) { + float weightScalingFactor = (float)(UINT16_MAX) / totalWeight; + for (size_t k = j; k < j + weightsPerVertex; ++k) { + reweightedDeformers.weights[k] = (uint16_t)(weightScalingFactor * weightAccumulators[k] + ALMOST_HALF); + } + } else { + reweightedDeformers.weights[j] = (uint16_t)((float)(UINT16_MAX) + ALMOST_HALF); + } + } + + return reweightedDeformers; +} + +void buildGraphicsMesh(const hfm::Mesh& hfmMesh, graphics::MeshPointer& graphicsMeshPointer, const baker::MeshNormals& meshNormals, const baker::MeshTangents& meshTangentsIn, const hfm::DynamicTransform* dynamicTransform, const std::vector meshDeformers) { auto graphicsMesh = std::make_shared(); // Fill tangents with a dummy value to force tangents to be present if there are normals @@ -86,13 +163,19 @@ void buildGraphicsMesh(const hfm::Mesh& hfmMesh, graphics::MeshPointer& graphics // Support for 4 skinning clusters: // 4 Indices are uint8 ideally, uint16 if more than 256. - const auto clusterIndiceElement = (hfmMesh.clusters.size() < UINT8_MAX ? gpu::Element(gpu::VEC4, gpu::UINT8, gpu::XYZW) : gpu::Element(gpu::VEC4, gpu::UINT16, gpu::XYZW)); + const auto clusterIndiceElement = ((meshDeformers.size() < (size_t)UINT8_MAX) ? gpu::Element(gpu::VEC4, gpu::UINT8, gpu::XYZW) : gpu::Element(gpu::VEC4, gpu::UINT16, gpu::XYZW)); // 4 Weights are normalized 16bits const auto clusterWeightElement = gpu::Element(gpu::VEC4, gpu::NUINT16, gpu::XYZW); + // Calculate a more condensed view of all the deformer weights + const uint16_t NUM_CLUSTERS_PER_VERT = 4; + ReweightedDeformers reweightedDeformers = getReweightedDeformers(hfmMesh.vertices.size(), dynamicTransform, meshDeformers, NUM_CLUSTERS_PER_VERT); // Cluster indices and weights must be the same sizes - const int NUM_CLUSTERS_PER_VERT = 4; - const int numVertClusters = (hfmMesh.clusterIndices.size() == hfmMesh.clusterWeights.size() ? hfmMesh.clusterIndices.size() / NUM_CLUSTERS_PER_VERT : 0); + if (reweightedDeformers.trimmedToMatch) { + HIFI_FCDEBUG_ID(model_baker(), repeatMessageID, "BuildGraphicsMeshTask -- The number of indices and weights for a blendshape had different sizes and have been trimmed to match"); + } + // Record cluster sizes + const int numVertClusters = reweightedDeformers.indices.size() / NUM_CLUSTERS_PER_VERT; const int clusterIndicesSize = numVertClusters * clusterIndiceElement.getSize(); const int clusterWeightsSize = numVertClusters * clusterWeightElement.getSize(); @@ -181,22 +264,22 @@ void buildGraphicsMesh(const hfm::Mesh& hfmMesh, graphics::MeshPointer& graphics // Clusters data if (clusterIndicesSize > 0) { - if (hfmMesh.clusters.size() < UINT8_MAX) { + if (meshDeformers.size() < UINT8_MAX) { // yay! we can fit the clusterIndices within 8-bits - int32_t numIndices = hfmMesh.clusterIndices.size(); - QVector clusterIndices; - clusterIndices.resize(numIndices); + int32_t numIndices = reweightedDeformers.indices.size(); + std::vector packedDeformerIndices; + packedDeformerIndices.resize(numIndices); for (int32_t i = 0; i < numIndices; ++i) { assert(hfmMesh.clusterIndices[i] <= UINT8_MAX); - clusterIndices[i] = (uint8_t)(hfmMesh.clusterIndices[i]); + packedDeformerIndices[i] = (uint8_t)(reweightedDeformers.indices[i]); } - vertBuffer->setSubData(clusterIndicesOffset, clusterIndicesSize, (const gpu::Byte*) clusterIndices.constData()); + vertBuffer->setSubData(clusterIndicesOffset, clusterIndicesSize, (const gpu::Byte*) packedDeformerIndices.data()); } else { - vertBuffer->setSubData(clusterIndicesOffset, clusterIndicesSize, (const gpu::Byte*) hfmMesh.clusterIndices.constData()); + vertBuffer->setSubData(clusterIndicesOffset, clusterIndicesSize, (const gpu::Byte*) reweightedDeformers.indices.data()); } } if (clusterWeightsSize > 0) { - vertBuffer->setSubData(clusterWeightsOffset, clusterWeightsSize, (const gpu::Byte*) hfmMesh.clusterWeights.constData()); + vertBuffer->setSubData(clusterWeightsOffset, clusterWeightsSize, (const gpu::Byte*) reweightedDeformers.weights.data()); } @@ -377,6 +460,18 @@ void BuildGraphicsMeshTask::run(const baker::BakeContextPointer& context, const const auto& meshIndicesToModelNames = input.get2(); const auto& normalsPerMesh = input.get3(); const auto& tangentsPerMesh = input.get4(); + const auto& shapes = input.get5(); + const auto& dynamicTransforms = input.get6(); + const auto& deformers = input.get7(); + + // Currently, there is only (at most) one dynamicTransform per mesh + // An undefined shape.dynamicTransform has the value hfm::Shape::UNDEFINED_KEY + std::vector dynamicTransformPerMesh; + dynamicTransformPerMesh.resize(meshes.size(), hfm::Shape::UNDEFINED_KEY); + for (const auto& shape : shapes) { + uint32_t dynamicTransformIndex = shape.dynamicTransform; + dynamicTransformPerMesh[shape.mesh] = dynamicTransformIndex; + } auto& graphicsMeshes = output; @@ -384,9 +479,20 @@ void BuildGraphicsMeshTask::run(const baker::BakeContextPointer& context, const for (int i = 0; i < n; i++) { graphicsMeshes.emplace_back(); auto& graphicsMesh = graphicsMeshes[i]; - + + auto dynamicTransformIndex = dynamicTransformPerMesh[i]; + const hfm::DynamicTransform* dynamicTransform = nullptr; + std::vector meshDeformers; + if (dynamicTransformIndex != hfm::Shape::UNDEFINED_KEY) { + dynamicTransform = &dynamicTransforms[dynamicTransformIndex]; + for (const auto& deformerIndex : dynamicTransform->deformers) { + const auto& deformer = deformers[deformerIndex]; + meshDeformers.push_back(&deformer); + } + } + // Try to create the graphics::Mesh - buildGraphicsMesh(meshes[i], graphicsMesh, baker::safeGet(normalsPerMesh, i), baker::safeGet(tangentsPerMesh, i)); + buildGraphicsMesh(meshes[i], graphicsMesh, baker::safeGet(normalsPerMesh, i), baker::safeGet(tangentsPerMesh, i), dynamicTransform, meshDeformers); // Choose a name for the mesh if (graphicsMesh) { diff --git a/libraries/model-baker/src/model-baker/BuildGraphicsMeshTask.h b/libraries/model-baker/src/model-baker/BuildGraphicsMeshTask.h index bb4136c086..be1e4350be 100644 --- a/libraries/model-baker/src/model-baker/BuildGraphicsMeshTask.h +++ b/libraries/model-baker/src/model-baker/BuildGraphicsMeshTask.h @@ -20,7 +20,7 @@ class BuildGraphicsMeshTask { public: - using Input = baker::VaryingSet5, hifi::URL, baker::MeshIndicesToModelNames, baker::NormalsPerMesh, baker::TangentsPerMesh>; + using Input = baker::VaryingSet8, hifi::URL, baker::MeshIndicesToModelNames, baker::NormalsPerMesh, baker::TangentsPerMesh, std::vector, std::vector, std::vector>; using Output = std::vector; using JobModel = baker::Job::ModelIO;