From 66ad6451f94dee68fa405889ca88883d65b71fbb Mon Sep 17 00:00:00 2001 From: raveenajain Date: Fri, 26 Apr 2019 00:40:44 +0100 Subject: [PATCH 1/2] morph targets for avatars --- libraries/fbx/src/GLTFSerializer.cpp | 111 ++++++++++++++++++++++++++- libraries/fbx/src/GLTFSerializer.h | 18 ++++- 2 files changed, 125 insertions(+), 4 deletions(-) diff --git a/libraries/fbx/src/GLTFSerializer.cpp b/libraries/fbx/src/GLTFSerializer.cpp index 0264accdc3..3ae5e5c0db 100755 --- a/libraries/fbx/src/GLTFSerializer.cpp +++ b/libraries/fbx/src/GLTFSerializer.cpp @@ -33,6 +33,7 @@ #include #include #include +#include #include "FBXSerializer.h" @@ -483,7 +484,7 @@ bool GLTFSerializer::addMesh(const QJsonObject& object) { GLTFMeshPrimitiveAttr target; foreach(const QString & tarKey, tarKeys) { int tarVal; - getIntVal(jsAttributes, tarKey, tarVal, target.defined); + getIntVal(jsTarget, tarKey, tarVal, target.defined); target.values.insert(tarKey, tarVal); } primitive.targets.push_back(target); @@ -493,7 +494,18 @@ bool GLTFSerializer::addMesh(const QJsonObject& object) { mesh.primitives.push_back(primitive); } } + } + QJsonObject jsExtras; + GLTFMeshExtra extras; + if (getObjectVal(object, "extras", jsExtras, mesh.defined)) { + QJsonArray jsTargetNames; + if (getObjectArrayVal(jsExtras, "targetNames", jsTargetNames, extras.defined)) { + foreach (const QJsonValue& tarName, jsTargetNames) { + extras.targetNames.push_back(tarName.toString()); + } + } + mesh.extras = extras; } _file.meshes.push_back(mesh); @@ -751,7 +763,24 @@ void GLTFSerializer::getSkinInverseBindMatrices(std::vector>& } } -bool GLTFSerializer::buildGeometry(HFMModel& hfmModel, const hifi::URL& url) { +void GLTFSerializer::generateTargetData(int index, float weight, QVector& returnVector) { + GLTFAccessor& accessor = _file.accessors[index]; + GLTFBufferView& bufferview = _file.bufferviews[accessor.bufferView]; + GLTFBuffer& buffer = _file.buffers[bufferview.buffer]; + int accBoffset = accessor.defined["byteOffset"] ? accessor.byteOffset : 0; + QVector storedValues; + addArrayOfType(buffer.blob, + bufferview.byteOffset + accBoffset, + accessor.count, + storedValues, + accessor.type, + accessor.componentType); + for (int n = 0; n < storedValues.size(); n = n + 3) { + returnVector.push_back(glm::vec3(weight * storedValues[n], weight * storedValues[n + 1], weight * storedValues[n + 2])); + } +} + +bool GLTFSerializer::buildGeometry(HFMModel& hfmModel, const hifi::VariantHash& mapping, const hifi::URL& url) { int numNodes = _file.nodes.size(); // Build dependencies @@ -1138,6 +1167,82 @@ bool GLTFSerializer::buildGeometry(HFMModel& hfmModel, const hifi::URL& url) { if (mesh.texCoords.size() == 0 && !hfmModel.hasSkeletonJoints) { for (int i = 0; i < part.triangleIndices.size(); i++) { mesh.texCoords.push_back(glm::vec2(0.0, 1.0)); } } + + // Build morph targets (blend shapes) + if (!primitive.targets.isEmpty()) { + + // Build list of blendshapes from FST + typedef QPair WeightedIndex; + hifi::VariantHash blendshapeMappings = mapping.value("bs").toHash(); + QMultiHash blendshapeIndices; + + for (int i = 0;; i++) { + hifi::ByteArray blendshapeName = FACESHIFT_BLENDSHAPES[i]; + if (blendshapeName.isEmpty()) { + break; + } + QList mappings = blendshapeMappings.values(blendshapeName); + foreach (const QVariant& mapping, mappings) { + QVariantList blendshapeMapping = mapping.toList(); + blendshapeIndices.insert(blendshapeMapping.at(0).toByteArray(), WeightedIndex(i, blendshapeMapping.at(1).toFloat())); + } + } + + // glTF morph targets may or may not have names. if they are labeled, add them based on + // the corresponding names from the FST. otherwise, just add them in the order they are given + mesh.blendshapes.resize(blendshapeMappings.size()); + auto values = blendshapeIndices.values(); + auto keys = blendshapeIndices.keys(); + auto names = _file.meshes[node.mesh].extras.targetNames; + QVector weights = _file.meshes[node.mesh].weights; + + for (int weightedIndex = 0; weightedIndex < values.size(); weightedIndex++) { + float weight = 0.1f; + int indexFromMapping = weightedIndex; + int targetIndex = weightedIndex; + hfmModel.blendshapeChannelNames.push_back("target_" + weightedIndex); + + if (!names.isEmpty()) { + targetIndex = names.indexOf(keys[weightedIndex]); + indexFromMapping = values[weightedIndex].first; + weight = weight * values[weightedIndex].second; + hfmModel.blendshapeChannelNames[weightedIndex] = keys[weightedIndex]; + } + HFMBlendshape& blendshape = mesh.blendshapes[indexFromMapping]; + blendshape.indices = part.triangleIndices; + auto target = primitive.targets[targetIndex]; + + QVector normals; + QVector vertices; + + if (weights.size() == primitive.targets.size()) { + int targetWeight = weights[targetIndex]; + if (targetWeight != 0) { + weight = weight * targetWeight; + } + } + + if (target.values.contains((QString) "NORMAL")) { + generateTargetData(target.values.value((QString) "NORMAL"), weight, normals); + } + if (target.values.contains((QString) "POSITION")) { + generateTargetData(target.values.value((QString) "POSITION"), weight, vertices); + } + bool isNewBlendshape = blendshape.vertices.size() < vertices.size(); + int count = 0; + for (int i : blendshape.indices) { + if (isNewBlendshape) { + blendshape.vertices.push_back(vertices[i]); + blendshape.normals.push_back(normals[i]); + } else { + blendshape.vertices[count] = blendshape.vertices[count] + vertices[i]; + blendshape.normals[count] = blendshape.normals[count] + normals[i]; + count++; + } + } + } + } + mesh.meshExtents.reset(); foreach(const glm::vec3& vertex, mesh.vertices) { mesh.meshExtents.addPoint(vertex); @@ -1183,7 +1288,7 @@ HFMModel::Pointer GLTFSerializer::read(const hifi::ByteArray& data, const hifi:: //_file.dump(); auto hfmModelPtr = std::make_shared(); HFMModel& hfmModel = *hfmModelPtr; - buildGeometry(hfmModel, _url); + buildGeometry(hfmModel, mapping, _url); //hfmDebugDump(data); return hfmModelPtr; diff --git a/libraries/fbx/src/GLTFSerializer.h b/libraries/fbx/src/GLTFSerializer.h index 8784667d1e..b1c1bc4e44 100755 --- a/libraries/fbx/src/GLTFSerializer.h +++ b/libraries/fbx/src/GLTFSerializer.h @@ -159,9 +159,20 @@ struct GLTFMeshPrimitive { } }; +struct GLTFMeshExtra { + QVector targetNames; + QMap defined; + void dump() { + if (defined["targetNames"]) { + qCDebug(modelformat) << "targetNames: " << targetNames; + } + } +}; + struct GLTFMesh { QString name; QVector primitives; + GLTFMeshExtra extras; QVector weights; QMap defined; void dump() { @@ -172,6 +183,10 @@ struct GLTFMesh { qCDebug(modelformat) << "primitives: "; foreach(auto prim, primitives) prim.dump(); } + if (defined["extras"]) { + qCDebug(modelformat) << "extras: "; + extras.dump(); + } if (defined["weights"]) { qCDebug(modelformat) << "weights: " << weights; } @@ -713,8 +728,9 @@ private: glm::mat4 getModelTransform(const GLTFNode& node); void getSkinInverseBindMatrices(std::vector>& inverseBindMatrixValues); + void generateTargetData(int index, float weight, QVector& returnVector); - bool buildGeometry(HFMModel& hfmModel, const hifi::URL& url); + bool buildGeometry(HFMModel& hfmModel, const hifi::VariantHash& mapping, const hifi::URL& url); bool parseGLTF(const hifi::ByteArray& data); bool getStringVal(const QJsonObject& object, const QString& fieldname, From 0be36783bf49001a8bf4f5c0d6bbc37b829ed9dc Mon Sep 17 00:00:00 2001 From: raveenajain Date: Fri, 26 Apr 2019 01:37:26 +0100 Subject: [PATCH 2/2] hfm blendshape name --- libraries/fbx/src/GLTFSerializer.cpp | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/libraries/fbx/src/GLTFSerializer.cpp b/libraries/fbx/src/GLTFSerializer.cpp index 3ae5e5c0db..622fb92ce7 100755 --- a/libraries/fbx/src/GLTFSerializer.cpp +++ b/libraries/fbx/src/GLTFSerializer.cpp @@ -1200,7 +1200,7 @@ bool GLTFSerializer::buildGeometry(HFMModel& hfmModel, const hifi::VariantHash& float weight = 0.1f; int indexFromMapping = weightedIndex; int targetIndex = weightedIndex; - hfmModel.blendshapeChannelNames.push_back("target_" + weightedIndex); + hfmModel.blendshapeChannelNames.push_back("target_" + QString::number(weightedIndex)); if (!names.isEmpty()) { targetIndex = names.indexOf(keys[weightedIndex]);