From 71bcabfc0aebc547d57efcb852f530317bda0a99 Mon Sep 17 00:00:00 2001 From: David Rowe Date: Sat, 17 Apr 2021 21:30:46 +1200 Subject: [PATCH 1/7] Remove unused enum --- libraries/shared/src/BlendshapeConstants.h | 10 ---------- 1 file changed, 10 deletions(-) diff --git a/libraries/shared/src/BlendshapeConstants.h b/libraries/shared/src/BlendshapeConstants.h index 1c37892088..c254e33910 100644 --- a/libraries/shared/src/BlendshapeConstants.h +++ b/libraries/shared/src/BlendshapeConstants.h @@ -87,16 +87,6 @@ enum class Blendshapes : int { BlendshapeCount }; -enum class LegacyBlendshpaes : int { - JawChew, // not in ARKit - LipsUpperUp, // split in ARKit - LipsLowerDown, // split in ARKit - ChinLowerRaise, // not in ARKit - ChinUpperRaise, // not in ARKit - Sneer, // split in ARKit - LegacyBlendshapeCount -}; - // Original blendshapes were per Faceshift. // NEW in ARKit From 0b536f0cae9517c0db54bc084117b94974bffead Mon Sep 17 00:00:00 2001 From: David Rowe Date: Mon, 19 Apr 2021 11:10:25 +1200 Subject: [PATCH 2/7] Support ReadyPlayerMe blendshapes in glTF files --- libraries/fbx/src/GLTFSerializer.cpp | 16 ++++++++++++++-- libraries/shared/src/BlendshapeConstants.h | 18 ++++++++++++++++++ 2 files changed, 32 insertions(+), 2 deletions(-) diff --git a/libraries/fbx/src/GLTFSerializer.cpp b/libraries/fbx/src/GLTFSerializer.cpp index 0916316ec1..ec4777933a 100755 --- a/libraries/fbx/src/GLTFSerializer.cpp +++ b/libraries/fbx/src/GLTFSerializer.cpp @@ -1550,7 +1550,6 @@ bool GLTFSerializer::buildGeometry(HFMModel& hfmModel, const hifi::VariantHash& typedef QPair WeightedIndex; hifi::VariantHash blendshapeMappings = mapping.value("bs").toHash(); QMultiHash blendshapeIndices; - for (int i = 0;; ++i) { auto blendshapeName = QString(BLENDSHAPE_NAMES[i]); if (blendshapeName.isEmpty()) { @@ -1561,7 +1560,7 @@ bool GLTFSerializer::buildGeometry(HFMModel& hfmModel, const hifi::VariantHash& // Use blendshape from mapping. foreach(const QVariant& mapping, mappings) { auto blendshapeMapping = mapping.toList(); - blendshapeIndices.insert(blendshapeMapping.at(0).toString(), + blendshapeIndices.insert(blendshapeMapping.at(0).toString(), WeightedIndex(i, blendshapeMapping.at(1).toFloat())); } } else { @@ -1572,6 +1571,19 @@ bool GLTFSerializer::buildGeometry(HFMModel& hfmModel, const hifi::VariantHash& } } + // Augment list of blendshapes from synonyms in model. + QMap>::const_iterator synonym = BLENDSHAPE_SYNONYMS_MAP.constBegin(); + while (synonym != BLENDSHAPE_SYNONYMS_MAP.constEnd()) { + if (_file.meshes[node.mesh].extras.targetNames.contains(synonym.key())) { + auto blendshape = BLENDSHAPE_LOOKUP_MAP.find(synonym.value().first); + if (blendshape != BLENDSHAPE_LOOKUP_MAP.end()) { + blendshapeIndices.insert(synonym.key(), + WeightedIndex(blendshape.value(), synonym.value().second)); + } + } + ++synonym; + } + // Create blendshapes. if (!blendshapeIndices.isEmpty()) { mesh.blendshapes.resize((int)Blendshapes::BlendshapeCount); diff --git a/libraries/shared/src/BlendshapeConstants.h b/libraries/shared/src/BlendshapeConstants.h index c254e33910..47e887cf57 100644 --- a/libraries/shared/src/BlendshapeConstants.h +++ b/libraries/shared/src/BlendshapeConstants.h @@ -113,6 +113,24 @@ enum class Blendshapes : int { // * LipsUpperOpen (not in ARKit) // * LipsLowerOpen (not in ARKit) +const QMap> BLENDSHAPE_SYNONYMS_MAP { + // ReadyPlayerMe + {"browDownLeft", {"BrowsD_L", 0.5f}}, + {"browDownRight", {"BrowsD_R", 0.5f}}, + {"browInnerUp", {"BrowsU_C", 0.3f}}, + {"browOuterUpLeft", {"BrowsU_L", 0.3f}}, + {"browOuterUpRight", {"BrowsU_R", 0.3f}}, + {"eyeBlinkLeft", {"EyeBlink_L", 1.0f}}, + {"eyeBlinkRight", {"EyeBlink_R", 1.0f}}, + {"eyeWideLeft", {"EyeOpen_L", 1.0f}}, + {"eyeWideRight", {"EyeOpen_R", 1.0f}}, + {"mouthLeft", {"MouthSmile_L", 0.6f}}, + {"mouthOpen", {"JawOpen", 1.0f}}, + {"mouthRight", {"MouthSmile_R", 0.6f}}, + {"mouthShrugLower", {"LipsUpperClose", 0.1f}}, + {"viseme_O", {"LipsFunnel", 0.5f}} +}; + struct BlendshapeOffsetPacked { glm::uvec4 packedPosNorTan; }; From a6d8e150f00cac24c571052e20ef88435584c761 Mon Sep 17 00:00:00 2001 From: David Rowe Date: Wed, 21 Apr 2021 09:47:24 +1200 Subject: [PATCH 3/7] Move and rename ReadyPlayerMe synonyms --- .../model-serializers/src/GLTFSerializer.cpp | 4 ++-- libraries/shared/src/BlendshapeConstants.cpp | 18 ++++++++++++++++++ libraries/shared/src/BlendshapeConstants.h | 19 +------------------ 3 files changed, 21 insertions(+), 20 deletions(-) diff --git a/libraries/model-serializers/src/GLTFSerializer.cpp b/libraries/model-serializers/src/GLTFSerializer.cpp index 092fa3d694..1c1e45fbf3 100755 --- a/libraries/model-serializers/src/GLTFSerializer.cpp +++ b/libraries/model-serializers/src/GLTFSerializer.cpp @@ -1572,8 +1572,8 @@ bool GLTFSerializer::buildGeometry(HFMModel& hfmModel, const hifi::VariantHash& } // Augment list of blendshapes from synonyms in model. - QMap>::const_iterator synonym = BLENDSHAPE_SYNONYMS_MAP.constBegin(); - while (synonym != BLENDSHAPE_SYNONYMS_MAP.constEnd()) { + QMap>::const_iterator synonym = READYPLAYERME_BLENDSHAPES_MAP.constBegin(); + while (synonym != READYPLAYERME_BLENDSHAPES_MAP.constEnd()) { if (_file.meshes[node.mesh].extras.targetNames.contains(synonym.key())) { auto blendshape = BLENDSHAPE_LOOKUP_MAP.find(synonym.value().first); if (blendshape != BLENDSHAPE_LOOKUP_MAP.end()) { diff --git a/libraries/shared/src/BlendshapeConstants.cpp b/libraries/shared/src/BlendshapeConstants.cpp index 9035b10529..39641851b6 100644 --- a/libraries/shared/src/BlendshapeConstants.cpp +++ b/libraries/shared/src/BlendshapeConstants.cpp @@ -84,3 +84,21 @@ const QMap BLENDSHAPE_LOOKUP_MAP = [] { } return toReturn; }(); + +const QMap> READYPLAYERME_BLENDSHAPES_MAP = { + // ReadyPlayerMe blendshape synonyms. + {"browDownLeft", {"BrowsD_L", 0.5f}}, + {"browDownRight", {"BrowsD_R", 0.5f}}, + {"browInnerUp", {"BrowsU_C", 0.3f}}, + {"browOuterUpLeft", {"BrowsU_L", 0.3f}}, + {"browOuterUpRight", {"BrowsU_R", 0.3f}}, + {"eyeBlinkLeft", {"EyeBlink_L", 1.0f}}, + {"eyeBlinkRight", {"EyeBlink_R", 1.0f}}, + {"eyeWideLeft", {"EyeOpen_L", 1.0f}}, + {"eyeWideRight", {"EyeOpen_R", 1.0f}}, + {"mouthLeft", {"MouthSmile_L", 0.6f}}, + {"mouthOpen", {"JawOpen", 1.0f}}, + {"mouthRight", {"MouthSmile_R", 0.6f}}, + {"mouthShrugLower", {"LipsUpperClose", 0.1f}}, + {"viseme_O", {"LipsFunnel", 0.5f}} +}; diff --git a/libraries/shared/src/BlendshapeConstants.h b/libraries/shared/src/BlendshapeConstants.h index 47e887cf57..b2a6630224 100644 --- a/libraries/shared/src/BlendshapeConstants.h +++ b/libraries/shared/src/BlendshapeConstants.h @@ -20,6 +20,7 @@ /// The names of the supported blendshapes, terminated with an empty string. extern const char* BLENDSHAPE_NAMES[]; extern const QMap BLENDSHAPE_LOOKUP_MAP; +extern const QMap> READYPLAYERME_BLENDSHAPES_MAP; enum class Blendshapes : int { EyeBlink_L = 0, @@ -113,24 +114,6 @@ enum class Blendshapes : int { // * LipsUpperOpen (not in ARKit) // * LipsLowerOpen (not in ARKit) -const QMap> BLENDSHAPE_SYNONYMS_MAP { - // ReadyPlayerMe - {"browDownLeft", {"BrowsD_L", 0.5f}}, - {"browDownRight", {"BrowsD_R", 0.5f}}, - {"browInnerUp", {"BrowsU_C", 0.3f}}, - {"browOuterUpLeft", {"BrowsU_L", 0.3f}}, - {"browOuterUpRight", {"BrowsU_R", 0.3f}}, - {"eyeBlinkLeft", {"EyeBlink_L", 1.0f}}, - {"eyeBlinkRight", {"EyeBlink_R", 1.0f}}, - {"eyeWideLeft", {"EyeOpen_L", 1.0f}}, - {"eyeWideRight", {"EyeOpen_R", 1.0f}}, - {"mouthLeft", {"MouthSmile_L", 0.6f}}, - {"mouthOpen", {"JawOpen", 1.0f}}, - {"mouthRight", {"MouthSmile_R", 0.6f}}, - {"mouthShrugLower", {"LipsUpperClose", 0.1f}}, - {"viseme_O", {"LipsFunnel", 0.5f}} -}; - struct BlendshapeOffsetPacked { glm::uvec4 packedPosNorTan; }; From f4af0be95b1e1fe400b7ae284ad08e1dfcbff368 Mon Sep 17 00:00:00 2001 From: David Rowe Date: Wed, 21 Apr 2021 10:14:46 +1200 Subject: [PATCH 4/7] Conditionally apply ReadyPlayerMe synonyms --- .../model-serializers/src/GLTFSerializer.cpp | 31 +++++++++++++------ 1 file changed, 22 insertions(+), 9 deletions(-) diff --git a/libraries/model-serializers/src/GLTFSerializer.cpp b/libraries/model-serializers/src/GLTFSerializer.cpp index 1c1e45fbf3..dd9471f77b 100755 --- a/libraries/model-serializers/src/GLTFSerializer.cpp +++ b/libraries/model-serializers/src/GLTFSerializer.cpp @@ -1571,17 +1571,30 @@ bool GLTFSerializer::buildGeometry(HFMModel& hfmModel, const hifi::VariantHash& } } - // Augment list of blendshapes from synonyms in model. - QMap>::const_iterator synonym = READYPLAYERME_BLENDSHAPES_MAP.constBegin(); - while (synonym != READYPLAYERME_BLENDSHAPES_MAP.constEnd()) { - if (_file.meshes[node.mesh].extras.targetNames.contains(synonym.key())) { - auto blendshape = BLENDSHAPE_LOOKUP_MAP.find(synonym.value().first); - if (blendshape != BLENDSHAPE_LOOKUP_MAP.end()) { - blendshapeIndices.insert(synonym.key(), - WeightedIndex(blendshape.value(), synonym.value().second)); + // If an FST isn't being used and the model is likely from ReadyPlayerMe, add blendshape synonyms. + auto fileTargetNames = _file.meshes[node.mesh].extras.targetNames; + bool likelyReadyPlayerMeFile = + fileTargetNames.contains("browOuterUpLeft") + && fileTargetNames.contains("browInnerUp") + && fileTargetNames.contains("browDownLeft") + && fileTargetNames.contains("eyeBlinkLeft") + && fileTargetNames.contains("eyeWideLeft") + && fileTargetNames.contains("mouthLeft") + && fileTargetNames.contains("viseme_O") + && fileTargetNames.contains("mouthShrugLower"); + if (fileTargetNames.count() == 0 && likelyReadyPlayerMeFile) { + QMap>::const_iterator synonym + = READYPLAYERME_BLENDSHAPES_MAP.constBegin(); + while (synonym != READYPLAYERME_BLENDSHAPES_MAP.constEnd()) { + if (fileTargetNames.contains(synonym.key())) { + auto blendshape = BLENDSHAPE_LOOKUP_MAP.find(synonym.value().first); + if (blendshape != BLENDSHAPE_LOOKUP_MAP.end()) { + blendshapeIndices.insert(synonym.key(), + WeightedIndex(blendshape.value(), synonym.value().second)); + } } + ++synonym; } - ++synonym; } // Create blendshapes. From e9fa76ae6e58c0be6e015e55ea7657268cd45d74 Mon Sep 17 00:00:00 2001 From: David Rowe Date: Sat, 24 Apr 2021 08:49:57 +1200 Subject: [PATCH 5/7] Fix conditional application --- libraries/model-serializers/src/GLTFSerializer.cpp | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/libraries/model-serializers/src/GLTFSerializer.cpp b/libraries/model-serializers/src/GLTFSerializer.cpp index dd9471f77b..adacf942ff 100755 --- a/libraries/model-serializers/src/GLTFSerializer.cpp +++ b/libraries/model-serializers/src/GLTFSerializer.cpp @@ -1582,7 +1582,7 @@ bool GLTFSerializer::buildGeometry(HFMModel& hfmModel, const hifi::VariantHash& && fileTargetNames.contains("mouthLeft") && fileTargetNames.contains("viseme_O") && fileTargetNames.contains("mouthShrugLower"); - if (fileTargetNames.count() == 0 && likelyReadyPlayerMeFile) { + if (blendshapeMappings.count() == 0 && likelyReadyPlayerMeFile) { QMap>::const_iterator synonym = READYPLAYERME_BLENDSHAPES_MAP.constBegin(); while (synonym != READYPLAYERME_BLENDSHAPES_MAP.constEnd()) { From 24ddbc0d56d795500def24352ff41f4524caa44e Mon Sep 17 00:00:00 2001 From: David Rowe Date: Sat, 24 Apr 2021 09:10:32 +1200 Subject: [PATCH 6/7] Update ReadyPlayerMe default mapping --- libraries/shared/src/BlendshapeConstants.cpp | 49 ++++++++++++++------ 1 file changed, 34 insertions(+), 15 deletions(-) diff --git a/libraries/shared/src/BlendshapeConstants.cpp b/libraries/shared/src/BlendshapeConstants.cpp index 39641851b6..6e28f5079c 100644 --- a/libraries/shared/src/BlendshapeConstants.cpp +++ b/libraries/shared/src/BlendshapeConstants.cpp @@ -86,19 +86,38 @@ const QMap BLENDSHAPE_LOOKUP_MAP = [] { }(); const QMap> READYPLAYERME_BLENDSHAPES_MAP = { - // ReadyPlayerMe blendshape synonyms. - {"browDownLeft", {"BrowsD_L", 0.5f}}, - {"browDownRight", {"BrowsD_R", 0.5f}}, - {"browInnerUp", {"BrowsU_C", 0.3f}}, - {"browOuterUpLeft", {"BrowsU_L", 0.3f}}, - {"browOuterUpRight", {"BrowsU_R", 0.3f}}, - {"eyeBlinkLeft", {"EyeBlink_L", 1.0f}}, - {"eyeBlinkRight", {"EyeBlink_R", 1.0f}}, - {"eyeWideLeft", {"EyeOpen_L", 1.0f}}, - {"eyeWideRight", {"EyeOpen_R", 1.0f}}, - {"mouthLeft", {"MouthSmile_L", 0.6f}}, - {"mouthOpen", {"JawOpen", 1.0f}}, - {"mouthRight", {"MouthSmile_R", 0.6f}}, - {"mouthShrugLower", {"LipsUpperClose", 0.1f}}, - {"viseme_O", {"LipsFunnel", 0.5f}} + // ReadyPlayerMe blendshape default mapping. + { "mouthOpen", { "JawOpen", 1.0f } }, + { "eyeBlinkLeft", { "EyeBlink_L", 1.0f } }, + { "eyeBlinkRight", { "EyeBlink_R", 1.0f } }, + { "eyeSquintLeft", { "EyeSquint_L", 1.0f } }, + { "eyeSquintRight", { "EyeSquint_R", 1.0f } }, + { "eyeWideLeft", { "EyeOpen_L", 1.0f } }, + { "eyeWideRight", { "EyeOpen_R", 1.0f } }, + { "browDownLeft", { "BrowsD_L", 1.0f } }, + { "browDownRight", { "BrowsD_R", 1.0f } }, + { "browInnerUp", { "BrowsU_C", 1.0f } }, + { "browOuterUpLeft", { "BrowsU_L", 1.0f } }, + { "browOuterUpRight", { "BrowsU_R", 1.0f } }, + { "mouthFrownLeft", { "MouthFrown_L", 1.0f } }, + { "mouthFrownRight", { "MouthFrown_R", 1.0f } }, + { "mouthPucker", { "LipsPucker", 1.0f } }, + { "jawForward", { "JawFwd", 1.0f } }, + { "jawLeft", { "JawLeft", 1.0f } }, + { "jawRight", { "JawRight", 1.0f } }, + { "mouthLeft", { "MouthLeft", 1.0f } }, + { "mouthRight", { "MouthRight", 1.0f } }, + { "noseSneerLeft", { "NoseSneer_L", 1.0f } }, + { "noseSneerRight", { "NoseSneer_R", 1.0f } }, + { "mouthLowerDownLeft", { "MouthLowerDown_L", 1.0f } }, + { "mouthLowerDownRight", { "MouthLowerDown_R", 1.0f } }, + { "mouthShrugLower", { "MouthShrugLower", 1.0f } }, + { "mouthShrugUpper", { "MouthShrugUpper", 1.0f } }, + { "viseme_sil", { "MouthClose", 1.0f } }, + { "mouthSmile", { "MouthSmile_L", 1.0f } }, + { "mouthSmile", { "MouthSmile_R", 1.0f } }, + { "viseme_CH", { "LipsFunnel", 1.0f } }, + { "viseme_PP", { "LipsUpperClose", 1.0f } }, + { "mouthShrugLower", { "LipsLowerClose", 1.0f } }, + { "viseme_FF", { "Puff", 1.0f } } }; From 58a6ac384e3253a7f9c52f4097406f215aa23664 Mon Sep 17 00:00:00 2001 From: David Rowe Date: Thu, 29 Apr 2021 08:15:37 +1200 Subject: [PATCH 7/7] Replace some QMap occurrences with QHash --- libraries/model-serializers/src/GLTFSerializer.cpp | 2 +- libraries/shared/src/BlendshapeConstants.cpp | 6 +++--- libraries/shared/src/BlendshapeConstants.h | 6 +++--- 3 files changed, 7 insertions(+), 7 deletions(-) diff --git a/libraries/model-serializers/src/GLTFSerializer.cpp b/libraries/model-serializers/src/GLTFSerializer.cpp index d957d7ae77..9b0145c112 100755 --- a/libraries/model-serializers/src/GLTFSerializer.cpp +++ b/libraries/model-serializers/src/GLTFSerializer.cpp @@ -1583,7 +1583,7 @@ bool GLTFSerializer::buildGeometry(HFMModel& hfmModel, const hifi::VariantHash& && fileTargetNames.contains("viseme_O") && fileTargetNames.contains("mouthShrugLower"); if (blendshapeMappings.count() == 0 && likelyReadyPlayerMeFile) { - QMap>::const_iterator synonym + QHash>::const_iterator synonym = READYPLAYERME_BLENDSHAPES_MAP.constBegin(); while (synonym != READYPLAYERME_BLENDSHAPES_MAP.constEnd()) { if (fileTargetNames.contains(synonym.key())) { diff --git a/libraries/shared/src/BlendshapeConstants.cpp b/libraries/shared/src/BlendshapeConstants.cpp index 6e28f5079c..01f8c2db19 100644 --- a/libraries/shared/src/BlendshapeConstants.cpp +++ b/libraries/shared/src/BlendshapeConstants.cpp @@ -77,15 +77,15 @@ const char* BLENDSHAPE_NAMES[] = { "" }; -const QMap BLENDSHAPE_LOOKUP_MAP = [] { - QMap toReturn; +const QHash BLENDSHAPE_LOOKUP_MAP = [] { + QHash toReturn; for (int i = 0; i < (int)Blendshapes::BlendshapeCount; i++) { toReturn[BLENDSHAPE_NAMES[i]] = i; } return toReturn; }(); -const QMap> READYPLAYERME_BLENDSHAPES_MAP = { +const QHash> READYPLAYERME_BLENDSHAPES_MAP = { // ReadyPlayerMe blendshape default mapping. { "mouthOpen", { "JawOpen", 1.0f } }, { "eyeBlinkLeft", { "EyeBlink_L", 1.0f } }, diff --git a/libraries/shared/src/BlendshapeConstants.h b/libraries/shared/src/BlendshapeConstants.h index b2a6630224..5dd1f0abbb 100644 --- a/libraries/shared/src/BlendshapeConstants.h +++ b/libraries/shared/src/BlendshapeConstants.h @@ -12,15 +12,15 @@ #ifndef hifi_BlendshapeConstants_h #define hifi_BlendshapeConstants_h -#include +#include #include #include /// The names of the supported blendshapes, terminated with an empty string. extern const char* BLENDSHAPE_NAMES[]; -extern const QMap BLENDSHAPE_LOOKUP_MAP; -extern const QMap> READYPLAYERME_BLENDSHAPES_MAP; +extern const QHash BLENDSHAPE_LOOKUP_MAP; +extern const QHash> READYPLAYERME_BLENDSHAPES_MAP; enum class Blendshapes : int { EyeBlink_L = 0,