Merge pull request #1183 from ctrlaltdavid/feature/rpm-blendshapes

ReadyPlayerMe blendshapes support

libraries/model-serializers/src/GLTFSerializer.cpp

@@ -1550,7 +1550,6 @@ bool GLTFSerializer::buildGeometry(HFMModel& hfmModel, const hifi::VariantHash&
                     typedef QPair<int, float> WeightedIndex;
                     hifi::VariantHash blendshapeMappings = mapping.value("bs").toHash();
                     QMultiHash<QString, WeightedIndex> blendshapeIndices;
-
                     for (int i = 0;; ++i) {
                         auto blendshapeName = QString(BLENDSHAPE_NAMES[i]);
                         if (blendshapeName.isEmpty()) {
@@ -1572,6 +1571,32 @@ bool GLTFSerializer::buildGeometry(HFMModel& hfmModel, const hifi::VariantHash&
                         }
                     }
 
+                    // If an FST isn't being used and the model is likely from ReadyPlayerMe, add blendshape synonyms.
+                    auto fileTargetNames = _file.meshes[node.mesh].extras.targetNames;
+                    bool likelyReadyPlayerMeFile =
+                           fileTargetNames.contains("browOuterUpLeft")
+                        && fileTargetNames.contains("browInnerUp")
+                        && fileTargetNames.contains("browDownLeft")
+                        && fileTargetNames.contains("eyeBlinkLeft")
+                        && fileTargetNames.contains("eyeWideLeft")
+                        && fileTargetNames.contains("mouthLeft")
+                        && fileTargetNames.contains("viseme_O")
+                        && fileTargetNames.contains("mouthShrugLower");
+                    if (blendshapeMappings.count() == 0 && likelyReadyPlayerMeFile) {
+                        QHash<QString, QPair<QString, float>>::const_iterator synonym 
+                            = READYPLAYERME_BLENDSHAPES_MAP.constBegin();
+                        while (synonym != READYPLAYERME_BLENDSHAPES_MAP.constEnd()) {
+                            if (fileTargetNames.contains(synonym.key())) {
+                                auto blendshape = BLENDSHAPE_LOOKUP_MAP.find(synonym.value().first);
+                                if (blendshape != BLENDSHAPE_LOOKUP_MAP.end()) {
+                                    blendshapeIndices.insert(synonym.key(),
+                                        WeightedIndex(blendshape.value(), synonym.value().second));
+                                }
+                            }
+                            ++synonym;
+                        }
+                    }
+
                     // Create blendshapes.
                     if (!blendshapeIndices.isEmpty()) {
                         mesh.blendshapes.resize((int)Blendshapes::BlendshapeCount);

libraries/shared/src/BlendshapeConstants.cpp

@@ -77,10 +77,47 @@ const char* BLENDSHAPE_NAMES[] = {
     ""
 };
 
-const QMap<QString, int> BLENDSHAPE_LOOKUP_MAP = [] {
+const QHash<QString, int> BLENDSHAPE_LOOKUP_MAP = [] {
-    QMap<QString, int> toReturn;
+    QHash<QString, int> toReturn;
     for (int i = 0; i < (int)Blendshapes::BlendshapeCount; i++) {
         toReturn[BLENDSHAPE_NAMES[i]] = i;
     }
     return toReturn;
 }();
+
+const QHash<QString, QPair<QString, float>> READYPLAYERME_BLENDSHAPES_MAP = {
+    // ReadyPlayerMe blendshape default mapping.
+    { "mouthOpen", { "JawOpen", 1.0f } },
+    { "eyeBlinkLeft", { "EyeBlink_L", 1.0f } },
+    { "eyeBlinkRight", { "EyeBlink_R", 1.0f } },
+    { "eyeSquintLeft", { "EyeSquint_L", 1.0f } },
+    { "eyeSquintRight", { "EyeSquint_R", 1.0f } },
+    { "eyeWideLeft", { "EyeOpen_L", 1.0f } },
+    { "eyeWideRight", { "EyeOpen_R", 1.0f } },
+    { "browDownLeft", { "BrowsD_L", 1.0f } },
+    { "browDownRight", { "BrowsD_R", 1.0f } },
+    { "browInnerUp", { "BrowsU_C", 1.0f } },
+    { "browOuterUpLeft", { "BrowsU_L", 1.0f } },
+    { "browOuterUpRight", { "BrowsU_R", 1.0f } },
+    { "mouthFrownLeft", { "MouthFrown_L", 1.0f } },
+    { "mouthFrownRight", { "MouthFrown_R", 1.0f } },
+    { "mouthPucker", { "LipsPucker", 1.0f } },
+    { "jawForward", { "JawFwd", 1.0f } },
+    { "jawLeft", { "JawLeft", 1.0f } },
+    { "jawRight", { "JawRight", 1.0f } },
+    { "mouthLeft", { "MouthLeft", 1.0f } },
+    { "mouthRight", { "MouthRight", 1.0f } },
+    { "noseSneerLeft", { "NoseSneer_L", 1.0f } },
+    { "noseSneerRight", { "NoseSneer_R", 1.0f } },
+    { "mouthLowerDownLeft", { "MouthLowerDown_L", 1.0f } },
+    { "mouthLowerDownRight", { "MouthLowerDown_R", 1.0f } },
+    { "mouthShrugLower", { "MouthShrugLower", 1.0f } },
+    { "mouthShrugUpper", { "MouthShrugUpper", 1.0f } },
+    { "viseme_sil", { "MouthClose", 1.0f } },
+    { "mouthSmile", { "MouthSmile_L", 1.0f } },
+    { "mouthSmile", { "MouthSmile_R", 1.0f } },
+    { "viseme_CH", { "LipsFunnel", 1.0f } },
+    { "viseme_PP", { "LipsUpperClose", 1.0f } },
+    { "mouthShrugLower", { "LipsLowerClose", 1.0f } },
+    { "viseme_FF", { "Puff", 1.0f } }
+};

libraries/shared/src/BlendshapeConstants.h

@@ -12,14 +12,15 @@
 #ifndef hifi_BlendshapeConstants_h
 #define hifi_BlendshapeConstants_h
 
-#include <QMap>
+#include <QHash>
 #include <QString>
 
 #include <glm/glm.hpp>
 
 /// The names of the supported blendshapes, terminated with an empty string.
 extern const char* BLENDSHAPE_NAMES[];
-extern const QMap<QString, int> BLENDSHAPE_LOOKUP_MAP;
+extern const QHash<QString, int> BLENDSHAPE_LOOKUP_MAP;
+extern const QHash<QString, QPair<QString, float>> READYPLAYERME_BLENDSHAPES_MAP;
 
 enum class Blendshapes : int {
     EyeBlink_L = 0,
@@ -87,16 +88,6 @@ enum class Blendshapes : int {
     BlendshapeCount
 };
 
-enum class LegacyBlendshpaes : int {
-    JawChew,  // not in ARKit
-    LipsUpperUp, // split in ARKit
-    LipsLowerDown, // split in ARKit
-    ChinLowerRaise, // not in ARKit
-    ChinUpperRaise, // not in ARKit
-    Sneer, // split in ARKit
-    LegacyBlendshapeCount
-};
-
 // Original blendshapes were per Faceshift.
 
 // NEW in ARKit