Merge pull request #4368 from ctrlaltdavid/20377

CR for 20377

interface/src/devices/DdeFaceTracker.cpp

@@ -17,15 +17,89 @@
 #include <QElapsedTimer>
 
 #include "DdeFaceTracker.h"
+#include "FaceshiftConstants.h"
 
 static const QHostAddress DDE_FEATURE_POINT_SERVER_ADDR("127.0.0.1");
 static const quint16 DDE_FEATURE_POINT_SERVER_PORT = 5555;
 
-static const int NUM_EXPRESSION = 46;
+static const int NUM_EXPRESSIONS = 46;
-static const int MIN_PACKET_SIZE = (8 + NUM_EXPRESSION) * sizeof(float) + sizeof(int);
+static const int MIN_PACKET_SIZE = (8 + NUM_EXPRESSIONS) * sizeof(float) + sizeof(int);
 static const int MAX_NAME_SIZE = 31;
 
-struct Packet{
+// There's almost but not quite a 1-1 correspondence between DDE's 46 and Faceshift 1.3's 48 packets.
+// The best guess at mapping is to:
+// - Swap L and R values
+// - Skip two Faceshift values: JawChew (22) and LipsLowerDown (37)
+static const int DDE_TO_FACESHIFT_MAPPING[] = {
+    1, 0, 3, 2, 5, 4, 7, 6, 9, 8, 11, 10, 13, 12, 15, 14,
+    16,
+    18, 17,
+    19,
+    23,
+    21,
+    // Skip JawChew
+    20,
+    25, 24, 27, 26, 29, 28, 31, 30, 33, 32,
+    34, 35, 36,
+    // Skip LipsLowerDown
+    38, 39, 40, 41, 42, 43, 44, 45,
+    47, 46
+};
+
+// The DDE coefficients, overall, range from -0.2 to 1.5 or so. However, individual coefficients typically vary much 
+// less than this.
+static const float DDE_COEFFICIENT_SCALES[] = {
+    4.0f, // EyeBlink_L
+    4.0f, // EyeBlink_R
+    1.0f, // EyeSquint_L
+    1.0f, // EyeSquint_R
+    1.0f, // EyeDown_L
+    1.0f, // EyeDown_R
+    1.0f, // EyeIn_L
+    1.0f, // EyeIn_R
+    4.0f, // EyeOpen_L
+    4.0f, // EyeOpen_R
+    1.0f, // EyeOut_L
+    1.0f, // EyeOut_R
+    1.0f, // EyeUp_L
+    1.0f, // EyeUp_R
+    3.0f, // BrowsD_L
+    3.0f, // BrowsD_R
+    3.0f, // BrowsU_C
+    3.0f, // BrowsU_L
+    3.0f, // BrowsU_R
+    1.0f, // JawFwd
+    1.5f, // JawLeft
+    1.8f, // JawOpen
+    1.0f, // JawChew
+    1.5f, // JawRight
+    1.5f, // MouthLeft
+    1.5f, // MouthRight
+    1.5f, // MouthFrown_L
+    1.5f, // MouthFrown_R
+    1.5f, // MouthSmile_L
+    1.5f, // MouthSmile_R
+    1.0f, // MouthDimple_L
+    1.0f, // MouthDimple_R
+    1.0f, // LipsStretch_L
+    1.0f, // LipsStretch_R
+    1.0f, // LipsUpperClose
+    1.0f, // LipsLowerClose
+    1.0f, // LipsUpperUp
+    1.0f, // LipsLowerDown
+    1.0f, // LipsUpperOpen
+    1.0f, // LipsLowerOpen
+    2.5f, // LipsFunnel
+    2.0f, // LipsPucker
+    1.5f, // ChinLowerRaise
+    1.5f, // ChinUpperRaise
+    1.0f, // Sneer
+    3.0f, // Puff
+    1.0f, // CheekSquint_L
+    1.0f  // CheekSquint_R
+};
+
+struct Packet {
     //roughly in mm
     float focal_length[1];
     float translation[3];
@@ -33,8 +107,9 @@ struct Packet{
     //quaternion
     float rotation[4];
     
-    //blendshape coefficients ranging between -0.2 and 1.5
+    // The DDE coefficients, overall, range from -0.2 to 1.5 or so. However, individual coefficients typically vary much 
-    float expressions[NUM_EXPRESSION];
+    // less than this.
+    float expressions[NUM_EXPRESSIONS];
     
     //avatar id selected on the UI
     int avatar_id;
@@ -69,8 +144,10 @@ DdeFaceTracker::DdeFaceTracker(const QHostAddress& host, quint16 port) :
     _previousTranslation(glm::vec3()),
     _previousRotation(glm::quat())
 {
-    _blendshapeCoefficients.resize(NUM_EXPRESSION);
+    _coefficients.resize(NUM_FACESHIFT_BLENDSHAPES);
-    _previousExpressions.resize(NUM_EXPRESSION);
+    _previousCoefficients.resize(NUM_FACESHIFT_BLENDSHAPES);
+
+    _blendshapeCoefficients.resize(NUM_FACESHIFT_BLENDSHAPES);
     
     connect(&_udpSocket, SIGNAL(readyRead()), SLOT(readPendingDatagrams()));
     connect(&_udpSocket, SIGNAL(error(QAbstractSocket::SocketError)), SLOT(socketErrorOccurred(QAbstractSocket::SocketError)));
@@ -172,66 +249,52 @@ void DdeFaceTracker::decodePacket(const QByteArray& buffer) {
         _headRotation = (rotation + _previousRotation) / 2.0f;
         _previousRotation = rotation;
 
-        // The DDE coefficients, overall, range from -0.2 to 1.5 or so. However, individual coefficients typically vary much 
+        // Translate DDE coefficients to Faceshift compatible coefficients
-        // less than this.packet.expressions[1]
+        for (int i = 0; i < NUM_EXPRESSIONS; i += 1) {
-
+            _coefficients[DDE_TO_FACESHIFT_MAPPING[i]] = packet.expressions[i];
-        // Eye blendshapes
-        static const float RELAXED_EYE_VALUE = 0.2f;
-        static const float EYE_OPEN_SCALE = 4.0f;
-        static const float EYE_BLINK_SCALE = 2.5f;
-        float leftEye = (packet.expressions[1] + _previousExpressions[1]) / 2.0f - RELAXED_EYE_VALUE;
-        float rightEye = (packet.expressions[0] + _previousExpressions[0]) / 2.0f - RELAXED_EYE_VALUE;
-        if (leftEye > 0.0f) {
-            _blendshapeCoefficients[_leftBlinkIndex] = glm::clamp(EYE_BLINK_SCALE * leftEye, 0.0f, 1.0f);
-            _blendshapeCoefficients[_leftEyeOpenIndex] = 0.0f;
-        } else {
-            _blendshapeCoefficients[_leftBlinkIndex] = 0.0f;
-            _blendshapeCoefficients[_leftEyeOpenIndex] = glm::clamp(EYE_OPEN_SCALE * -leftEye, 0.0f, 1.0f);
         }
-        if (rightEye > 0.0f) {
-            _blendshapeCoefficients[_rightBlinkIndex] = glm::clamp(EYE_BLINK_SCALE * rightEye, 0.0f, 1.0f);
-            _blendshapeCoefficients[_rightEyeOpenIndex] = 0.0f;
-        } else {
-            _blendshapeCoefficients[_rightBlinkIndex] = 0.0f;
-            _blendshapeCoefficients[_rightEyeOpenIndex] = glm::clamp(EYE_OPEN_SCALE * -rightEye, 0.0f, 1.0f);
-        }
-        _previousExpressions[1] = packet.expressions[1];
-        _previousExpressions[0] = packet.expressions[0];
 
-        // Eyebrow blendshapes
+        // Use EyeBlink values to control both EyeBlink and EyeOpen
-        static const float BROW_UP_SCALE = 3.0f;
+        static const float RELAXED_EYE_VALUE = 0.1f;
-        static const float BROW_DOWN_SCALE = 3.0f;
+        float leftEye = (_coefficients[_leftBlinkIndex] + _previousCoefficients[_leftBlinkIndex]) / 2.0f;
-        float browCenter = (packet.expressions[17] + _previousExpressions[17]) / 2.0f;
+        float rightEye = (_coefficients[_rightBlinkIndex] + _previousCoefficients[_rightBlinkIndex]) / 2.0f;
-        if (browCenter > 0) {
+        if (leftEye > RELAXED_EYE_VALUE) {
-            float browUp = glm::clamp(BROW_UP_SCALE * browCenter, 0.0f, 1.0f);
+            _coefficients[_leftBlinkIndex] = leftEye - RELAXED_EYE_VALUE;
-            _blendshapeCoefficients[_browUpCenterIndex] = browUp;
+            _coefficients[_leftEyeOpenIndex] = 0.0f;
-            _blendshapeCoefficients[_browUpLeftIndex] = browUp;
-            _blendshapeCoefficients[_browDownLeftIndex] = 0.0f;
-            _blendshapeCoefficients[_browUpRightIndex] = browUp;
-            _blendshapeCoefficients[_browDownRightIndex] = 0.0f;
         } else {
-            float browDown = glm::clamp(BROW_DOWN_SCALE * -browCenter, 0.0f, 1.0f);
+            _coefficients[_leftBlinkIndex] = 0.0f;
-            _blendshapeCoefficients[_browUpCenterIndex] = 0.0f;
+            _coefficients[_leftEyeOpenIndex] = RELAXED_EYE_VALUE - leftEye;
-            _blendshapeCoefficients[_browUpLeftIndex] = 0.0f;
+        }
-            _blendshapeCoefficients[_browDownLeftIndex] = browDown;
+        if (rightEye > RELAXED_EYE_VALUE) {
-            _blendshapeCoefficients[_browUpRightIndex] = 0.0f;
+            _coefficients[_rightBlinkIndex] = rightEye - RELAXED_EYE_VALUE;
-            _blendshapeCoefficients[_browDownRightIndex] = browDown;
+            _coefficients[_rightEyeOpenIndex] = 0.0f;
+        } else {
+            _coefficients[_rightBlinkIndex] = 0.0f;
+            _coefficients[_rightEyeOpenIndex] = RELAXED_EYE_VALUE - rightEye;
         }
-        _previousExpressions[17] = packet.expressions[17];
 
-        // Mouth blendshapes
+        // Use BrowsU_C to control both brows' up and down
+        _coefficients[_browDownLeftIndex] = -_coefficients[_browUpCenterIndex];
+        _coefficients[_browDownRightIndex] = -_coefficients[_browUpCenterIndex];
+        _coefficients[_browUpLeftIndex] = _coefficients[_browUpCenterIndex];
+        _coefficients[_browUpRightIndex] = _coefficients[_browUpCenterIndex];
+
+        // Offset jaw open coefficient
         static const float JAW_OPEN_THRESHOLD = 0.16f;
-        static const float JAW_OPEN_SCALE = 1.4f;
+        _coefficients[_jawOpenIndex] = _coefficients[_jawOpenIndex] - JAW_OPEN_THRESHOLD;
+
+        // Offset smile coefficients
         static const float SMILE_THRESHOLD = 0.18f;
-        static const float SMILE_SCALE = 1.5f;
+        _coefficients[_mouthSmileLeftIndex] = _coefficients[_mouthSmileLeftIndex] - SMILE_THRESHOLD;
-        float smileLeft = (packet.expressions[24] + _previousExpressions[24]) / 2.0f;
+        _coefficients[_mouthSmileRightIndex] = _coefficients[_mouthSmileRightIndex] - SMILE_THRESHOLD;
-        float smileRight = (packet.expressions[23] + _previousExpressions[23]) / 2.0f;
+
-        _blendshapeCoefficients[_jawOpenIndex] = glm::clamp(JAW_OPEN_SCALE * (packet.expressions[21] - JAW_OPEN_THRESHOLD),
+
-            0.0f, 1.0f);
+        // Scale all coefficients
-        _blendshapeCoefficients[_mouthSmileLeftIndex] = glm::clamp(SMILE_SCALE * (smileLeft - SMILE_THRESHOLD), 0.0f, 1.0f);
+        for (int i = 0; i < NUM_EXPRESSIONS; i += 1) {
-        _blendshapeCoefficients[_mouthSmileRightIndex] = glm::clamp(SMILE_SCALE * (smileRight - SMILE_THRESHOLD), 0.0f, 1.0f);
+            _blendshapeCoefficients[i]
-        _previousExpressions[24] = packet.expressions[24];
+                = glm::clamp(DDE_COEFFICIENT_SCALES[i] * (_coefficients[i] + _previousCoefficients[i]) / 2.0f, 0.0f, 1.0f);
-        _previousExpressions[23] = packet.expressions[23];
+            _previousCoefficients[i] = _coefficients[i];
+        }
 
     } else {
         qDebug() << "[Error] DDE Face Tracker Decode Error";

interface/src/devices/DdeFaceTracker.h

@@ -89,10 +89,12 @@ private:
     
     int _jawOpenIndex;
 
+    QVector<float> _coefficients;
+
     // Previous values for simple smoothing
     glm::vec3 _previousTranslation;
     glm::quat _previousRotation;
-    QVector<float> _previousExpressions;
+    QVector<float> _previousCoefficients;
 };
 
 #endif // hifi_DdeFaceTracker_h