From dd678dbc497ec6b0adf09a0e70fed9cdce105648 Mon Sep 17 00:00:00 2001 From: David Rowe Date: Mon, 2 Mar 2015 19:05:13 -0800 Subject: [PATCH 1/2] Use all DDE coefficients to achieve better face control --- interface/src/devices/DdeFaceTracker.cpp | 183 ++++++++++++++++------- interface/src/devices/DdeFaceTracker.h | 4 +- 2 files changed, 128 insertions(+), 59 deletions(-) diff --git a/interface/src/devices/DdeFaceTracker.cpp b/interface/src/devices/DdeFaceTracker.cpp index 151afd98db..9ebe1ef718 100644 --- a/interface/src/devices/DdeFaceTracker.cpp +++ b/interface/src/devices/DdeFaceTracker.cpp @@ -17,15 +17,89 @@ #include #include "DdeFaceTracker.h" +#include "FaceshiftConstants.h" static const QHostAddress DDE_FEATURE_POINT_SERVER_ADDR("127.0.0.1"); static const quint16 DDE_FEATURE_POINT_SERVER_PORT = 5555; -static const int NUM_EXPRESSION = 46; -static const int MIN_PACKET_SIZE = (8 + NUM_EXPRESSION) * sizeof(float) + sizeof(int); +static const int NUM_EXPRESSIONS = 46; +static const int MIN_PACKET_SIZE = (8 + NUM_EXPRESSIONS) * sizeof(float) + sizeof(int); static const int MAX_NAME_SIZE = 31; -struct Packet{ +// There's almost but not quite a 1-1 correspondence between DDE's 46 and Faceshift 1.3's 48 packets. +// The best guess at mapping is to: +// - Swap L and R values +// - Skip two Faceshift values: JawChew (22) and LipsLowerDown (37) +static const int DDE_TO_FACESHIFT_MAPPING[] = { + 1, 0, 3, 2, 5, 4, 7, 6, 9, 8, 11, 10, 13, 12, 15, 14, + 16, + 18, 17, + 19, + 23, + 21, + // Skip JawChew + 20, + 25, 24, 27, 26, 29, 28, 31, 30, 33, 32, + 34, 35, 36, + // Skip LipsLowerDown + 38, 39, 40, 41, 42, 43, 44, 45, + 47, 46 +}; + +// The DDE coefficients, overall, range from -0.2 to 1.5 or so. However, individual coefficients typically vary much +// less than this. +static const float DDE_COEFFICIENT_SCALES[] = { + 4.0f, // EyeBlink_L + 4.0f, // EyeBlink_R + 1.0f, // EyeSquint_L + 1.0f, // EyeSquint_R + 1.0f, // EyeDown_L + 1.0f, // EyeDown_R + 1.0f, // EyeIn_L + 1.0f, // EyeIn_R + 4.0f, // EyeOpen_L + 4.0f, // EyeOpen_R + 1.0f, // EyeOut_L + 1.0f, // EyeOut_R + 1.0f, // EyeUp_L + 1.0f, // EyeUp_R + 3.0f, // BrowsD_L + 3.0f, // BrowsD_R + 3.0f, // BrowsU_C + 3.0f, // BrowsU_L + 3.0f, // BrowsU_R + 1.0f, // JawFwd + 1.5f, // JawLeft + 1.8f, // JawOpen + 1.0f, // JawChew + 1.5f, // JawRight + 1.5f, // MouthLeft + 1.5f, // MouthRight + 1.5f, // MouthFrown_L + 1.5f, // MouthFrown_R + 1.5f, // MouthSmile_L + 1.5f, // MouthSmile_R + 1.0f, // MouthDimple_L + 1.0f, // MouthDimple_R + 1.0f, // LipsStretch_L + 1.0f, // LipsStretch_R + 1.0f, // LipsUpperClose + 1.0f, // LipsLowerClose + 1.0f, // LipsUpperUp + 1.0f, // LipsLowerDown + 1.0f, // LipsUpperOpen + 1.0f, // LipsLowerOpen + 2.5f, // LipsFunnel + 2.0f, // LipsPucker + 1.5f, // ChinLowerRaise + 1.5f, // ChinUpperRaise + 1.0f, // Sneer + 3.0f, // Puff + 1.0f, // CheekSquint_L + 1.0f // CheekSquint_R +}; + +struct Packet { //roughly in mm float focal_length[1]; float translation[3]; @@ -33,8 +107,9 @@ struct Packet{ //quaternion float rotation[4]; - //blendshape coefficients ranging between -0.2 and 1.5 - float expressions[NUM_EXPRESSION]; + // The DDE coefficients, overall, range from -0.2 to 1.5 or so. However, individual coefficients typically vary much + // less than this. + float expressions[NUM_EXPRESSIONS]; //avatar id selected on the UI int avatar_id; @@ -69,8 +144,10 @@ DdeFaceTracker::DdeFaceTracker(const QHostAddress& host, quint16 port) : _previousTranslation(glm::vec3()), _previousRotation(glm::quat()) { - _blendshapeCoefficients.resize(NUM_EXPRESSION); - _previousExpressions.resize(NUM_EXPRESSION); + _coefficients.resize(NUM_FACESHIFT_BLENDSHAPES); + _previousCoefficients.resize(NUM_FACESHIFT_BLENDSHAPES); + + _blendshapeCoefficients.resize(NUM_FACESHIFT_BLENDSHAPES); connect(&_udpSocket, SIGNAL(readyRead()), SLOT(readPendingDatagrams())); connect(&_udpSocket, SIGNAL(error(QAbstractSocket::SocketError)), SLOT(socketErrorOccurred(QAbstractSocket::SocketError))); @@ -172,66 +249,56 @@ void DdeFaceTracker::decodePacket(const QByteArray& buffer) { _headRotation = (rotation + _previousRotation) / 2.0f; _previousRotation = rotation; - // The DDE coefficients, overall, range from -0.2 to 1.5 or so. However, individual coefficients typically vary much - // less than this.packet.expressions[1] - - // Eye blendshapes - static const float RELAXED_EYE_VALUE = 0.2f; - static const float EYE_OPEN_SCALE = 4.0f; - static const float EYE_BLINK_SCALE = 2.5f; - float leftEye = (packet.expressions[1] + _previousExpressions[1]) / 2.0f - RELAXED_EYE_VALUE; - float rightEye = (packet.expressions[0] + _previousExpressions[0]) / 2.0f - RELAXED_EYE_VALUE; - if (leftEye > 0.0f) { - _blendshapeCoefficients[_leftBlinkIndex] = glm::clamp(EYE_BLINK_SCALE * leftEye, 0.0f, 1.0f); - _blendshapeCoefficients[_leftEyeOpenIndex] = 0.0f; - } else { - _blendshapeCoefficients[_leftBlinkIndex] = 0.0f; - _blendshapeCoefficients[_leftEyeOpenIndex] = glm::clamp(EYE_OPEN_SCALE * -leftEye, 0.0f, 1.0f); + // Translate DDE coefficients to Faceshift compatible coefficients + for (int i = 0; i < NUM_EXPRESSIONS; i += 1) { + _coefficients[DDE_TO_FACESHIFT_MAPPING[i]] = packet.expressions[i]; } - if (rightEye > 0.0f) { - _blendshapeCoefficients[_rightBlinkIndex] = glm::clamp(EYE_BLINK_SCALE * rightEye, 0.0f, 1.0f); - _blendshapeCoefficients[_rightEyeOpenIndex] = 0.0f; - } else { - _blendshapeCoefficients[_rightBlinkIndex] = 0.0f; - _blendshapeCoefficients[_rightEyeOpenIndex] = glm::clamp(EYE_OPEN_SCALE * -rightEye, 0.0f, 1.0f); - } - _previousExpressions[1] = packet.expressions[1]; - _previousExpressions[0] = packet.expressions[0]; - // Eyebrow blendshapes - static const float BROW_UP_SCALE = 3.0f; - static const float BROW_DOWN_SCALE = 3.0f; - float browCenter = (packet.expressions[17] + _previousExpressions[17]) / 2.0f; - if (browCenter > 0) { - float browUp = glm::clamp(BROW_UP_SCALE * browCenter, 0.0f, 1.0f); - _blendshapeCoefficients[_browUpCenterIndex] = browUp; - _blendshapeCoefficients[_browUpLeftIndex] = browUp; + // Use EyeBlink values to control both EyeBlink and EyeOpen + static const float RELAXED_EYE_VALUE = 0.1f; + float leftEye = (_coefficients[_leftBlinkIndex] + _previousCoefficients[_leftBlinkIndex]) / 2.0f; + float rightEye = (_coefficients[_rightBlinkIndex] + _previousCoefficients[_rightBlinkIndex]) / 2.0f; + if (leftEye > RELAXED_EYE_VALUE) { + _coefficients[_leftBlinkIndex] = leftEye - RELAXED_EYE_VALUE; + _coefficients[_leftEyeOpenIndex] = 0.0f; + } else { + _coefficients[_leftBlinkIndex] = 0.0f; + _coefficients[_leftEyeOpenIndex] = RELAXED_EYE_VALUE - leftEye; + } + if (rightEye > RELAXED_EYE_VALUE) { + _coefficients[_rightBlinkIndex] = rightEye - RELAXED_EYE_VALUE; + _coefficients[_rightEyeOpenIndex] = 0.0f; + } else { + _coefficients[_rightBlinkIndex] = 0.0f; + _coefficients[_rightEyeOpenIndex] = RELAXED_EYE_VALUE - rightEye; + } + + // Offset jaw open coefficient + static const float JAW_OPEN_THRESHOLD = 0.16f; + _coefficients[_jawOpenIndex] = _coefficients[_jawOpenIndex] - JAW_OPEN_THRESHOLD; + + // Offset smile coefficients + static const float SMILE_THRESHOLD = 0.18f; + _coefficients[_mouthSmileLeftIndex] = _coefficients[_mouthSmileLeftIndex] - SMILE_THRESHOLD; + _coefficients[_mouthSmileRightIndex] = _coefficients[_mouthSmileRightIndex] - SMILE_THRESHOLD; + + + // Scale all coefficients + for (int i = 0; i < NUM_EXPRESSIONS; i += 1) { + _blendshapeCoefficients[i] + = glm::clamp(DDE_COEFFICIENT_SCALES[i] * (_coefficients[i] + _previousCoefficients[i]) / 2.0f, 0.0f, 1.0f); + _previousCoefficients[i] = _coefficients[i]; + } + + // Postcondition brow coefficients + if (_blendshapeCoefficients[_browUpCenterIndex] > 0) { _blendshapeCoefficients[_browDownLeftIndex] = 0.0f; - _blendshapeCoefficients[_browUpRightIndex] = browUp; _blendshapeCoefficients[_browDownRightIndex] = 0.0f; } else { - float browDown = glm::clamp(BROW_DOWN_SCALE * -browCenter, 0.0f, 1.0f); _blendshapeCoefficients[_browUpCenterIndex] = 0.0f; _blendshapeCoefficients[_browUpLeftIndex] = 0.0f; - _blendshapeCoefficients[_browDownLeftIndex] = browDown; _blendshapeCoefficients[_browUpRightIndex] = 0.0f; - _blendshapeCoefficients[_browDownRightIndex] = browDown; } - _previousExpressions[17] = packet.expressions[17]; - - // Mouth blendshapes - static const float JAW_OPEN_THRESHOLD = 0.16f; - static const float JAW_OPEN_SCALE = 1.4f; - static const float SMILE_THRESHOLD = 0.18f; - static const float SMILE_SCALE = 1.5f; - float smileLeft = (packet.expressions[24] + _previousExpressions[24]) / 2.0f; - float smileRight = (packet.expressions[23] + _previousExpressions[23]) / 2.0f; - _blendshapeCoefficients[_jawOpenIndex] = glm::clamp(JAW_OPEN_SCALE * (packet.expressions[21] - JAW_OPEN_THRESHOLD), - 0.0f, 1.0f); - _blendshapeCoefficients[_mouthSmileLeftIndex] = glm::clamp(SMILE_SCALE * (smileLeft - SMILE_THRESHOLD), 0.0f, 1.0f); - _blendshapeCoefficients[_mouthSmileRightIndex] = glm::clamp(SMILE_SCALE * (smileRight - SMILE_THRESHOLD), 0.0f, 1.0f); - _previousExpressions[24] = packet.expressions[24]; - _previousExpressions[23] = packet.expressions[23]; } else { qDebug() << "[Error] DDE Face Tracker Decode Error"; diff --git a/interface/src/devices/DdeFaceTracker.h b/interface/src/devices/DdeFaceTracker.h index cd63639f76..3c95666a6c 100644 --- a/interface/src/devices/DdeFaceTracker.h +++ b/interface/src/devices/DdeFaceTracker.h @@ -89,10 +89,12 @@ private: int _jawOpenIndex; + QVector _coefficients; + // Previous values for simple smoothing glm::vec3 _previousTranslation; glm::quat _previousRotation; - QVector _previousExpressions; + QVector _previousCoefficients; }; #endif // hifi_DdeFaceTracker_h \ No newline at end of file From 1b34b80b1a53555fc975acecc30ce05c9de94d41 Mon Sep 17 00:00:00 2001 From: David Rowe Date: Tue, 3 Mar 2015 11:05:59 -0800 Subject: [PATCH 2/2] Use BrowsC_U to control both brows' up and down --- interface/src/devices/DdeFaceTracker.cpp | 16 ++++++---------- 1 file changed, 6 insertions(+), 10 deletions(-) diff --git a/interface/src/devices/DdeFaceTracker.cpp b/interface/src/devices/DdeFaceTracker.cpp index 9ebe1ef718..9d753d228d 100644 --- a/interface/src/devices/DdeFaceTracker.cpp +++ b/interface/src/devices/DdeFaceTracker.cpp @@ -273,6 +273,12 @@ void DdeFaceTracker::decodePacket(const QByteArray& buffer) { _coefficients[_rightEyeOpenIndex] = RELAXED_EYE_VALUE - rightEye; } + // Use BrowsU_C to control both brows' up and down + _coefficients[_browDownLeftIndex] = -_coefficients[_browUpCenterIndex]; + _coefficients[_browDownRightIndex] = -_coefficients[_browUpCenterIndex]; + _coefficients[_browUpLeftIndex] = _coefficients[_browUpCenterIndex]; + _coefficients[_browUpRightIndex] = _coefficients[_browUpCenterIndex]; + // Offset jaw open coefficient static const float JAW_OPEN_THRESHOLD = 0.16f; _coefficients[_jawOpenIndex] = _coefficients[_jawOpenIndex] - JAW_OPEN_THRESHOLD; @@ -290,16 +296,6 @@ void DdeFaceTracker::decodePacket(const QByteArray& buffer) { _previousCoefficients[i] = _coefficients[i]; } - // Postcondition brow coefficients - if (_blendshapeCoefficients[_browUpCenterIndex] > 0) { - _blendshapeCoefficients[_browDownLeftIndex] = 0.0f; - _blendshapeCoefficients[_browDownRightIndex] = 0.0f; - } else { - _blendshapeCoefficients[_browUpCenterIndex] = 0.0f; - _blendshapeCoefficients[_browUpLeftIndex] = 0.0f; - _blendshapeCoefficients[_browUpRightIndex] = 0.0f; - } - } else { qDebug() << "[Error] DDE Face Tracker Decode Error"; }