From 1d7420d0dc908cabca046ceba5b4b8c6836710bf Mon Sep 17 00:00:00 2001 From: David Rowe Date: Tue, 28 Apr 2015 12:53:11 -0700 Subject: [PATCH 1/2] Adapt DDE control of eyes to be predominantly open or closed --- interface/src/devices/DdeFaceTracker.cpp | 155 ++++++++++++++++------- interface/src/devices/DdeFaceTracker.h | 21 ++- 2 files changed, 127 insertions(+), 49 deletions(-) diff --git a/interface/src/devices/DdeFaceTracker.cpp b/interface/src/devices/DdeFaceTracker.cpp index be25c0794d..ece9667816 100644 --- a/interface/src/devices/DdeFaceTracker.cpp +++ b/interface/src/devices/DdeFaceTracker.cpp @@ -65,16 +65,16 @@ static const int DDE_TO_FACESHIFT_MAPPING[] = { // The DDE coefficients, overall, range from -0.2 to 1.5 or so. However, individual coefficients typically vary much // less than this. static const float DDE_COEFFICIENT_SCALES[] = { - 4.0f, // EyeBlink_L - 4.0f, // EyeBlink_R + 1.0f, // EyeBlink_L + 1.0f, // EyeBlink_R 1.0f, // EyeSquint_L 1.0f, // EyeSquint_R 1.0f, // EyeDown_L 1.0f, // EyeDown_R 1.0f, // EyeIn_L 1.0f, // EyeIn_R - 4.0f, // EyeOpen_L - 4.0f, // EyeOpen_R + 1.0f, // EyeOpen_L + 1.0f, // EyeOpen_R 1.0f, // EyeOut_L 1.0f, // EyeOut_R 1.0f, // EyeUp_L @@ -136,6 +136,16 @@ struct Packet { const float STARTING_DDE_MESSAGE_TIME = 0.033f; +const int FPS_TIMER_DELAY = 2000; // ms +const int FPS_TIMER_DURATION = 2000; // ms + +#ifdef WIN32 +// warning C4351: new behavior: elements of array 'DdeFaceTracker::_lastEyeBlinks' will be default initialized +// warning C4351: new behavior: elements of array 'DdeFaceTracker::_filteredEyeBlinks' will be default initialized +// warning C4351: new behavior: elements of array 'DdeFaceTracker::_lastEyeCoefficients' will be default initialized +#pragma warning(disable:4351) +#endif + DdeFaceTracker::DdeFaceTracker() : DdeFaceTracker(QHostAddress::Any, DDE_SERVER_PORT, DDE_CONTROL_PORT) { @@ -166,24 +176,33 @@ DdeFaceTracker::DdeFaceTracker(const QHostAddress& host, quint16 serverPort, qui _averageMessageTime(STARTING_DDE_MESSAGE_TIME), _lastHeadTranslation(glm::vec3(0.0f)), _filteredHeadTranslation(glm::vec3(0.0f)), - _lastLeftEyeBlink(0.0f), - _filteredLeftEyeBlink(0.0f), - _lastRightEyeBlink(0.0f), - _filteredRightEyeBlink(0.0f) + _lastEyeBlinks(), + _filteredEyeBlinks(), + _lastEyeCoefficients(), + _isCalculatingFPS(false), + _frameCount(0) { _coefficients.resize(NUM_FACESHIFT_BLENDSHAPES); _blendshapeCoefficients.resize(NUM_FACESHIFT_BLENDSHAPES); - + + _eyeStates[0] = EYE_OPEN; + _eyeStates[1] = EYE_OPEN; + connect(&_udpSocket, SIGNAL(readyRead()), SLOT(readPendingDatagrams())); connect(&_udpSocket, SIGNAL(error(QAbstractSocket::SocketError)), SLOT(socketErrorOccurred(QAbstractSocket::SocketError))); - connect(&_udpSocket, SIGNAL(stateChanged(QAbstractSocket::SocketState)), SLOT(socketStateChanged(QAbstractSocket::SocketState))); + connect(&_udpSocket, SIGNAL(stateChanged(QAbstractSocket::SocketState)), + SLOT(socketStateChanged(QAbstractSocket::SocketState))); } DdeFaceTracker::~DdeFaceTracker() { setEnabled(false); } +#ifdef WIN32 +#pragma warning(default:4351) +#endif + void DdeFaceTracker::setEnabled(bool enabled) { #ifdef HAVE_DDE // isOpen() does not work as one might expect on QUdpSocket; don't test isOpen() before closing socket. @@ -354,40 +373,6 @@ void DdeFaceTracker::decodePacket(const QByteArray& buffer) { _coefficients[DDE_TO_FACESHIFT_MAPPING[i]] = packet.expressions[i]; } - // Use EyeBlink values to control both EyeBlink and EyeOpen - static const float RELAXED_EYE_VALUE = 0.1f; - float leftEye = _coefficients[_leftBlinkIndex]; - float rightEye = _coefficients[_rightBlinkIndex]; - if (isFiltering) { - const float BLINK_VELOCITY_FILTER_STRENGTH = 0.3f; - - float velocity = fabs(leftEye - _lastLeftEyeBlink) / _averageMessageTime; - float velocityFilter = glm::clamp(velocity * BLINK_VELOCITY_FILTER_STRENGTH, 0.0f, 1.0f); - _filteredLeftEyeBlink = velocityFilter * leftEye + (1.0f - velocityFilter) * _filteredLeftEyeBlink; - _lastLeftEyeBlink = leftEye; - leftEye = _filteredLeftEyeBlink; - - velocity = fabs(rightEye - _lastRightEyeBlink) / _averageMessageTime; - velocityFilter = glm::clamp(velocity * BLINK_VELOCITY_FILTER_STRENGTH, 0.0f, 1.0f); - _filteredRightEyeBlink = velocityFilter * rightEye + (1.0f - velocityFilter) * _filteredRightEyeBlink; - _lastRightEyeBlink = rightEye; - rightEye = _filteredRightEyeBlink; - } - if (leftEye > RELAXED_EYE_VALUE) { - _coefficients[_leftBlinkIndex] = leftEye - RELAXED_EYE_VALUE; - _coefficients[_leftEyeOpenIndex] = 0.0f; - } else { - _coefficients[_leftBlinkIndex] = 0.0f; - _coefficients[_leftEyeOpenIndex] = RELAXED_EYE_VALUE - leftEye; - } - if (rightEye > RELAXED_EYE_VALUE) { - _coefficients[_rightBlinkIndex] = rightEye - RELAXED_EYE_VALUE; - _coefficients[_rightEyeOpenIndex] = 0.0f; - } else { - _coefficients[_rightBlinkIndex] = 0.0f; - _coefficients[_rightEyeOpenIndex] = RELAXED_EYE_VALUE - rightEye; - } - // Use BrowsU_C to control both brows' up and down _coefficients[_browDownLeftIndex] = -_coefficients[_browUpCenterIndex]; _coefficients[_browDownRightIndex] = -_coefficients[_browUpCenterIndex]; @@ -403,6 +388,88 @@ void DdeFaceTracker::decodePacket(const QByteArray& buffer) { _coefficients[_mouthSmileLeftIndex] = _coefficients[_mouthSmileLeftIndex] - SMILE_THRESHOLD; _coefficients[_mouthSmileRightIndex] = _coefficients[_mouthSmileRightIndex] - SMILE_THRESHOLD; + // Velocity filter EyeBlink values + const float DDE_EYEBLINK_SCALE = 3.0f; + float eyeBlinks[] = { DDE_EYEBLINK_SCALE * _coefficients[_leftBlinkIndex], DDE_EYEBLINK_SCALE * _coefficients[_rightBlinkIndex] }; + if (isFiltering) { + const float BLINK_VELOCITY_FILTER_STRENGTH = 0.3f; + for (int i = 0; i < 2; i += 1) { + float velocity = fabs(eyeBlinks[i] - _lastEyeBlinks[i]) / _averageMessageTime; + float velocityFilter = glm::clamp(velocity * BLINK_VELOCITY_FILTER_STRENGTH, 0.0f, 1.0f); + _filteredEyeBlinks[i] = velocityFilter * eyeBlinks[i] + (1.0f - velocityFilter) * _filteredEyeBlinks[i]; + _lastEyeBlinks[i] = eyeBlinks[i]; + } + } + + // Finesse EyeBlink values + float eyeCoefficients[2]; + for (int i = 0; i < 2; i += 1) { + // Scale EyeBlink values so that they can be used to control both EyeBlink and EyeOpen + // -ve values control EyeOpen; +ve values control EyeBlink + static const float EYE_CONTROL_THRESHOLD = 0.5f; // Resting eye value + eyeCoefficients[i] = (_filteredEyeBlinks[i] - EYE_CONTROL_THRESHOLD) / (1.0f - EYE_CONTROL_THRESHOLD); + + // Change to closing or opening states + const float EYE_CONTROL_HYSTERISIS = 0.25f; + const float EYE_CLOSING_THRESHOLD = 0.95f; + const float EYE_OPENING_THRESHOLD = EYE_CONTROL_THRESHOLD - EYE_CONTROL_HYSTERISIS; + if ((_eyeStates[i] == EYE_OPEN || _eyeStates[i] == EYE_OPENING) && eyeCoefficients[i] > EYE_CLOSING_THRESHOLD) { + _eyeStates[i] = EYE_CLOSING; + } else if ((_eyeStates[i] == EYE_CLOSED || _eyeStates[i] == EYE_CLOSING) + && eyeCoefficients[i] < EYE_OPENING_THRESHOLD) { + _eyeStates[i] = EYE_OPENING; + } + + const float EYELID_MOVEMENT_RATE = 10.0f; // units/second + const float EYE_OPEN_SCALE = 0.2f; + if (_eyeStates[i] == EYE_CLOSING) { + // Close eyelid until it's fully closed + float closingValue = _lastEyeCoefficients[i] + EYELID_MOVEMENT_RATE * _averageMessageTime; + if (closingValue >= 1.0) { + _eyeStates[i] = EYE_CLOSED; + eyeCoefficients[i] = 1.0; + } else { + eyeCoefficients[i] = closingValue; + } + } else if (_eyeStates[i] == EYE_OPENING) { + // Open eyelid until it meets the current adjusted value + float openingValue = _lastEyeCoefficients[i] - EYELID_MOVEMENT_RATE * _averageMessageTime; + if (openingValue < eyeCoefficients[i] * EYE_OPEN_SCALE) { + _eyeStates[i] = EYE_OPEN; + eyeCoefficients[i] = eyeCoefficients[i] * EYE_OPEN_SCALE; + } else { + eyeCoefficients[i] = openingValue; + } + } else if (_eyeStates[i] == EYE_OPEN) { + // Reduce eyelid movement + eyeCoefficients[i] = eyeCoefficients[i] * EYE_OPEN_SCALE; + } else if (_eyeStates[i] == EYE_CLOSED) { + // Keep eyelid fully closed + eyeCoefficients[i] = 1.0; + } + } + if (_eyeStates[0] == EYE_OPEN && _eyeStates[1] == EYE_OPEN) { + // Couple eyelids + eyeCoefficients[0] = eyeCoefficients[1] = (eyeCoefficients[0] + eyeCoefficients[0]) / 2.0f; + } + _lastEyeCoefficients[0] = eyeCoefficients[0]; + _lastEyeCoefficients[1] = eyeCoefficients[1]; + + // Use EyeBlink values to control both EyeBlink and EyeOpen + if (eyeCoefficients[0] > 0) { + _coefficients[_leftBlinkIndex] = eyeCoefficients[0]; + _coefficients[_leftEyeOpenIndex] = 0.0f; + } else { + _coefficients[_leftBlinkIndex] = 0.0f; + _coefficients[_leftEyeOpenIndex] = -eyeCoefficients[0]; + } + if (eyeCoefficients[1] > 0) { + _coefficients[_rightBlinkIndex] = eyeCoefficients[1]; + _coefficients[_rightEyeOpenIndex] = 0.0f; + } else { + _coefficients[_rightBlinkIndex] = 0.0f; + _coefficients[_rightEyeOpenIndex] = -eyeCoefficients[1]; + } // Scale all coefficients for (int i = 0; i < NUM_EXPRESSIONS; i += 1) { diff --git a/interface/src/devices/DdeFaceTracker.h b/interface/src/devices/DdeFaceTracker.h index d9df5723cf..7ae4a07fea 100644 --- a/interface/src/devices/DdeFaceTracker.h +++ b/interface/src/devices/DdeFaceTracker.h @@ -103,12 +103,23 @@ private: quint64 _lastMessageReceived; float _averageMessageTime; + glm::vec3 _lastHeadTranslation; glm::vec3 _filteredHeadTranslation; - float _lastLeftEyeBlink; - float _filteredLeftEyeBlink; - float _lastRightEyeBlink; - float _filteredRightEyeBlink; + + enum EyeState { + EYE_OPEN, + EYE_CLOSING, + EYE_CLOSED, + EYE_OPENING + }; + EyeState _eyeStates[2]; + float _lastEyeBlinks[2]; + float _filteredEyeBlinks[2]; + float _lastEyeCoefficients[2]; + + bool _isCalculatingFPS; + int _frameCount; }; -#endif // hifi_DdeFaceTracker_h \ No newline at end of file +#endif // hifi_DdeFaceTracker_h From 81e23eab28ae151f6ddd5f16bd222787663f7e29 Mon Sep 17 00:00:00 2001 From: David Rowe Date: Wed, 29 Apr 2015 22:15:49 -0700 Subject: [PATCH 2/2] Coding standard --- interface/src/devices/DdeFaceTracker.cpp | 8 ++++---- 1 file changed, 4 insertions(+), 4 deletions(-) diff --git a/interface/src/devices/DdeFaceTracker.cpp b/interface/src/devices/DdeFaceTracker.cpp index ece9667816..e28496335f 100644 --- a/interface/src/devices/DdeFaceTracker.cpp +++ b/interface/src/devices/DdeFaceTracker.cpp @@ -369,7 +369,7 @@ void DdeFaceTracker::decodePacket(const QByteArray& buffer) { } // Translate DDE coefficients to Faceshift compatible coefficients - for (int i = 0; i < NUM_EXPRESSIONS; i += 1) { + for (int i = 0; i < NUM_EXPRESSIONS; i++) { _coefficients[DDE_TO_FACESHIFT_MAPPING[i]] = packet.expressions[i]; } @@ -393,7 +393,7 @@ void DdeFaceTracker::decodePacket(const QByteArray& buffer) { float eyeBlinks[] = { DDE_EYEBLINK_SCALE * _coefficients[_leftBlinkIndex], DDE_EYEBLINK_SCALE * _coefficients[_rightBlinkIndex] }; if (isFiltering) { const float BLINK_VELOCITY_FILTER_STRENGTH = 0.3f; - for (int i = 0; i < 2; i += 1) { + for (int i = 0; i < 2; i++) { float velocity = fabs(eyeBlinks[i] - _lastEyeBlinks[i]) / _averageMessageTime; float velocityFilter = glm::clamp(velocity * BLINK_VELOCITY_FILTER_STRENGTH, 0.0f, 1.0f); _filteredEyeBlinks[i] = velocityFilter * eyeBlinks[i] + (1.0f - velocityFilter) * _filteredEyeBlinks[i]; @@ -403,7 +403,7 @@ void DdeFaceTracker::decodePacket(const QByteArray& buffer) { // Finesse EyeBlink values float eyeCoefficients[2]; - for (int i = 0; i < 2; i += 1) { + for (int i = 0; i < 2; i++) { // Scale EyeBlink values so that they can be used to control both EyeBlink and EyeOpen // -ve values control EyeOpen; +ve values control EyeBlink static const float EYE_CONTROL_THRESHOLD = 0.5f; // Resting eye value @@ -472,7 +472,7 @@ void DdeFaceTracker::decodePacket(const QByteArray& buffer) { } // Scale all coefficients - for (int i = 0; i < NUM_EXPRESSIONS; i += 1) { + for (int i = 0; i < NUM_EXPRESSIONS; i++) { _blendshapeCoefficients[i] = glm::clamp(DDE_COEFFICIENT_SCALES[i] * _coefficients[i], 0.0f, 1.0f); }