From 2264425f9fb865f699927d40b836d531e8338d6b Mon Sep 17 00:00:00 2001 From: luiscuenca Date: Thu, 28 Jun 2018 16:41:39 -0700 Subject: [PATCH 1/4] Fix eyelid behaviour --- .../src/avatars-renderer/Head.cpp | 56 ++++++++++++++----- libraries/avatars/src/HeadData.cpp | 38 +++++++------ libraries/avatars/src/HeadData.h | 4 ++ 3 files changed, 66 insertions(+), 32 deletions(-) diff --git a/libraries/avatars-renderer/src/avatars-renderer/Head.cpp b/libraries/avatars-renderer/src/avatars-renderer/Head.cpp index 5800c1404b..0d939dce82 100644 --- a/libraries/avatars-renderer/src/avatars-renderer/Head.cpp +++ b/libraries/avatars-renderer/src/avatars-renderer/Head.cpp @@ -220,30 +220,56 @@ void Head::calculateMouthShapes(float deltaTime) { void Head::applyEyelidOffset(glm::quat headOrientation) { // Adjusts the eyelid blendshape coefficients so that the eyelid follows the iris as the head pitches. - - if (disableEyelidAdjustment) { + bool isBlinking = (_rightEyeBlinkVelocity != 0.0f && _rightEyeBlinkVelocity != 0.0f); + if (disableEyelidAdjustment || isBlinking) { return; } + const std::vector eyeBlinkBlendShapes = { "EyeBlink_L", "EyeBlink_R" }; + const std::vector eyeOpenBlendShapes = { "EyeOpen_L", "EyeOpen_R" }; + const std::vector browsBlendShapes = { "BrowsU_L", "BrowsU_R" }; + + const float EYE_PITCH_TO_COEFFICIENT = 3.5f; // Empirically determined + const float MAX_EYELID_OFFSET = 1.5f; + const float BLINK_DOWN_MULTIPLIER = 0.25f; + const float OPEN_DOWN_MULTIPLIER = 0.3f; + const float BROW_UP_MULTIPLIER = 0.5f; + glm::quat eyeRotation = rotationBetween(headOrientation * IDENTITY_FORWARD, getLookAtPosition() - _eyePosition); eyeRotation = eyeRotation * glm::angleAxis(safeEulerAngles(headOrientation).y, IDENTITY_UP); // Rotation w.r.t. head float eyePitch = safeEulerAngles(eyeRotation).x; + float eyelidOffset = glm::clamp(abs(eyePitch * EYE_PITCH_TO_COEFFICIENT), 0.0f, MAX_EYELID_OFFSET); - const float EYE_PITCH_TO_COEFFICIENT = 1.6f; // Empirically determined - const float MAX_EYELID_OFFSET = 0.8f; // So that don't fully close eyes when looking way down - float eyelidOffset = glm::clamp(-eyePitch * EYE_PITCH_TO_COEFFICIENT, -1.0f, MAX_EYELID_OFFSET); + std::vector eyeBlinkIndices, eyeOpenIndices, browsIndices; - for (int i = 0; i < 2; i++) { - const int LEFT_EYE = 8; - float eyeCoefficient = _transientBlendshapeCoefficients[i] - _transientBlendshapeCoefficients[LEFT_EYE + i]; - eyeCoefficient = glm::clamp(eyelidOffset + eyeCoefficient * (1.0f - eyelidOffset), -1.0f, 1.0f); - if (eyeCoefficient > 0.0f) { - _transientBlendshapeCoefficients[i] = eyeCoefficient; - _transientBlendshapeCoefficients[LEFT_EYE + i] = 0.0f; + getBlendshapeIndices(eyeBlinkBlendShapes, eyeBlinkIndices); + getBlendshapeIndices(eyeOpenBlendShapes, eyeOpenIndices); + getBlendshapeIndices(browsBlendShapes, browsIndices); - } else { - _transientBlendshapeCoefficients[i] = 0.0f; - _transientBlendshapeCoefficients[LEFT_EYE + i] = -eyeCoefficient; + bool isLookingUp = (eyePitch > 0); + + for (auto& blinkIndex : eyeBlinkIndices) { + float lookingUpCoefficient = -eyelidOffset; + float lookingDownCoefficient = BLINK_DOWN_MULTIPLIER * eyelidOffset; + if (blinkIndex >= 0 && blinkIndex < _transientBlendshapeCoefficients.size()) { + _transientBlendshapeCoefficients[blinkIndex] = isLookingUp ? lookingUpCoefficient : lookingDownCoefficient; + } + + } + + for (auto& openIndex : eyeOpenIndices) { + float lookingUpCoefficient = eyelidOffset; + float lookingDownCoefficient = OPEN_DOWN_MULTIPLIER * eyelidOffset; + if (openIndex >= 0 && openIndex < _transientBlendshapeCoefficients.size()) { + _transientBlendshapeCoefficients[openIndex] = isLookingUp ? lookingUpCoefficient : lookingDownCoefficient; + } + } + + for (auto& browIndex : browsIndices) { + float lookingUpCoefficient = BROW_UP_MULTIPLIER * eyelidOffset; + float lookingDownCoefficient = 0.0f; + if (browIndex >= 0 && browIndex < _transientBlendshapeCoefficients.size()) { + _transientBlendshapeCoefficients[browIndex] = isLookingUp ? lookingUpCoefficient : lookingDownCoefficient; } } } diff --git a/libraries/avatars/src/HeadData.cpp b/libraries/avatars/src/HeadData.cpp index 4119d7a459..c68bb26e34 100644 --- a/libraries/avatars/src/HeadData.cpp +++ b/libraries/avatars/src/HeadData.cpp @@ -33,7 +33,7 @@ HeadData::HeadData(AvatarData* owningAvatar) : _summedBlendshapeCoefficients(QVector(0, 0.0f)), _owningAvatar(owningAvatar) { - + computeBlendshapesLookupMap(); } glm::quat HeadData::getRawOrientation() const { @@ -71,16 +71,10 @@ void HeadData::setOrientation(const glm::quat& orientation) { setHeadOrientation(orientation); } -//Lazily construct a lookup map from the blendshapes -static const QMap& getBlendshapesLookupMap() { - static std::once_flag once; - static QMap blendshapeLookupMap; - std::call_once(once, [&] { - for (int i = 0; i < NUM_FACESHIFT_BLENDSHAPES; i++) { - blendshapeLookupMap[FACESHIFT_BLENDSHAPES[i]] = i; - } - }); - return blendshapeLookupMap; +void HeadData::computeBlendshapesLookupMap(){ + for (int i = 0; i < NUM_FACESHIFT_BLENDSHAPES; i++) { + _blendshapeLookupMap[FACESHIFT_BLENDSHAPES[i]] = i; + } } int HeadData::getNumSummedBlendshapeCoefficients() const { @@ -108,11 +102,10 @@ const QVector& HeadData::getSummedBlendshapeCoefficients() { } void HeadData::setBlendshape(QString name, float val) { - const auto& blendshapeLookupMap = getBlendshapesLookupMap(); //Check to see if the named blendshape exists, and then set its value if it does - auto it = blendshapeLookupMap.find(name); - if (it != blendshapeLookupMap.end()) { + auto it = _blendshapeLookupMap.find(name); + if (it != _blendshapeLookupMap.end()) { if (_blendshapeCoefficients.size() <= it.value()) { _blendshapeCoefficients.resize(it.value() + 1); } @@ -123,6 +116,18 @@ void HeadData::setBlendshape(QString name, float val) { } } +int HeadData::getBlendshapeIndex(const QString& name) { + auto it = _blendshapeLookupMap.find(name); + int index = it != _blendshapeLookupMap.end() ? it.value() : -1; + return index; +} + +void HeadData::getBlendshapeIndices(const std::vector& blendShapeNames, std::vector& indexes) { + for (auto& name : blendShapeNames) { + indexes.push_back(getBlendshapeIndex(name)); + } +} + static const QString JSON_AVATAR_HEAD_ROTATION = QStringLiteral("rotation"); static const QString JSON_AVATAR_HEAD_BLENDSHAPE_COEFFICIENTS = QStringLiteral("blendShapes"); static const QString JSON_AVATAR_HEAD_LEAN_FORWARD = QStringLiteral("leanForward"); @@ -131,10 +136,9 @@ static const QString JSON_AVATAR_HEAD_LOOKAT = QStringLiteral("lookAt"); QJsonObject HeadData::toJson() const { QJsonObject headJson; - const auto& blendshapeLookupMap = getBlendshapesLookupMap(); QJsonObject blendshapesJson; - for (auto name : blendshapeLookupMap.keys()) { - auto index = blendshapeLookupMap[name]; + for (auto name : _blendshapeLookupMap.keys()) { + auto index = _blendshapeLookupMap[name]; float value = 0.0f; if (index < _blendshapeCoefficients.size()) { value += _blendshapeCoefficients[index]; diff --git a/libraries/avatars/src/HeadData.h b/libraries/avatars/src/HeadData.h index f9c4b52139..834629596a 100644 --- a/libraries/avatars/src/HeadData.h +++ b/libraries/avatars/src/HeadData.h @@ -55,6 +55,8 @@ public: void setOrientation(const glm::quat& orientation); void setBlendshape(QString name, float val); + int getBlendshapeIndex(const QString& name); + void getBlendshapeIndices(const std::vector& blendShapeNames, std::vector& indexes); const QVector& getBlendshapeCoefficients() const { return _blendshapeCoefficients; } const QVector& getSummedBlendshapeCoefficients(); int getNumSummedBlendshapeCoefficients() const; @@ -114,6 +116,7 @@ protected: QVector _blendshapeCoefficients; QVector _transientBlendshapeCoefficients; QVector _summedBlendshapeCoefficients; + QMap _blendshapeLookupMap; AvatarData* _owningAvatar; private: @@ -122,6 +125,7 @@ private: HeadData& operator= (const HeadData&); void setHeadOrientation(const glm::quat& orientation); + void computeBlendshapesLookupMap(); }; #endif // hifi_HeadData_h From 24d03ace4c71a686f2b5fc856977745b4a029da1 Mon Sep 17 00:00:00 2001 From: luiscuenca Date: Fri, 29 Jun 2018 14:34:17 -0700 Subject: [PATCH 2/4] Changes from CR and decreased looking at angle to mitigate freaky eyes --- libraries/animation/src/Rig.cpp | 4 +- .../src/avatars-renderer/Head.cpp | 53 ++++++++----------- libraries/avatars/src/HeadData.cpp | 1 - libraries/avatars/src/HeadData.h | 1 + libraries/shared/src/FaceshiftConstants.cpp | 15 ++++++ libraries/shared/src/FaceshiftConstants.h | 6 ++- 6 files changed, 44 insertions(+), 36 deletions(-) diff --git a/libraries/animation/src/Rig.cpp b/libraries/animation/src/Rig.cpp index 0833b28142..b10bba1c25 100644 --- a/libraries/animation/src/Rig.cpp +++ b/libraries/animation/src/Rig.cpp @@ -1434,9 +1434,9 @@ void Rig::updateEyeJoint(int index, const glm::vec3& modelTranslation, const glm glm::quat deltaQuat = desiredQuat * glm::inverse(headQuat); - // limit swing rotation of the deltaQuat by a 30 degree cone. + // limit swing rotation of the deltaQuat by a 25 degree cone. // TODO: use swing twist decomposition constraint instead, for off axis rotation clamping. - const float MAX_ANGLE = 30.0f * RADIANS_PER_DEGREE; + const float MAX_ANGLE = 25.0f * RADIANS_PER_DEGREE; if (fabsf(glm::angle(deltaQuat)) > MAX_ANGLE) { deltaQuat = glm::angleAxis(glm::clamp(glm::angle(deltaQuat), -MAX_ANGLE, MAX_ANGLE), glm::axis(deltaQuat)); } diff --git a/libraries/avatars-renderer/src/avatars-renderer/Head.cpp b/libraries/avatars-renderer/src/avatars-renderer/Head.cpp index 0d939dce82..b76e401761 100644 --- a/libraries/avatars-renderer/src/avatars-renderer/Head.cpp +++ b/libraries/avatars-renderer/src/avatars-renderer/Head.cpp @@ -225,10 +225,6 @@ void Head::applyEyelidOffset(glm::quat headOrientation) { return; } - const std::vector eyeBlinkBlendShapes = { "EyeBlink_L", "EyeBlink_R" }; - const std::vector eyeOpenBlendShapes = { "EyeOpen_L", "EyeOpen_R" }; - const std::vector browsBlendShapes = { "BrowsU_L", "BrowsU_R" }; - const float EYE_PITCH_TO_COEFFICIENT = 3.5f; // Empirically determined const float MAX_EYELID_OFFSET = 1.5f; const float BLINK_DOWN_MULTIPLIER = 0.25f; @@ -236,40 +232,33 @@ void Head::applyEyelidOffset(glm::quat headOrientation) { const float BROW_UP_MULTIPLIER = 0.5f; glm::quat eyeRotation = rotationBetween(headOrientation * IDENTITY_FORWARD, getLookAtPosition() - _eyePosition); - eyeRotation = eyeRotation * glm::angleAxis(safeEulerAngles(headOrientation).y, IDENTITY_UP); // Rotation w.r.t. head + auto worldUpDirection = _owningAvatar->getWorldOrientation() * Vectors::UNIT_Y; + eyeRotation = eyeRotation * glm::angleAxis(safeEulerAngles(headOrientation).y, worldUpDirection); // Rotation w.r.t. head float eyePitch = safeEulerAngles(eyeRotation).x; float eyelidOffset = glm::clamp(abs(eyePitch * EYE_PITCH_TO_COEFFICIENT), 0.0f, MAX_EYELID_OFFSET); - std::vector eyeBlinkIndices, eyeOpenIndices, browsIndices; - - getBlendshapeIndices(eyeBlinkBlendShapes, eyeBlinkIndices); - getBlendshapeIndices(eyeOpenBlendShapes, eyeOpenIndices); - getBlendshapeIndices(browsBlendShapes, browsIndices); + float blinkUpCoefficient = -eyelidOffset; + float blinkDownCoefficient = BLINK_DOWN_MULTIPLIER * eyelidOffset; + + float openUpCoefficient = eyelidOffset; + float openDownCoefficient = OPEN_DOWN_MULTIPLIER * eyelidOffset; + + float browsUpCoefficient = BROW_UP_MULTIPLIER * eyelidOffset; + float browsDownCoefficient = 0.0f; bool isLookingUp = (eyePitch > 0); - - for (auto& blinkIndex : eyeBlinkIndices) { - float lookingUpCoefficient = -eyelidOffset; - float lookingDownCoefficient = BLINK_DOWN_MULTIPLIER * eyelidOffset; - if (blinkIndex >= 0 && blinkIndex < _transientBlendshapeCoefficients.size()) { - _transientBlendshapeCoefficients[blinkIndex] = isLookingUp ? lookingUpCoefficient : lookingDownCoefficient; - } - - } - - for (auto& openIndex : eyeOpenIndices) { - float lookingUpCoefficient = eyelidOffset; - float lookingDownCoefficient = OPEN_DOWN_MULTIPLIER * eyelidOffset; - if (openIndex >= 0 && openIndex < _transientBlendshapeCoefficients.size()) { - _transientBlendshapeCoefficients[openIndex] = isLookingUp ? lookingUpCoefficient : lookingDownCoefficient; - } - } - for (auto& browIndex : browsIndices) { - float lookingUpCoefficient = BROW_UP_MULTIPLIER * eyelidOffset; - float lookingDownCoefficient = 0.0f; - if (browIndex >= 0 && browIndex < _transientBlendshapeCoefficients.size()) { - _transientBlendshapeCoefficients[browIndex] = isLookingUp ? lookingUpCoefficient : lookingDownCoefficient; + if (isLookingUp) { + for (int i = 0; i < 2; i++) { + _transientBlendshapeCoefficients[EYE_BLINK_INDICES[i]] = blinkUpCoefficient; + _transientBlendshapeCoefficients[EYE_OPEN_INDICES[i]] = openUpCoefficient; + _transientBlendshapeCoefficients[BROWS_U_INDICES[i]] = browsUpCoefficient; + } + } else { + for (int i = 0; i < 2; i++) { + _transientBlendshapeCoefficients[EYE_BLINK_INDICES[i]] = blinkDownCoefficient; + _transientBlendshapeCoefficients[EYE_OPEN_INDICES[i]] = openDownCoefficient; + _transientBlendshapeCoefficients[BROWS_U_INDICES[i]] = browsDownCoefficient; } } } diff --git a/libraries/avatars/src/HeadData.cpp b/libraries/avatars/src/HeadData.cpp index c68bb26e34..19f5efcd16 100644 --- a/libraries/avatars/src/HeadData.cpp +++ b/libraries/avatars/src/HeadData.cpp @@ -17,7 +17,6 @@ #include #include -#include #include #include diff --git a/libraries/avatars/src/HeadData.h b/libraries/avatars/src/HeadData.h index 834629596a..6d211da2cd 100644 --- a/libraries/avatars/src/HeadData.h +++ b/libraries/avatars/src/HeadData.h @@ -20,6 +20,7 @@ #include #include +#include // degrees const float MIN_HEAD_YAW = -180.0f; diff --git a/libraries/shared/src/FaceshiftConstants.cpp b/libraries/shared/src/FaceshiftConstants.cpp index e6c929409a..d5013236fc 100644 --- a/libraries/shared/src/FaceshiftConstants.cpp +++ b/libraries/shared/src/FaceshiftConstants.cpp @@ -64,3 +64,18 @@ const char* FACESHIFT_BLENDSHAPES[] = { }; const int NUM_FACESHIFT_BLENDSHAPES = sizeof(FACESHIFT_BLENDSHAPES) / sizeof(char*); + +const int EYE_BLINK_L_INDEX = 0; +const int EYE_BLINK_R_INDEX = 1; +const int EYE_SQUINT_L_INDEX = 2; +const int EYE_SQUINT_R_INDEX = 3; +const int EYE_OPEN_L_INDEX = 8; +const int EYE_OPEN_R_INDEX = 9; +const int BROWS_U_L_INDEX = 17; +const int BROWS_U_R_INDEX = 18; + + +const int EYE_BLINK_INDICES[] = { EYE_BLINK_L_INDEX, EYE_BLINK_R_INDEX }; +const int EYE_SQUINT_INDICES[] = { EYE_SQUINT_L_INDEX, EYE_SQUINT_R_INDEX }; +const int EYE_OPEN_INDICES[] = { EYE_OPEN_L_INDEX, EYE_OPEN_R_INDEX }; +const int BROWS_U_INDICES[] = { BROWS_U_L_INDEX, BROWS_U_R_INDEX }; \ No newline at end of file diff --git a/libraries/shared/src/FaceshiftConstants.h b/libraries/shared/src/FaceshiftConstants.h index ee6e43fdbc..88012c3d19 100644 --- a/libraries/shared/src/FaceshiftConstants.h +++ b/libraries/shared/src/FaceshiftConstants.h @@ -16,5 +16,9 @@ extern const char* FACESHIFT_BLENDSHAPES[]; /// The size of FACESHIFT_BLENDSHAPES extern const int NUM_FACESHIFT_BLENDSHAPES; - +// Eyes and Brows indices +extern const int EYE_BLINK_INDICES[]; +extern const int EYE_OPEN_INDICES[]; +extern const int BROWS_U_INDICES[]; +extern const int EYE_SQUINT_INDICES[]; #endif // hifi_FaceshiftConstants_h \ No newline at end of file From 97a0edf026513eee77c69e4e1a0ecbabb648aed4 Mon Sep 17 00:00:00 2001 From: luiscuenca Date: Fri, 29 Jun 2018 14:38:43 -0700 Subject: [PATCH 3/4] spaces --- libraries/shared/src/FaceshiftConstants.cpp | 2 +- libraries/shared/src/FaceshiftConstants.h | 3 ++- 2 files changed, 3 insertions(+), 2 deletions(-) diff --git a/libraries/shared/src/FaceshiftConstants.cpp b/libraries/shared/src/FaceshiftConstants.cpp index d5013236fc..0d6f718e49 100644 --- a/libraries/shared/src/FaceshiftConstants.cpp +++ b/libraries/shared/src/FaceshiftConstants.cpp @@ -78,4 +78,4 @@ const int BROWS_U_R_INDEX = 18; const int EYE_BLINK_INDICES[] = { EYE_BLINK_L_INDEX, EYE_BLINK_R_INDEX }; const int EYE_SQUINT_INDICES[] = { EYE_SQUINT_L_INDEX, EYE_SQUINT_R_INDEX }; const int EYE_OPEN_INDICES[] = { EYE_OPEN_L_INDEX, EYE_OPEN_R_INDEX }; -const int BROWS_U_INDICES[] = { BROWS_U_L_INDEX, BROWS_U_R_INDEX }; \ No newline at end of file +const int BROWS_U_INDICES[] = { BROWS_U_L_INDEX, BROWS_U_R_INDEX }; diff --git a/libraries/shared/src/FaceshiftConstants.h b/libraries/shared/src/FaceshiftConstants.h index 88012c3d19..4349a3a21e 100644 --- a/libraries/shared/src/FaceshiftConstants.h +++ b/libraries/shared/src/FaceshiftConstants.h @@ -21,4 +21,5 @@ extern const int EYE_BLINK_INDICES[]; extern const int EYE_OPEN_INDICES[]; extern const int BROWS_U_INDICES[]; extern const int EYE_SQUINT_INDICES[]; -#endif // hifi_FaceshiftConstants_h \ No newline at end of file + +#endif // hifi_FaceshiftConstants_h From f770bcd38978bd6fee94661dfbf054c9cac14c50 Mon Sep 17 00:00:00 2001 From: luiscuenca Date: Fri, 29 Jun 2018 17:15:03 -0700 Subject: [PATCH 4/4] Better eyePitch computation --- libraries/avatars-renderer/src/avatars-renderer/Head.cpp | 7 +++---- 1 file changed, 3 insertions(+), 4 deletions(-) diff --git a/libraries/avatars-renderer/src/avatars-renderer/Head.cpp b/libraries/avatars-renderer/src/avatars-renderer/Head.cpp index b76e401761..bdee6d9147 100644 --- a/libraries/avatars-renderer/src/avatars-renderer/Head.cpp +++ b/libraries/avatars-renderer/src/avatars-renderer/Head.cpp @@ -231,10 +231,9 @@ void Head::applyEyelidOffset(glm::quat headOrientation) { const float OPEN_DOWN_MULTIPLIER = 0.3f; const float BROW_UP_MULTIPLIER = 0.5f; - glm::quat eyeRotation = rotationBetween(headOrientation * IDENTITY_FORWARD, getLookAtPosition() - _eyePosition); - auto worldUpDirection = _owningAvatar->getWorldOrientation() * Vectors::UNIT_Y; - eyeRotation = eyeRotation * glm::angleAxis(safeEulerAngles(headOrientation).y, worldUpDirection); // Rotation w.r.t. head - float eyePitch = safeEulerAngles(eyeRotation).x; + glm::vec3 lookAt = glm::normalize(getLookAtPosition() - _eyePosition); + glm::vec3 headUp = headOrientation * Vectors::UNIT_Y; + float eyePitch = (PI / 2.0f) - acos(glm::dot(lookAt, headUp)); float eyelidOffset = glm::clamp(abs(eyePitch * EYE_PITCH_TO_COEFFICIENT), 0.0f, MAX_EYELID_OFFSET); float blinkUpCoefficient = -eyelidOffset;