From f4df223d2337870dbb14177c9561d403c13432d5 Mon Sep 17 00:00:00 2001 From: Andrew Meadows Date: Tue, 9 May 2017 14:48:42 -0700 Subject: [PATCH] cleanup Head::simulation() code --- interface/src/avatar/MyHead.cpp | 27 ++- .../src/avatars-renderer/Head.cpp | 199 +++++++++--------- .../src/avatars-renderer/Head.h | 5 +- libraries/avatars/src/HeadData.cpp | 6 - libraries/avatars/src/HeadData.h | 14 +- 5 files changed, 127 insertions(+), 124 deletions(-) diff --git a/interface/src/avatar/MyHead.cpp b/interface/src/avatar/MyHead.cpp index 34a75c5461..793fbb79c4 100644 --- a/interface/src/avatar/MyHead.cpp +++ b/interface/src/avatar/MyHead.cpp @@ -44,14 +44,17 @@ glm::quat MyHead::getCameraOrientation() const { void MyHead::simulate(float deltaTime) { auto player = DependencyManager::get(); // Only use face trackers when not playing back a recording. - if (!player->isPlaying()) { + if (player->isPlaying()) { + Parent::simulate(deltaTime); + } else { + computeAudioLoudness(deltaTime); + FaceTracker* faceTracker = qApp->getActiveFaceTracker(); - _isFaceTrackerConnected = faceTracker != NULL && !faceTracker->isMuted(); + _isFaceTrackerConnected = faceTracker && !faceTracker->isMuted(); if (_isFaceTrackerConnected) { _transientBlendshapeCoefficients = faceTracker->getBlendshapeCoefficients(); if (typeid(*faceTracker) == typeid(DdeFaceTracker)) { - if (Menu::getInstance()->isOptionChecked(MenuOption::UseAudioForMouth)) { calculateMouthShapes(deltaTime); @@ -68,9 +71,19 @@ void MyHead::simulate(float deltaTime) { } applyEyelidOffset(getFinalOrientationInWorldFrame()); } - } + } else { + computeFaceMovement(deltaTime); + } + auto eyeTracker = DependencyManager::get(); - _isEyeTrackerConnected = eyeTracker->isTracking(); + _isEyeTrackerConnected = eyeTracker && eyeTracker->isTracking(); + if (_isEyeTrackerConnected) { + // TODO? figure out where EyeTracker data harvested. Move it here? + _saccade = glm::vec3(); + } else { + computeEyeMovement(deltaTime); + } + } - Parent::simulate(deltaTime); -} \ No newline at end of file + computeEyePosition(); +} diff --git a/libraries/avatars-renderer/src/avatars-renderer/Head.cpp b/libraries/avatars-renderer/src/avatars-renderer/Head.cpp index 93fe246266..b4b0929c0c 100644 --- a/libraries/avatars-renderer/src/avatars-renderer/Head.cpp +++ b/libraries/avatars-renderer/src/avatars-renderer/Head.cpp @@ -23,9 +23,10 @@ #include "Avatar.h" +const float NORMAL_HZ = 60.0f; // the update rate the constant values were tuned for + using namespace std; -static bool fixGaze { false }; static bool disableEyelidAdjustment { false }; Head::Head(Avatar* owningAvatar) : @@ -42,17 +43,11 @@ void Head::reset() { _baseYaw = _basePitch = _baseRoll = 0.0f; } -void Head::simulate(float deltaTime) { - const float NORMAL_HZ = 60.0f; // the update rate the constant values were tuned for - +void Head::computeAudioLoudness(float deltaTime) { // grab the audio loudness from the owning avatar, if we have one - float audioLoudness = 0.0f; + float audioLoudness = _owningAvatar ? _owningAvatar->getAudioLoudness() : 0.0f; - if (_owningAvatar) { - audioLoudness = _owningAvatar->getAudioLoudness(); - } - - // Update audio trailing average for rendering facial animations + // Update audio trailing average for rendering facial animations const float AUDIO_AVERAGING_SECS = 0.05f; const float AUDIO_LONG_TERM_AVERAGING_SECS = 30.0f; _averageLoudness = glm::mix(_averageLoudness, audioLoudness, glm::min(deltaTime / AUDIO_AVERAGING_SECS, 1.0f)); @@ -63,116 +58,114 @@ void Head::simulate(float deltaTime) { _longTermAverageLoudness = glm::mix(_longTermAverageLoudness, _averageLoudness, glm::min(deltaTime / AUDIO_LONG_TERM_AVERAGING_SECS, 1.0f)); } - if (!_isFaceTrackerConnected) { - if (!_isEyeTrackerConnected) { - // Update eye saccades - const float AVERAGE_MICROSACCADE_INTERVAL = 1.0f; - const float AVERAGE_SACCADE_INTERVAL = 6.0f; - const float MICROSACCADE_MAGNITUDE = 0.002f; - const float SACCADE_MAGNITUDE = 0.04f; - const float NOMINAL_FRAME_RATE = 60.0f; + float audioAttackAveragingRate = (10.0f - deltaTime * NORMAL_HZ) / 10.0f; // --> 0.9 at 60 Hz + _audioAttack = audioAttackAveragingRate * _audioAttack + + (1.0f - audioAttackAveragingRate) * fabs((audioLoudness - _longTermAverageLoudness) - _lastLoudness); + _lastLoudness = (audioLoudness - _longTermAverageLoudness); +} - if (randFloat() < deltaTime / AVERAGE_MICROSACCADE_INTERVAL) { - _saccadeTarget = MICROSACCADE_MAGNITUDE * randVector(); - } else if (randFloat() < deltaTime / AVERAGE_SACCADE_INTERVAL) { - _saccadeTarget = SACCADE_MAGNITUDE * randVector(); - } - _saccade += (_saccadeTarget - _saccade) * pow(0.5f, NOMINAL_FRAME_RATE * deltaTime); - } else { - _saccade = glm::vec3(); - } +void Head::computeEyeMovement(float deltaTime) { + // Update eye saccades + const float AVERAGE_MICROSACCADE_INTERVAL = 1.0f; + const float AVERAGE_SACCADE_INTERVAL = 6.0f; + const float MICROSACCADE_MAGNITUDE = 0.002f; + const float SACCADE_MAGNITUDE = 0.04f; + const float NOMINAL_FRAME_RATE = 60.0f; - // Detect transition from talking to not; force blink after that and a delay - bool forceBlink = false; - const float TALKING_LOUDNESS = 100.0f; - const float BLINK_AFTER_TALKING = 0.25f; - _timeWithoutTalking += deltaTime; - if ((_averageLoudness - _longTermAverageLoudness) > TALKING_LOUDNESS) { - _timeWithoutTalking = 0.0f; - } else if (_timeWithoutTalking - deltaTime < BLINK_AFTER_TALKING && _timeWithoutTalking >= BLINK_AFTER_TALKING) { - forceBlink = true; - } + if (randFloat() < deltaTime / AVERAGE_MICROSACCADE_INTERVAL) { + _saccadeTarget = MICROSACCADE_MAGNITUDE * randVector(); + } else if (randFloat() < deltaTime / AVERAGE_SACCADE_INTERVAL) { + _saccadeTarget = SACCADE_MAGNITUDE * randVector(); + } + _saccade += (_saccadeTarget - _saccade) * pow(0.5f, NOMINAL_FRAME_RATE * deltaTime); - // Update audio attack data for facial animation (eyebrows and mouth) - float audioAttackAveragingRate = (10.0f - deltaTime * NORMAL_HZ) / 10.0f; // --> 0.9 at 60 Hz - _audioAttack = audioAttackAveragingRate * _audioAttack + - (1.0f - audioAttackAveragingRate) * fabs((audioLoudness - _longTermAverageLoudness) - _lastLoudness); - _lastLoudness = (audioLoudness - _longTermAverageLoudness); + // Detect transition from talking to not; force blink after that and a delay + bool forceBlink = false; + const float TALKING_LOUDNESS = 100.0f; + const float BLINK_AFTER_TALKING = 0.25f; + _timeWithoutTalking += deltaTime; + if ((_averageLoudness - _longTermAverageLoudness) > TALKING_LOUDNESS) { + _timeWithoutTalking = 0.0f; + } else if (_timeWithoutTalking - deltaTime < BLINK_AFTER_TALKING && _timeWithoutTalking >= BLINK_AFTER_TALKING) { + forceBlink = true; + } - const float BROW_LIFT_THRESHOLD = 100.0f; - if (_audioAttack > BROW_LIFT_THRESHOLD) { - _browAudioLift += sqrtf(_audioAttack) * 0.01f; - } - _browAudioLift = glm::clamp(_browAudioLift *= 0.7f, 0.0f, 1.0f); - - const float BLINK_SPEED = 10.0f; - const float BLINK_SPEED_VARIABILITY = 1.0f; - const float BLINK_START_VARIABILITY = 0.25f; - const float FULLY_OPEN = 0.0f; - const float FULLY_CLOSED = 1.0f; - if (_leftEyeBlinkVelocity == 0.0f && _rightEyeBlinkVelocity == 0.0f) { - // no blinking when brows are raised; blink less with increasing loudness - const float BASE_BLINK_RATE = 15.0f / 60.0f; - const float ROOT_LOUDNESS_TO_BLINK_INTERVAL = 0.25f; - if (forceBlink || (_browAudioLift < EPSILON && shouldDo(glm::max(1.0f, sqrt(fabs(_averageLoudness - _longTermAverageLoudness)) * - ROOT_LOUDNESS_TO_BLINK_INTERVAL) / BASE_BLINK_RATE, deltaTime))) { - _leftEyeBlinkVelocity = BLINK_SPEED + randFloat() * BLINK_SPEED_VARIABILITY; - _rightEyeBlinkVelocity = BLINK_SPEED + randFloat() * BLINK_SPEED_VARIABILITY; - if (randFloat() < 0.5f) { - _leftEyeBlink = BLINK_START_VARIABILITY; - } else { - _rightEyeBlink = BLINK_START_VARIABILITY; - } - } - } else { - _leftEyeBlink = glm::clamp(_leftEyeBlink + _leftEyeBlinkVelocity * deltaTime, FULLY_OPEN, FULLY_CLOSED); - _rightEyeBlink = glm::clamp(_rightEyeBlink + _rightEyeBlinkVelocity * deltaTime, FULLY_OPEN, FULLY_CLOSED); - - if (_leftEyeBlink == FULLY_CLOSED) { - _leftEyeBlinkVelocity = -BLINK_SPEED; - - } else if (_leftEyeBlink == FULLY_OPEN) { - _leftEyeBlinkVelocity = 0.0f; - } - if (_rightEyeBlink == FULLY_CLOSED) { - _rightEyeBlinkVelocity = -BLINK_SPEED; - - } else if (_rightEyeBlink == FULLY_OPEN) { - _rightEyeBlinkVelocity = 0.0f; + const float BLINK_SPEED = 10.0f; + const float BLINK_SPEED_VARIABILITY = 1.0f; + const float BLINK_START_VARIABILITY = 0.25f; + const float FULLY_OPEN = 0.0f; + const float FULLY_CLOSED = 1.0f; + if (_leftEyeBlinkVelocity == 0.0f && _rightEyeBlinkVelocity == 0.0f) { + // no blinking when brows are raised; blink less with increasing loudness + const float BASE_BLINK_RATE = 15.0f / 60.0f; + const float ROOT_LOUDNESS_TO_BLINK_INTERVAL = 0.25f; + if (forceBlink || (_browAudioLift < EPSILON && shouldDo(glm::max(1.0f, sqrt(fabs(_averageLoudness - _longTermAverageLoudness)) * + ROOT_LOUDNESS_TO_BLINK_INTERVAL) / BASE_BLINK_RATE, deltaTime))) { + _leftEyeBlinkVelocity = BLINK_SPEED + randFloat() * BLINK_SPEED_VARIABILITY; + _rightEyeBlinkVelocity = BLINK_SPEED + randFloat() * BLINK_SPEED_VARIABILITY; + if (randFloat() < 0.5f) { + _leftEyeBlink = BLINK_START_VARIABILITY; + } else { + _rightEyeBlink = BLINK_START_VARIABILITY; } } - - // use data to update fake Faceshift blendshape coefficients - calculateMouthShapes(deltaTime); - FaceTracker::updateFakeCoefficients(_leftEyeBlink, - _rightEyeBlink, - _browAudioLift, - _audioJawOpen, - _mouth2, - _mouth3, - _mouth4, - _transientBlendshapeCoefficients); - - applyEyelidOffset(getOrientation()); - } else { - _saccade = glm::vec3(); - } - if (fixGaze) { // if debug menu turns off, use no saccade - _saccade = glm::vec3(); + _leftEyeBlink = glm::clamp(_leftEyeBlink + _leftEyeBlinkVelocity * deltaTime, FULLY_OPEN, FULLY_CLOSED); + _rightEyeBlink = glm::clamp(_rightEyeBlink + _rightEyeBlinkVelocity * deltaTime, FULLY_OPEN, FULLY_CLOSED); + + if (_leftEyeBlink == FULLY_CLOSED) { + _leftEyeBlinkVelocity = -BLINK_SPEED; + + } else if (_leftEyeBlink == FULLY_OPEN) { + _leftEyeBlinkVelocity = 0.0f; + } + if (_rightEyeBlink == FULLY_CLOSED) { + _rightEyeBlinkVelocity = -BLINK_SPEED; + + } else if (_rightEyeBlink == FULLY_OPEN) { + _rightEyeBlinkVelocity = 0.0f; + } } + applyEyelidOffset(getOrientation()); +} + +void Head::computeFaceMovement(float deltaTime) { + // Update audio attack data for facial animation (eyebrows and mouth) + const float BROW_LIFT_THRESHOLD = 100.0f; + if (_audioAttack > BROW_LIFT_THRESHOLD) { + _browAudioLift += sqrtf(_audioAttack) * 0.01f; + } + _browAudioLift = glm::clamp(_browAudioLift *= 0.7f, 0.0f, 1.0f); + + // use data to update fake Faceshift blendshape coefficients + calculateMouthShapes(deltaTime); + FaceTracker::updateFakeCoefficients(_leftEyeBlink, + _rightEyeBlink, + _browAudioLift, + _audioJawOpen, + _mouth2, + _mouth3, + _mouth4, + _transientBlendshapeCoefficients); +} + +void Head::computeEyePosition() { _leftEyePosition = _rightEyePosition = getPosition(); - _eyePosition = getPosition(); - if (_owningAvatar) { auto skeletonModel = static_cast(_owningAvatar)->getSkeletonModel(); if (skeletonModel) { skeletonModel->getEyePositions(_leftEyePosition, _rightEyePosition); } } + _eyePosition = 0.5f * (_leftEyePosition + _rightEyePosition); +} - _eyePosition = calculateAverageEyePosition(); +void Head::simulate(float deltaTime) { + computeAudioLoudness(deltaTime); + computeFaceMovement(deltaTime); + computeEyeMovement(deltaTime); + computeEyePosition(); } void Head::calculateMouthShapes(float deltaTime) { diff --git a/libraries/avatars-renderer/src/avatars-renderer/Head.h b/libraries/avatars-renderer/src/avatars-renderer/Head.h index aea6a41528..39331500b5 100644 --- a/libraries/avatars-renderer/src/avatars-renderer/Head.h +++ b/libraries/avatars-renderer/src/avatars-renderer/Head.h @@ -83,7 +83,10 @@ public: float getTimeWithoutTalking() const { return _timeWithoutTalking; } protected: - glm::vec3 calculateAverageEyePosition() const { return _leftEyePosition + (_rightEyePosition - _leftEyePosition ) * 0.5f; } + void computeAudioLoudness(float deltaTime); + void computeEyeMovement(float deltaTime); + void computeFaceMovement(float deltaTime); + void computeEyePosition(); // disallow copies of the Head, copy of owning Avatar is disallowed too Head(const Head&); diff --git a/libraries/avatars/src/HeadData.cpp b/libraries/avatars/src/HeadData.cpp index 2e4eec73a8..2704b6539c 100644 --- a/libraries/avatars/src/HeadData.cpp +++ b/libraries/avatars/src/HeadData.cpp @@ -28,12 +28,6 @@ HeadData::HeadData(AvatarData* owningAvatar) : _basePitch(0.0f), _baseRoll(0.0f), _lookAtPosition(0.0f, 0.0f, 0.0f), - _isFaceTrackerConnected(false), - _isEyeTrackerConnected(false), - _leftEyeBlink(0.0f), - _rightEyeBlink(0.0f), - _averageLoudness(0.0f), - _browAudioLift(0.0f), _blendshapeCoefficients(QVector(0, 0.0f)), _transientBlendshapeCoefficients(QVector(0, 0.0f)), _summedBlendshapeCoefficients(QVector(0, 0.0f)), diff --git a/libraries/avatars/src/HeadData.h b/libraries/avatars/src/HeadData.h index 9b28616b3f..be9d54e93e 100644 --- a/libraries/avatars/src/HeadData.h +++ b/libraries/avatars/src/HeadData.h @@ -63,7 +63,7 @@ public: void setBlendshapeCoefficients(const QVector& blendshapeCoefficients) { _blendshapeCoefficients = blendshapeCoefficients; } const glm::vec3& getLookAtPosition() const { return _lookAtPosition; } - void setLookAtPosition(const glm::vec3& lookAtPosition) { + void setLookAtPosition(const glm::vec3& lookAtPosition) { if (_lookAtPosition != lookAtPosition) { _lookAtPositionChanged = usecTimestampNow(); } @@ -85,12 +85,12 @@ protected: glm::vec3 _lookAtPosition; quint64 _lookAtPositionChanged { 0 }; - bool _isFaceTrackerConnected; - bool _isEyeTrackerConnected; - float _leftEyeBlink; - float _rightEyeBlink; - float _averageLoudness; - float _browAudioLift; + bool _isFaceTrackerConnected { false }; + bool _isEyeTrackerConnected { false }; + float _leftEyeBlink { 0.0f }; + float _rightEyeBlink { 0.0f }; + float _averageLoudness { 0.0f }; + float _browAudioLift { 0.0f }; QVector _blendshapeCoefficients; QVector _transientBlendshapeCoefficients;