Mirrors/overte-lubosz
3
0
Fork 0
mirror of https://github.com/lubosz/overte.git synced 2025-04-27 14:35:30 +02:00
Code Issues Projects Releases Packages Wiki Activity Actions

cleanup Head::simulation() code

Browse source
This commit is contained in:
Andrew Meadows 2017-05-09 14:48:42 -07:00
parent 9d63113e62
commit f4df223d23
5 changed files with 127 additions and 124 deletions
interface/src/avatar
MyHead.cpp
libraries
avatars-renderer/src/avatars-renderer
Head.cppHead.h
avatars/src
HeadData.cppHeadData.h

27
interface/src/avatar/MyHead.cpp
View file

@ -44,14 +44,17 @@ glm::quat MyHead::getCameraOrientation() const {
void MyHead::simulate(float deltaTime) {
auto player = DependencyManager::get<recording::Deck>();
// Only use face trackers when not playing back a recording.
if (!player->isPlaying()) {
if (player->isPlaying()) {
Parent::simulate(deltaTime);
} else {
computeAudioLoudness(deltaTime);
FaceTracker* faceTracker = qApp->getActiveFaceTracker();
_isFaceTrackerConnected = faceTracker != NULL && !faceTracker->isMuted();
_isFaceTrackerConnected = faceTracker && !faceTracker->isMuted();
if (_isFaceTrackerConnected) {
_transientBlendshapeCoefficients = faceTracker->getBlendshapeCoefficients();
if (typeid(*faceTracker) == typeid(DdeFaceTracker)) {
if (Menu::getInstance()->isOptionChecked(MenuOption::UseAudioForMouth)) {
calculateMouthShapes(deltaTime);
@ -68,9 +71,19 @@ void MyHead::simulate(float deltaTime) {
}
applyEyelidOffset(getFinalOrientationInWorldFrame());
}
}
} else {
computeFaceMovement(deltaTime);
}
auto eyeTracker = DependencyManager::get<EyeTracker>();
_isEyeTrackerConnected = eyeTracker->isTracking();
_isEyeTrackerConnected = eyeTracker && eyeTracker->isTracking();
if (_isEyeTrackerConnected) {
// TODO? figure out where EyeTracker data harvested. Move it here?
_saccade = glm::vec3();
} else {
computeEyeMovement(deltaTime);
}
}
Parent::simulate(deltaTime);
}
computeEyePosition();
}

199
libraries/avatars-renderer/src/avatars-renderer/Head.cpp
View file

@ -23,9 +23,10 @@
#include "Avatar.h"
const float NORMAL_HZ = 60.0f; // the update rate the constant values were tuned for
using namespace std;
static bool fixGaze { false };
static bool disableEyelidAdjustment { false };
Head::Head(Avatar* owningAvatar) :
@ -42,17 +43,11 @@ void Head::reset() {
_baseYaw = _basePitch = _baseRoll = 0.0f;
}
void Head::simulate(float deltaTime) {
const float NORMAL_HZ = 60.0f; // the update rate the constant values were tuned for
void Head::computeAudioLoudness(float deltaTime) {
// grab the audio loudness from the owning avatar, if we have one
float audioLoudness = 0.0f;
float audioLoudness = _owningAvatar ? _owningAvatar->getAudioLoudness() : 0.0f;
if (_owningAvatar) {
audioLoudness = _owningAvatar->getAudioLoudness();
}
// Update audio trailing average for rendering facial animations
// Update audio trailing average for rendering facial animations
const float AUDIO_AVERAGING_SECS = 0.05f;
const float AUDIO_LONG_TERM_AVERAGING_SECS = 30.0f;
_averageLoudness = glm::mix(_averageLoudness, audioLoudness, glm::min(deltaTime / AUDIO_AVERAGING_SECS, 1.0f));
@ -63,116 +58,114 @@ void Head::simulate(float deltaTime) {
_longTermAverageLoudness = glm::mix(_longTermAverageLoudness, _averageLoudness, glm::min(deltaTime / AUDIO_LONG_TERM_AVERAGING_SECS, 1.0f));
}
if (!_isFaceTrackerConnected) {
if (!_isEyeTrackerConnected) {
// Update eye saccades
const float AVERAGE_MICROSACCADE_INTERVAL = 1.0f;
const float AVERAGE_SACCADE_INTERVAL = 6.0f;
const float MICROSACCADE_MAGNITUDE = 0.002f;
const float SACCADE_MAGNITUDE = 0.04f;
const float NOMINAL_FRAME_RATE = 60.0f;
float audioAttackAveragingRate = (10.0f - deltaTime * NORMAL_HZ) / 10.0f; // --> 0.9 at 60 Hz
_audioAttack = audioAttackAveragingRate * _audioAttack +
(1.0f - audioAttackAveragingRate) * fabs((audioLoudness - _longTermAverageLoudness) - _lastLoudness);
_lastLoudness = (audioLoudness - _longTermAverageLoudness);
}
if (randFloat() < deltaTime / AVERAGE_MICROSACCADE_INTERVAL) {
_saccadeTarget = MICROSACCADE_MAGNITUDE * randVector();
} else if (randFloat() < deltaTime / AVERAGE_SACCADE_INTERVAL) {
_saccadeTarget = SACCADE_MAGNITUDE * randVector();
}
_saccade += (_saccadeTarget - _saccade) * pow(0.5f, NOMINAL_FRAME_RATE * deltaTime);
} else {
_saccade = glm::vec3();
}
void Head::computeEyeMovement(float deltaTime) {
// Update eye saccades
const float AVERAGE_MICROSACCADE_INTERVAL = 1.0f;
const float AVERAGE_SACCADE_INTERVAL = 6.0f;
const float MICROSACCADE_MAGNITUDE = 0.002f;
const float SACCADE_MAGNITUDE = 0.04f;
const float NOMINAL_FRAME_RATE = 60.0f;
// Detect transition from talking to not; force blink after that and a delay
bool forceBlink = false;
const float TALKING_LOUDNESS = 100.0f;
const float BLINK_AFTER_TALKING = 0.25f;
_timeWithoutTalking += deltaTime;
if ((_averageLoudness - _longTermAverageLoudness) > TALKING_LOUDNESS) {
_timeWithoutTalking = 0.0f;
} else if (_timeWithoutTalking - deltaTime < BLINK_AFTER_TALKING && _timeWithoutTalking >= BLINK_AFTER_TALKING) {
forceBlink = true;
}
if (randFloat() < deltaTime / AVERAGE_MICROSACCADE_INTERVAL) {
_saccadeTarget = MICROSACCADE_MAGNITUDE * randVector();
} else if (randFloat() < deltaTime / AVERAGE_SACCADE_INTERVAL) {
_saccadeTarget = SACCADE_MAGNITUDE * randVector();
}
_saccade += (_saccadeTarget - _saccade) * pow(0.5f, NOMINAL_FRAME_RATE * deltaTime);
// Update audio attack data for facial animation (eyebrows and mouth)
float audioAttackAveragingRate = (10.0f - deltaTime * NORMAL_HZ) / 10.0f; // --> 0.9 at 60 Hz
_audioAttack = audioAttackAveragingRate * _audioAttack +
(1.0f - audioAttackAveragingRate) * fabs((audioLoudness - _longTermAverageLoudness) - _lastLoudness);
_lastLoudness = (audioLoudness - _longTermAverageLoudness);
// Detect transition from talking to not; force blink after that and a delay
bool forceBlink = false;
const float TALKING_LOUDNESS = 100.0f;
const float BLINK_AFTER_TALKING = 0.25f;
_timeWithoutTalking += deltaTime;
if ((_averageLoudness - _longTermAverageLoudness) > TALKING_LOUDNESS) {
_timeWithoutTalking = 0.0f;
} else if (_timeWithoutTalking - deltaTime < BLINK_AFTER_TALKING && _timeWithoutTalking >= BLINK_AFTER_TALKING) {
forceBlink = true;
}
const float BROW_LIFT_THRESHOLD = 100.0f;
if (_audioAttack > BROW_LIFT_THRESHOLD) {
_browAudioLift += sqrtf(_audioAttack) * 0.01f;
}
_browAudioLift = glm::clamp(_browAudioLift *= 0.7f, 0.0f, 1.0f);
const float BLINK_SPEED = 10.0f;
const float BLINK_SPEED_VARIABILITY = 1.0f;
const float BLINK_START_VARIABILITY = 0.25f;
const float FULLY_OPEN = 0.0f;
const float FULLY_CLOSED = 1.0f;
if (_leftEyeBlinkVelocity == 0.0f && _rightEyeBlinkVelocity == 0.0f) {
// no blinking when brows are raised; blink less with increasing loudness
const float BASE_BLINK_RATE = 15.0f / 60.0f;
const float ROOT_LOUDNESS_TO_BLINK_INTERVAL = 0.25f;
if (forceBlink || (_browAudioLift < EPSILON && shouldDo(glm::max(1.0f, sqrt(fabs(_averageLoudness - _longTermAverageLoudness)) *
ROOT_LOUDNESS_TO_BLINK_INTERVAL) / BASE_BLINK_RATE, deltaTime))) {
_leftEyeBlinkVelocity = BLINK_SPEED + randFloat() * BLINK_SPEED_VARIABILITY;
_rightEyeBlinkVelocity = BLINK_SPEED + randFloat() * BLINK_SPEED_VARIABILITY;
if (randFloat() < 0.5f) {
_leftEyeBlink = BLINK_START_VARIABILITY;
} else {
_rightEyeBlink = BLINK_START_VARIABILITY;
}
}
} else {
_leftEyeBlink = glm::clamp(_leftEyeBlink + _leftEyeBlinkVelocity * deltaTime, FULLY_OPEN, FULLY_CLOSED);
_rightEyeBlink = glm::clamp(_rightEyeBlink + _rightEyeBlinkVelocity * deltaTime, FULLY_OPEN, FULLY_CLOSED);
if (_leftEyeBlink == FULLY_CLOSED) {
_leftEyeBlinkVelocity = -BLINK_SPEED;
} else if (_leftEyeBlink == FULLY_OPEN) {
_leftEyeBlinkVelocity = 0.0f;
}
if (_rightEyeBlink == FULLY_CLOSED) {
_rightEyeBlinkVelocity = -BLINK_SPEED;
} else if (_rightEyeBlink == FULLY_OPEN) {
_rightEyeBlinkVelocity = 0.0f;
const float BLINK_SPEED = 10.0f;
const float BLINK_SPEED_VARIABILITY = 1.0f;
const float BLINK_START_VARIABILITY = 0.25f;
const float FULLY_OPEN = 0.0f;
const float FULLY_CLOSED = 1.0f;
if (_leftEyeBlinkVelocity == 0.0f && _rightEyeBlinkVelocity == 0.0f) {
// no blinking when brows are raised; blink less with increasing loudness
const float BASE_BLINK_RATE = 15.0f / 60.0f;
const float ROOT_LOUDNESS_TO_BLINK_INTERVAL = 0.25f;
if (forceBlink || (_browAudioLift < EPSILON && shouldDo(glm::max(1.0f, sqrt(fabs(_averageLoudness - _longTermAverageLoudness)) *
ROOT_LOUDNESS_TO_BLINK_INTERVAL) / BASE_BLINK_RATE, deltaTime))) {
_leftEyeBlinkVelocity = BLINK_SPEED + randFloat() * BLINK_SPEED_VARIABILITY;
_rightEyeBlinkVelocity = BLINK_SPEED + randFloat() * BLINK_SPEED_VARIABILITY;
if (randFloat() < 0.5f) {
_leftEyeBlink = BLINK_START_VARIABILITY;
} else {
_rightEyeBlink = BLINK_START_VARIABILITY;
}
}
// use data to update fake Faceshift blendshape coefficients
calculateMouthShapes(deltaTime);
FaceTracker::updateFakeCoefficients(_leftEyeBlink,
_rightEyeBlink,
_browAudioLift,
_audioJawOpen,
_mouth2,
_mouth3,
_mouth4,
_transientBlendshapeCoefficients);
applyEyelidOffset(getOrientation());
} else {
_saccade = glm::vec3();
}
if (fixGaze) { // if debug menu turns off, use no saccade
_saccade = glm::vec3();
_leftEyeBlink = glm::clamp(_leftEyeBlink + _leftEyeBlinkVelocity * deltaTime, FULLY_OPEN, FULLY_CLOSED);
_rightEyeBlink = glm::clamp(_rightEyeBlink + _rightEyeBlinkVelocity * deltaTime, FULLY_OPEN, FULLY_CLOSED);
if (_leftEyeBlink == FULLY_CLOSED) {
_leftEyeBlinkVelocity = -BLINK_SPEED;
} else if (_leftEyeBlink == FULLY_OPEN) {
_leftEyeBlinkVelocity = 0.0f;
}
if (_rightEyeBlink == FULLY_CLOSED) {
_rightEyeBlinkVelocity = -BLINK_SPEED;
} else if (_rightEyeBlink == FULLY_OPEN) {
_rightEyeBlinkVelocity = 0.0f;
}
}
applyEyelidOffset(getOrientation());
}
void Head::computeFaceMovement(float deltaTime) {
// Update audio attack data for facial animation (eyebrows and mouth)
const float BROW_LIFT_THRESHOLD = 100.0f;
if (_audioAttack > BROW_LIFT_THRESHOLD) {
_browAudioLift += sqrtf(_audioAttack) * 0.01f;
}
_browAudioLift = glm::clamp(_browAudioLift *= 0.7f, 0.0f, 1.0f);
// use data to update fake Faceshift blendshape coefficients
calculateMouthShapes(deltaTime);
FaceTracker::updateFakeCoefficients(_leftEyeBlink,
_rightEyeBlink,
_browAudioLift,
_audioJawOpen,
_mouth2,
_mouth3,
_mouth4,
_transientBlendshapeCoefficients);
}
void Head::computeEyePosition() {
_leftEyePosition = _rightEyePosition = getPosition();
_eyePosition = getPosition();
if (_owningAvatar) {
auto skeletonModel = static_cast<Avatar*>(_owningAvatar)->getSkeletonModel();
if (skeletonModel) {
skeletonModel->getEyePositions(_leftEyePosition, _rightEyePosition);
}
}
_eyePosition = 0.5f * (_leftEyePosition + _rightEyePosition);
}
_eyePosition = calculateAverageEyePosition();
void Head::simulate(float deltaTime) {
computeAudioLoudness(deltaTime);
computeFaceMovement(deltaTime);
computeEyeMovement(deltaTime);
computeEyePosition();
}
void Head::calculateMouthShapes(float deltaTime) {

5
libraries/avatars-renderer/src/avatars-renderer/Head.h
View file

@ -83,7 +83,10 @@ public:
float getTimeWithoutTalking() const { return _timeWithoutTalking; }
protected:
glm::vec3 calculateAverageEyePosition() const { return _leftEyePosition + (_rightEyePosition - _leftEyePosition ) * 0.5f; }
void computeAudioLoudness(float deltaTime);
void computeEyeMovement(float deltaTime);
void computeFaceMovement(float deltaTime);
void computeEyePosition();
// disallow copies of the Head, copy of owning Avatar is disallowed too
Head(const Head&);

6
libraries/avatars/src/HeadData.cpp
View file

@ -28,12 +28,6 @@ HeadData::HeadData(AvatarData* owningAvatar) :
_basePitch(0.0f),
_baseRoll(0.0f),
_lookAtPosition(0.0f, 0.0f, 0.0f),
_isFaceTrackerConnected(false),
_isEyeTrackerConnected(false),
_leftEyeBlink(0.0f),
_rightEyeBlink(0.0f),
_averageLoudness(0.0f),
_browAudioLift(0.0f),
_blendshapeCoefficients(QVector<float>(0, 0.0f)),
_transientBlendshapeCoefficients(QVector<float>(0, 0.0f)),
_summedBlendshapeCoefficients(QVector<float>(0, 0.0f)),

14
libraries/avatars/src/HeadData.h
View file

@ -63,7 +63,7 @@ public:
void setBlendshapeCoefficients(const QVector<float>& blendshapeCoefficients) { _blendshapeCoefficients = blendshapeCoefficients; }
const glm::vec3& getLookAtPosition() const { return _lookAtPosition; }
void setLookAtPosition(const glm::vec3& lookAtPosition) {
void setLookAtPosition(const glm::vec3& lookAtPosition) {
if (_lookAtPosition != lookAtPosition) {
_lookAtPositionChanged = usecTimestampNow();
}
@ -85,12 +85,12 @@ protected:
glm::vec3 _lookAtPosition;
quint64 _lookAtPositionChanged { 0 };
bool _isFaceTrackerConnected;
bool _isEyeTrackerConnected;
float _leftEyeBlink;
float _rightEyeBlink;
float _averageLoudness;
float _browAudioLift;
bool _isFaceTrackerConnected { false };
bool _isEyeTrackerConnected { false };
float _leftEyeBlink { 0.0f };
float _rightEyeBlink { 0.0f };
float _averageLoudness { 0.0f };
float _browAudioLift { 0.0f };
QVector<float> _blendshapeCoefficients;
QVector<float> _transientBlendshapeCoefficients;