Mirrors/overte-JulianGro
3
0
Fork 0
mirror of https://github.com/JulianGro/overte.git synced 2025-04-07 06:02:28 +02:00
Code Issues Projects Releases Packages Wiki Activity Actions

Merge pull request #4669 from ctrlaltdavid/20506

Browse source 
CR for Job #20506 - Use audio for mouth when using DDE
This commit is contained in:
Philip Rosedale 2015-04-20 21:19:46 -07:00
commit a0a22ec94f
5 changed files with 64 additions and 39 deletions
Show stats Download patch file Download diff file Expand all files Collapse all files

1
interface/src/Application.cpp
View file

@ -1745,6 +1745,7 @@ void Application::setActiveFaceTracker() {
#endif
#ifdef HAVE_DDE
bool isUsingDDE = Menu::getInstance()->isOptionChecked(MenuOption::DDEFaceRegression);
Menu::getInstance()->getActionForOption(MenuOption::UseAudioForMouth)->setVisible(isUsingDDE);
Menu::getInstance()->getActionForOption(MenuOption::DDEFiltering)->setVisible(isUsingDDE);
Menu::getInstance()->getActionForOption(MenuOption::ResetDDETracking)->setVisible(isUsingDDE);
DependencyManager::get<DdeFaceTracker>()->setEnabled(isUsingDDE);

2
interface/src/Menu.cpp
View file

@ -379,6 +379,8 @@ Menu::Menu() {
}
#ifdef HAVE_DDE
faceTrackingMenu->addSeparator();
QAction* useAudioForMouth = addCheckableActionToQMenuAndActionHash(faceTrackingMenu, MenuOption::UseAudioForMouth, 0, true);
useAudioForMouth->setVisible(false);
QAction* ddeFiltering = addCheckableActionToQMenuAndActionHash(faceTrackingMenu, MenuOption::DDEFiltering, 0, true);
ddeFiltering->setVisible(false);
QAction* ddeFaceTrackerReset = addActionToQMenuAndActionHash(faceTrackingMenu, MenuOption::ResetDDETracking,

3
interface/src/Menu.h
View file

@ -187,6 +187,7 @@ namespace MenuOption {
const QString OctreeStats = "Entity Statistics";
const QString OffAxisProjection = "Off-Axis Projection";
const QString OnlyDisplayTopTen = "Only Display Top Ten";
const QString PackageModel = "Package Model...";
const QString Pair = "Pair";
const QString PipelineWarnings = "Log Render Pipeline Warnings";
const QString Preferences = "Preferences...";
@ -245,7 +246,7 @@ namespace MenuOption {
const QString ToolWindow = "Tool Window";
const QString TransmitterDrive = "Transmitter Drive";
const QString TurnWithHead = "Turn using Head";
const QString PackageModel = "Package Model...";
const QString UseAudioForMouth = "Use Audio for Mouth";
const QString VisibleToEveryone = "Everyone";
const QString VisibleToFriends = "Friends";
const QString VisibleToNoOne = "No one";

96
interface/src/avatar/Head.cpp
View file

@ -76,12 +76,41 @@ void Head::simulate(float deltaTime, bool isMine, bool billboard) {
if (isMine) {
MyAvatar* myAvatar = static_cast<MyAvatar*>(_owningAvatar);
// Update audio trailing average for rendering facial animations
const float AUDIO_AVERAGING_SECS = 0.05f;
const float AUDIO_LONG_TERM_AVERAGING_SECS = 30.0f;
_averageLoudness = glm::mix(_averageLoudness, _audioLoudness, glm::min(deltaTime / AUDIO_AVERAGING_SECS, 1.0f));
if (_longTermAverageLoudness == -1.0) {
_longTermAverageLoudness = _averageLoudness;
} else {
_longTermAverageLoudness = glm::mix(_longTermAverageLoudness, _averageLoudness, glm::min(deltaTime / AUDIO_LONG_TERM_AVERAGING_SECS, 1.0f));
}
// Only use face trackers when not playing back a recording.
if (!myAvatar->isPlaying()) {
FaceTracker* faceTracker = Application::getInstance()->getActiveFaceTracker();
_isFaceTrackerConnected = faceTracker != NULL;
if (_isFaceTrackerConnected) {
_blendshapeCoefficients = faceTracker->getBlendshapeCoefficients();
if (typeid(*faceTracker) == typeid(DdeFaceTracker)
&& Menu::getInstance()->isOptionChecked(MenuOption::UseAudioForMouth)) {
calculateMouthShapes();
const int JAW_OPEN_BLENDSHAPE = 21;
const int MMMM_BLENDSHAPE = 34;
const int FUNNEL_BLENDSHAPE = 40;
const int SMILE_LEFT_BLENDSHAPE = 28;
const int SMILE_RIGHT_BLENDSHAPE = 29;
_blendshapeCoefficients[JAW_OPEN_BLENDSHAPE] += _audioJawOpen;
_blendshapeCoefficients[SMILE_LEFT_BLENDSHAPE] += _mouth4;
_blendshapeCoefficients[SMILE_RIGHT_BLENDSHAPE] += _mouth4;
_blendshapeCoefficients[MMMM_BLENDSHAPE] += _mouth2;
_blendshapeCoefficients[FUNNEL_BLENDSHAPE] += _mouth3;
}
}
}
// Twist the upper body to follow the rotation of the head, but only do this with my avatar,
@ -92,17 +121,6 @@ void Head::simulate(float deltaTime, bool isMine, bool billboard) {
setTorsoTwist(currentTwist + (getFinalYaw() * BODY_FOLLOW_HEAD_FACTOR - currentTwist) * BODY_FOLLOW_HEAD_YAW_RATE);
}
// Update audio trailing average for rendering facial animations
const float AUDIO_AVERAGING_SECS = 0.05f;
const float AUDIO_LONG_TERM_AVERAGING_SECS = 30.0f;
_averageLoudness = glm::mix(_averageLoudness, _audioLoudness, glm::min(deltaTime / AUDIO_AVERAGING_SECS, 1.0f));
if (_longTermAverageLoudness == -1.0) {
_longTermAverageLoudness = _averageLoudness;
} else {
_longTermAverageLoudness = glm::mix(_longTermAverageLoudness, _averageLoudness, glm::min(deltaTime / AUDIO_LONG_TERM_AVERAGING_SECS, 1.0f));
}
if (!(_isFaceTrackerConnected || billboard)) {
// Update eye saccades
const float AVERAGE_MICROSACCADE_INTERVAL = 0.50f;
@ -177,33 +195,7 @@ void Head::simulate(float deltaTime, bool isMine, bool billboard) {
}
// use data to update fake Faceshift blendshape coefficients
const float JAW_OPEN_SCALE = 0.015f;
const float JAW_OPEN_RATE = 0.9f;
const float JAW_CLOSE_RATE = 0.90f;
float audioDelta = sqrtf(glm::max(_averageLoudness - _longTermAverageLoudness, 0.0f)) * JAW_OPEN_SCALE;
if (audioDelta > _audioJawOpen) {
_audioJawOpen += (audioDelta - _audioJawOpen) * JAW_OPEN_RATE;
} else {
_audioJawOpen *= JAW_CLOSE_RATE;
}
_audioJawOpen = glm::clamp(_audioJawOpen, 0.0f, 1.0f);
// _mouth2 = "mmmm" shape
// _mouth3 = "funnel" shape
// _mouth4 = "smile" shape
const float FUNNEL_PERIOD = 0.985f;
const float FUNNEL_RANDOM_PERIOD = 0.01f;
const float MMMM_POWER = 0.25f;
const float MMMM_PERIOD = 0.91f;
const float MMMM_RANDOM_PERIOD = 0.15f;
const float SMILE_PERIOD = 0.925f;
const float SMILE_RANDOM_PERIOD = 0.05f;
_mouth3 = glm::mix(_audioJawOpen, _mouth3, FUNNEL_PERIOD + randFloat() * FUNNEL_RANDOM_PERIOD);
_mouth2 = glm::mix(_audioJawOpen * MMMM_POWER, _mouth2, MMMM_PERIOD + randFloat() * MMMM_RANDOM_PERIOD);
_mouth4 = glm::mix(_audioJawOpen, _mouth4, SMILE_PERIOD + randFloat() * SMILE_RANDOM_PERIOD);
calculateMouthShapes();
DependencyManager::get<Faceshift>()->updateFakeCoefficients(_leftEyeBlink,
_rightEyeBlink,
_browAudioLift,
@ -230,6 +222,34 @@ void Head::simulate(float deltaTime, bool isMine, bool billboard) {
}
void Head::calculateMouthShapes() {
const float JAW_OPEN_SCALE = 0.015f;
const float JAW_OPEN_RATE = 0.9f;
const float JAW_CLOSE_RATE = 0.90f;
float audioDelta = sqrtf(glm::max(_averageLoudness - _longTermAverageLoudness, 0.0f)) * JAW_OPEN_SCALE;
if (audioDelta > _audioJawOpen) {
_audioJawOpen += (audioDelta - _audioJawOpen) * JAW_OPEN_RATE;
} else {
_audioJawOpen *= JAW_CLOSE_RATE;
}
_audioJawOpen = glm::clamp(_audioJawOpen, 0.0f, 1.0f);
// _mouth2 = "mmmm" shape
// _mouth3 = "funnel" shape
// _mouth4 = "smile" shape
const float FUNNEL_PERIOD = 0.985f;
const float FUNNEL_RANDOM_PERIOD = 0.01f;
const float MMMM_POWER = 0.25f;
const float MMMM_PERIOD = 0.91f;
const float MMMM_RANDOM_PERIOD = 0.15f;
const float SMILE_PERIOD = 0.925f;
const float SMILE_RANDOM_PERIOD = 0.05f;
_mouth3 = glm::mix(_audioJawOpen, _mouth3, FUNNEL_PERIOD + randFloat() * FUNNEL_RANDOM_PERIOD);
_mouth2 = glm::mix(_audioJawOpen * MMMM_POWER, _mouth2, MMMM_PERIOD + randFloat() * MMMM_RANDOM_PERIOD);
_mouth4 = glm::mix(_audioJawOpen, _mouth4, SMILE_PERIOD + randFloat() * SMILE_RANDOM_PERIOD);
}
void Head::relaxLean(float deltaTime) {
// restore rotation, lean to neutral positions
const float LEAN_RELAXATION_PERIOD = 0.25f; // seconds

1
interface/src/avatar/Head.h
View file

@ -154,6 +154,7 @@ private:
// private methods
void renderLookatVectors(glm::vec3 leftEyePosition, glm::vec3 rightEyePosition, glm::vec3 lookatPosition);
void calculateMouthShapes();
friend class FaceModel;
};