diff --git a/interface/src/Application.cpp b/interface/src/Application.cpp index 9e00444fa5..a0f2b4cb05 100644 --- a/interface/src/Application.cpp +++ b/interface/src/Application.cpp @@ -1745,6 +1745,7 @@ void Application::setActiveFaceTracker() { #endif #ifdef HAVE_DDE bool isUsingDDE = Menu::getInstance()->isOptionChecked(MenuOption::DDEFaceRegression); + Menu::getInstance()->getActionForOption(MenuOption::UseAudioForMouth)->setVisible(isUsingDDE); Menu::getInstance()->getActionForOption(MenuOption::DDEFiltering)->setVisible(isUsingDDE); Menu::getInstance()->getActionForOption(MenuOption::ResetDDETracking)->setVisible(isUsingDDE); DependencyManager::get()->setEnabled(isUsingDDE); diff --git a/interface/src/Menu.cpp b/interface/src/Menu.cpp index 6b9075e917..df18e941fa 100644 --- a/interface/src/Menu.cpp +++ b/interface/src/Menu.cpp @@ -379,6 +379,8 @@ Menu::Menu() { } #ifdef HAVE_DDE faceTrackingMenu->addSeparator(); + QAction* useAudioForMouth = addCheckableActionToQMenuAndActionHash(faceTrackingMenu, MenuOption::UseAudioForMouth, 0, true); + useAudioForMouth->setVisible(false); QAction* ddeFiltering = addCheckableActionToQMenuAndActionHash(faceTrackingMenu, MenuOption::DDEFiltering, 0, true); ddeFiltering->setVisible(false); QAction* ddeFaceTrackerReset = addActionToQMenuAndActionHash(faceTrackingMenu, MenuOption::ResetDDETracking, diff --git a/interface/src/Menu.h b/interface/src/Menu.h index 3c31db834a..0df1b5a83a 100644 --- a/interface/src/Menu.h +++ b/interface/src/Menu.h @@ -187,6 +187,7 @@ namespace MenuOption { const QString OctreeStats = "Entity Statistics"; const QString OffAxisProjection = "Off-Axis Projection"; const QString OnlyDisplayTopTen = "Only Display Top Ten"; + const QString PackageModel = "Package Model..."; const QString Pair = "Pair"; const QString PipelineWarnings = "Log Render Pipeline Warnings"; const QString Preferences = "Preferences..."; @@ -245,7 +246,7 @@ namespace MenuOption { const QString ToolWindow = "Tool Window"; const QString TransmitterDrive = "Transmitter Drive"; const QString TurnWithHead = "Turn using Head"; - const QString PackageModel = "Package Model..."; + const QString UseAudioForMouth = "Use Audio for Mouth"; const QString VisibleToEveryone = "Everyone"; const QString VisibleToFriends = "Friends"; const QString VisibleToNoOne = "No one"; diff --git a/interface/src/avatar/Head.cpp b/interface/src/avatar/Head.cpp index b90b693139..b84fd23614 100644 --- a/interface/src/avatar/Head.cpp +++ b/interface/src/avatar/Head.cpp @@ -76,12 +76,41 @@ void Head::simulate(float deltaTime, bool isMine, bool billboard) { if (isMine) { MyAvatar* myAvatar = static_cast(_owningAvatar); + // Update audio trailing average for rendering facial animations + const float AUDIO_AVERAGING_SECS = 0.05f; + const float AUDIO_LONG_TERM_AVERAGING_SECS = 30.0f; + _averageLoudness = glm::mix(_averageLoudness, _audioLoudness, glm::min(deltaTime / AUDIO_AVERAGING_SECS, 1.0f)); + + if (_longTermAverageLoudness == -1.0) { + _longTermAverageLoudness = _averageLoudness; + } else { + _longTermAverageLoudness = glm::mix(_longTermAverageLoudness, _averageLoudness, glm::min(deltaTime / AUDIO_LONG_TERM_AVERAGING_SECS, 1.0f)); + } + // Only use face trackers when not playing back a recording. if (!myAvatar->isPlaying()) { FaceTracker* faceTracker = Application::getInstance()->getActiveFaceTracker(); _isFaceTrackerConnected = faceTracker != NULL; if (_isFaceTrackerConnected) { _blendshapeCoefficients = faceTracker->getBlendshapeCoefficients(); + + if (typeid(*faceTracker) == typeid(DdeFaceTracker) + && Menu::getInstance()->isOptionChecked(MenuOption::UseAudioForMouth)) { + + calculateMouthShapes(); + + const int JAW_OPEN_BLENDSHAPE = 21; + const int MMMM_BLENDSHAPE = 34; + const int FUNNEL_BLENDSHAPE = 40; + const int SMILE_LEFT_BLENDSHAPE = 28; + const int SMILE_RIGHT_BLENDSHAPE = 29; + _blendshapeCoefficients[JAW_OPEN_BLENDSHAPE] += _audioJawOpen; + _blendshapeCoefficients[SMILE_LEFT_BLENDSHAPE] += _mouth4; + _blendshapeCoefficients[SMILE_RIGHT_BLENDSHAPE] += _mouth4; + _blendshapeCoefficients[MMMM_BLENDSHAPE] += _mouth2; + _blendshapeCoefficients[FUNNEL_BLENDSHAPE] += _mouth3; + + } } } // Twist the upper body to follow the rotation of the head, but only do this with my avatar, @@ -92,17 +121,6 @@ void Head::simulate(float deltaTime, bool isMine, bool billboard) { setTorsoTwist(currentTwist + (getFinalYaw() * BODY_FOLLOW_HEAD_FACTOR - currentTwist) * BODY_FOLLOW_HEAD_YAW_RATE); } - // Update audio trailing average for rendering facial animations - const float AUDIO_AVERAGING_SECS = 0.05f; - const float AUDIO_LONG_TERM_AVERAGING_SECS = 30.0f; - _averageLoudness = glm::mix(_averageLoudness, _audioLoudness, glm::min(deltaTime / AUDIO_AVERAGING_SECS, 1.0f)); - - if (_longTermAverageLoudness == -1.0) { - _longTermAverageLoudness = _averageLoudness; - } else { - _longTermAverageLoudness = glm::mix(_longTermAverageLoudness, _averageLoudness, glm::min(deltaTime / AUDIO_LONG_TERM_AVERAGING_SECS, 1.0f)); - } - if (!(_isFaceTrackerConnected || billboard)) { // Update eye saccades const float AVERAGE_MICROSACCADE_INTERVAL = 0.50f; @@ -177,33 +195,7 @@ void Head::simulate(float deltaTime, bool isMine, bool billboard) { } // use data to update fake Faceshift blendshape coefficients - - const float JAW_OPEN_SCALE = 0.015f; - const float JAW_OPEN_RATE = 0.9f; - const float JAW_CLOSE_RATE = 0.90f; - float audioDelta = sqrtf(glm::max(_averageLoudness - _longTermAverageLoudness, 0.0f)) * JAW_OPEN_SCALE; - if (audioDelta > _audioJawOpen) { - _audioJawOpen += (audioDelta - _audioJawOpen) * JAW_OPEN_RATE; - } else { - _audioJawOpen *= JAW_CLOSE_RATE; - } - _audioJawOpen = glm::clamp(_audioJawOpen, 0.0f, 1.0f); - - // _mouth2 = "mmmm" shape - // _mouth3 = "funnel" shape - // _mouth4 = "smile" shape - const float FUNNEL_PERIOD = 0.985f; - const float FUNNEL_RANDOM_PERIOD = 0.01f; - const float MMMM_POWER = 0.25f; - const float MMMM_PERIOD = 0.91f; - const float MMMM_RANDOM_PERIOD = 0.15f; - const float SMILE_PERIOD = 0.925f; - const float SMILE_RANDOM_PERIOD = 0.05f; - - _mouth3 = glm::mix(_audioJawOpen, _mouth3, FUNNEL_PERIOD + randFloat() * FUNNEL_RANDOM_PERIOD); - _mouth2 = glm::mix(_audioJawOpen * MMMM_POWER, _mouth2, MMMM_PERIOD + randFloat() * MMMM_RANDOM_PERIOD); - _mouth4 = glm::mix(_audioJawOpen, _mouth4, SMILE_PERIOD + randFloat() * SMILE_RANDOM_PERIOD); - + calculateMouthShapes(); DependencyManager::get()->updateFakeCoefficients(_leftEyeBlink, _rightEyeBlink, _browAudioLift, @@ -230,6 +222,34 @@ void Head::simulate(float deltaTime, bool isMine, bool billboard) { } +void Head::calculateMouthShapes() { + const float JAW_OPEN_SCALE = 0.015f; + const float JAW_OPEN_RATE = 0.9f; + const float JAW_CLOSE_RATE = 0.90f; + float audioDelta = sqrtf(glm::max(_averageLoudness - _longTermAverageLoudness, 0.0f)) * JAW_OPEN_SCALE; + if (audioDelta > _audioJawOpen) { + _audioJawOpen += (audioDelta - _audioJawOpen) * JAW_OPEN_RATE; + } else { + _audioJawOpen *= JAW_CLOSE_RATE; + } + _audioJawOpen = glm::clamp(_audioJawOpen, 0.0f, 1.0f); + + // _mouth2 = "mmmm" shape + // _mouth3 = "funnel" shape + // _mouth4 = "smile" shape + const float FUNNEL_PERIOD = 0.985f; + const float FUNNEL_RANDOM_PERIOD = 0.01f; + const float MMMM_POWER = 0.25f; + const float MMMM_PERIOD = 0.91f; + const float MMMM_RANDOM_PERIOD = 0.15f; + const float SMILE_PERIOD = 0.925f; + const float SMILE_RANDOM_PERIOD = 0.05f; + + _mouth3 = glm::mix(_audioJawOpen, _mouth3, FUNNEL_PERIOD + randFloat() * FUNNEL_RANDOM_PERIOD); + _mouth2 = glm::mix(_audioJawOpen * MMMM_POWER, _mouth2, MMMM_PERIOD + randFloat() * MMMM_RANDOM_PERIOD); + _mouth4 = glm::mix(_audioJawOpen, _mouth4, SMILE_PERIOD + randFloat() * SMILE_RANDOM_PERIOD); +} + void Head::relaxLean(float deltaTime) { // restore rotation, lean to neutral positions const float LEAN_RELAXATION_PERIOD = 0.25f; // seconds diff --git a/interface/src/avatar/Head.h b/interface/src/avatar/Head.h index 5465d74172..d6d4cc1352 100644 --- a/interface/src/avatar/Head.h +++ b/interface/src/avatar/Head.h @@ -154,6 +154,7 @@ private: // private methods void renderLookatVectors(glm::vec3 leftEyePosition, glm::vec3 rightEyePosition, glm::vec3 lookatPosition); + void calculateMouthShapes(); friend class FaceModel; };