From f4df223d2337870dbb14177c9561d403c13432d5 Mon Sep 17 00:00:00 2001 From: Andrew Meadows Date: Tue, 9 May 2017 14:48:42 -0700 Subject: [PATCH 1/5] cleanup Head::simulation() code --- interface/src/avatar/MyHead.cpp | 27 ++- .../src/avatars-renderer/Head.cpp | 199 +++++++++--------- .../src/avatars-renderer/Head.h | 5 +- libraries/avatars/src/HeadData.cpp | 6 - libraries/avatars/src/HeadData.h | 14 +- 5 files changed, 127 insertions(+), 124 deletions(-) diff --git a/interface/src/avatar/MyHead.cpp b/interface/src/avatar/MyHead.cpp index 34a75c5461..793fbb79c4 100644 --- a/interface/src/avatar/MyHead.cpp +++ b/interface/src/avatar/MyHead.cpp @@ -44,14 +44,17 @@ glm::quat MyHead::getCameraOrientation() const { void MyHead::simulate(float deltaTime) { auto player = DependencyManager::get(); // Only use face trackers when not playing back a recording. - if (!player->isPlaying()) { + if (player->isPlaying()) { + Parent::simulate(deltaTime); + } else { + computeAudioLoudness(deltaTime); + FaceTracker* faceTracker = qApp->getActiveFaceTracker(); - _isFaceTrackerConnected = faceTracker != NULL && !faceTracker->isMuted(); + _isFaceTrackerConnected = faceTracker && !faceTracker->isMuted(); if (_isFaceTrackerConnected) { _transientBlendshapeCoefficients = faceTracker->getBlendshapeCoefficients(); if (typeid(*faceTracker) == typeid(DdeFaceTracker)) { - if (Menu::getInstance()->isOptionChecked(MenuOption::UseAudioForMouth)) { calculateMouthShapes(deltaTime); @@ -68,9 +71,19 @@ void MyHead::simulate(float deltaTime) { } applyEyelidOffset(getFinalOrientationInWorldFrame()); } - } + } else { + computeFaceMovement(deltaTime); + } + auto eyeTracker = DependencyManager::get(); - _isEyeTrackerConnected = eyeTracker->isTracking(); + _isEyeTrackerConnected = eyeTracker && eyeTracker->isTracking(); + if (_isEyeTrackerConnected) { + // TODO? figure out where EyeTracker data harvested. Move it here? + _saccade = glm::vec3(); + } else { + computeEyeMovement(deltaTime); + } + } - Parent::simulate(deltaTime); -} \ No newline at end of file + computeEyePosition(); +} diff --git a/libraries/avatars-renderer/src/avatars-renderer/Head.cpp b/libraries/avatars-renderer/src/avatars-renderer/Head.cpp index 93fe246266..b4b0929c0c 100644 --- a/libraries/avatars-renderer/src/avatars-renderer/Head.cpp +++ b/libraries/avatars-renderer/src/avatars-renderer/Head.cpp @@ -23,9 +23,10 @@ #include "Avatar.h" +const float NORMAL_HZ = 60.0f; // the update rate the constant values were tuned for + using namespace std; -static bool fixGaze { false }; static bool disableEyelidAdjustment { false }; Head::Head(Avatar* owningAvatar) : @@ -42,17 +43,11 @@ void Head::reset() { _baseYaw = _basePitch = _baseRoll = 0.0f; } -void Head::simulate(float deltaTime) { - const float NORMAL_HZ = 60.0f; // the update rate the constant values were tuned for - +void Head::computeAudioLoudness(float deltaTime) { // grab the audio loudness from the owning avatar, if we have one - float audioLoudness = 0.0f; + float audioLoudness = _owningAvatar ? _owningAvatar->getAudioLoudness() : 0.0f; - if (_owningAvatar) { - audioLoudness = _owningAvatar->getAudioLoudness(); - } - - // Update audio trailing average for rendering facial animations + // Update audio trailing average for rendering facial animations const float AUDIO_AVERAGING_SECS = 0.05f; const float AUDIO_LONG_TERM_AVERAGING_SECS = 30.0f; _averageLoudness = glm::mix(_averageLoudness, audioLoudness, glm::min(deltaTime / AUDIO_AVERAGING_SECS, 1.0f)); @@ -63,116 +58,114 @@ void Head::simulate(float deltaTime) { _longTermAverageLoudness = glm::mix(_longTermAverageLoudness, _averageLoudness, glm::min(deltaTime / AUDIO_LONG_TERM_AVERAGING_SECS, 1.0f)); } - if (!_isFaceTrackerConnected) { - if (!_isEyeTrackerConnected) { - // Update eye saccades - const float AVERAGE_MICROSACCADE_INTERVAL = 1.0f; - const float AVERAGE_SACCADE_INTERVAL = 6.0f; - const float MICROSACCADE_MAGNITUDE = 0.002f; - const float SACCADE_MAGNITUDE = 0.04f; - const float NOMINAL_FRAME_RATE = 60.0f; + float audioAttackAveragingRate = (10.0f - deltaTime * NORMAL_HZ) / 10.0f; // --> 0.9 at 60 Hz + _audioAttack = audioAttackAveragingRate * _audioAttack + + (1.0f - audioAttackAveragingRate) * fabs((audioLoudness - _longTermAverageLoudness) - _lastLoudness); + _lastLoudness = (audioLoudness - _longTermAverageLoudness); +} - if (randFloat() < deltaTime / AVERAGE_MICROSACCADE_INTERVAL) { - _saccadeTarget = MICROSACCADE_MAGNITUDE * randVector(); - } else if (randFloat() < deltaTime / AVERAGE_SACCADE_INTERVAL) { - _saccadeTarget = SACCADE_MAGNITUDE * randVector(); - } - _saccade += (_saccadeTarget - _saccade) * pow(0.5f, NOMINAL_FRAME_RATE * deltaTime); - } else { - _saccade = glm::vec3(); - } +void Head::computeEyeMovement(float deltaTime) { + // Update eye saccades + const float AVERAGE_MICROSACCADE_INTERVAL = 1.0f; + const float AVERAGE_SACCADE_INTERVAL = 6.0f; + const float MICROSACCADE_MAGNITUDE = 0.002f; + const float SACCADE_MAGNITUDE = 0.04f; + const float NOMINAL_FRAME_RATE = 60.0f; - // Detect transition from talking to not; force blink after that and a delay - bool forceBlink = false; - const float TALKING_LOUDNESS = 100.0f; - const float BLINK_AFTER_TALKING = 0.25f; - _timeWithoutTalking += deltaTime; - if ((_averageLoudness - _longTermAverageLoudness) > TALKING_LOUDNESS) { - _timeWithoutTalking = 0.0f; - } else if (_timeWithoutTalking - deltaTime < BLINK_AFTER_TALKING && _timeWithoutTalking >= BLINK_AFTER_TALKING) { - forceBlink = true; - } + if (randFloat() < deltaTime / AVERAGE_MICROSACCADE_INTERVAL) { + _saccadeTarget = MICROSACCADE_MAGNITUDE * randVector(); + } else if (randFloat() < deltaTime / AVERAGE_SACCADE_INTERVAL) { + _saccadeTarget = SACCADE_MAGNITUDE * randVector(); + } + _saccade += (_saccadeTarget - _saccade) * pow(0.5f, NOMINAL_FRAME_RATE * deltaTime); - // Update audio attack data for facial animation (eyebrows and mouth) - float audioAttackAveragingRate = (10.0f - deltaTime * NORMAL_HZ) / 10.0f; // --> 0.9 at 60 Hz - _audioAttack = audioAttackAveragingRate * _audioAttack + - (1.0f - audioAttackAveragingRate) * fabs((audioLoudness - _longTermAverageLoudness) - _lastLoudness); - _lastLoudness = (audioLoudness - _longTermAverageLoudness); + // Detect transition from talking to not; force blink after that and a delay + bool forceBlink = false; + const float TALKING_LOUDNESS = 100.0f; + const float BLINK_AFTER_TALKING = 0.25f; + _timeWithoutTalking += deltaTime; + if ((_averageLoudness - _longTermAverageLoudness) > TALKING_LOUDNESS) { + _timeWithoutTalking = 0.0f; + } else if (_timeWithoutTalking - deltaTime < BLINK_AFTER_TALKING && _timeWithoutTalking >= BLINK_AFTER_TALKING) { + forceBlink = true; + } - const float BROW_LIFT_THRESHOLD = 100.0f; - if (_audioAttack > BROW_LIFT_THRESHOLD) { - _browAudioLift += sqrtf(_audioAttack) * 0.01f; - } - _browAudioLift = glm::clamp(_browAudioLift *= 0.7f, 0.0f, 1.0f); - - const float BLINK_SPEED = 10.0f; - const float BLINK_SPEED_VARIABILITY = 1.0f; - const float BLINK_START_VARIABILITY = 0.25f; - const float FULLY_OPEN = 0.0f; - const float FULLY_CLOSED = 1.0f; - if (_leftEyeBlinkVelocity == 0.0f && _rightEyeBlinkVelocity == 0.0f) { - // no blinking when brows are raised; blink less with increasing loudness - const float BASE_BLINK_RATE = 15.0f / 60.0f; - const float ROOT_LOUDNESS_TO_BLINK_INTERVAL = 0.25f; - if (forceBlink || (_browAudioLift < EPSILON && shouldDo(glm::max(1.0f, sqrt(fabs(_averageLoudness - _longTermAverageLoudness)) * - ROOT_LOUDNESS_TO_BLINK_INTERVAL) / BASE_BLINK_RATE, deltaTime))) { - _leftEyeBlinkVelocity = BLINK_SPEED + randFloat() * BLINK_SPEED_VARIABILITY; - _rightEyeBlinkVelocity = BLINK_SPEED + randFloat() * BLINK_SPEED_VARIABILITY; - if (randFloat() < 0.5f) { - _leftEyeBlink = BLINK_START_VARIABILITY; - } else { - _rightEyeBlink = BLINK_START_VARIABILITY; - } - } - } else { - _leftEyeBlink = glm::clamp(_leftEyeBlink + _leftEyeBlinkVelocity * deltaTime, FULLY_OPEN, FULLY_CLOSED); - _rightEyeBlink = glm::clamp(_rightEyeBlink + _rightEyeBlinkVelocity * deltaTime, FULLY_OPEN, FULLY_CLOSED); - - if (_leftEyeBlink == FULLY_CLOSED) { - _leftEyeBlinkVelocity = -BLINK_SPEED; - - } else if (_leftEyeBlink == FULLY_OPEN) { - _leftEyeBlinkVelocity = 0.0f; - } - if (_rightEyeBlink == FULLY_CLOSED) { - _rightEyeBlinkVelocity = -BLINK_SPEED; - - } else if (_rightEyeBlink == FULLY_OPEN) { - _rightEyeBlinkVelocity = 0.0f; + const float BLINK_SPEED = 10.0f; + const float BLINK_SPEED_VARIABILITY = 1.0f; + const float BLINK_START_VARIABILITY = 0.25f; + const float FULLY_OPEN = 0.0f; + const float FULLY_CLOSED = 1.0f; + if (_leftEyeBlinkVelocity == 0.0f && _rightEyeBlinkVelocity == 0.0f) { + // no blinking when brows are raised; blink less with increasing loudness + const float BASE_BLINK_RATE = 15.0f / 60.0f; + const float ROOT_LOUDNESS_TO_BLINK_INTERVAL = 0.25f; + if (forceBlink || (_browAudioLift < EPSILON && shouldDo(glm::max(1.0f, sqrt(fabs(_averageLoudness - _longTermAverageLoudness)) * + ROOT_LOUDNESS_TO_BLINK_INTERVAL) / BASE_BLINK_RATE, deltaTime))) { + _leftEyeBlinkVelocity = BLINK_SPEED + randFloat() * BLINK_SPEED_VARIABILITY; + _rightEyeBlinkVelocity = BLINK_SPEED + randFloat() * BLINK_SPEED_VARIABILITY; + if (randFloat() < 0.5f) { + _leftEyeBlink = BLINK_START_VARIABILITY; + } else { + _rightEyeBlink = BLINK_START_VARIABILITY; } } - - // use data to update fake Faceshift blendshape coefficients - calculateMouthShapes(deltaTime); - FaceTracker::updateFakeCoefficients(_leftEyeBlink, - _rightEyeBlink, - _browAudioLift, - _audioJawOpen, - _mouth2, - _mouth3, - _mouth4, - _transientBlendshapeCoefficients); - - applyEyelidOffset(getOrientation()); - } else { - _saccade = glm::vec3(); - } - if (fixGaze) { // if debug menu turns off, use no saccade - _saccade = glm::vec3(); + _leftEyeBlink = glm::clamp(_leftEyeBlink + _leftEyeBlinkVelocity * deltaTime, FULLY_OPEN, FULLY_CLOSED); + _rightEyeBlink = glm::clamp(_rightEyeBlink + _rightEyeBlinkVelocity * deltaTime, FULLY_OPEN, FULLY_CLOSED); + + if (_leftEyeBlink == FULLY_CLOSED) { + _leftEyeBlinkVelocity = -BLINK_SPEED; + + } else if (_leftEyeBlink == FULLY_OPEN) { + _leftEyeBlinkVelocity = 0.0f; + } + if (_rightEyeBlink == FULLY_CLOSED) { + _rightEyeBlinkVelocity = -BLINK_SPEED; + + } else if (_rightEyeBlink == FULLY_OPEN) { + _rightEyeBlinkVelocity = 0.0f; + } } + applyEyelidOffset(getOrientation()); +} + +void Head::computeFaceMovement(float deltaTime) { + // Update audio attack data for facial animation (eyebrows and mouth) + const float BROW_LIFT_THRESHOLD = 100.0f; + if (_audioAttack > BROW_LIFT_THRESHOLD) { + _browAudioLift += sqrtf(_audioAttack) * 0.01f; + } + _browAudioLift = glm::clamp(_browAudioLift *= 0.7f, 0.0f, 1.0f); + + // use data to update fake Faceshift blendshape coefficients + calculateMouthShapes(deltaTime); + FaceTracker::updateFakeCoefficients(_leftEyeBlink, + _rightEyeBlink, + _browAudioLift, + _audioJawOpen, + _mouth2, + _mouth3, + _mouth4, + _transientBlendshapeCoefficients); +} + +void Head::computeEyePosition() { _leftEyePosition = _rightEyePosition = getPosition(); - _eyePosition = getPosition(); - if (_owningAvatar) { auto skeletonModel = static_cast(_owningAvatar)->getSkeletonModel(); if (skeletonModel) { skeletonModel->getEyePositions(_leftEyePosition, _rightEyePosition); } } + _eyePosition = 0.5f * (_leftEyePosition + _rightEyePosition); +} - _eyePosition = calculateAverageEyePosition(); +void Head::simulate(float deltaTime) { + computeAudioLoudness(deltaTime); + computeFaceMovement(deltaTime); + computeEyeMovement(deltaTime); + computeEyePosition(); } void Head::calculateMouthShapes(float deltaTime) { diff --git a/libraries/avatars-renderer/src/avatars-renderer/Head.h b/libraries/avatars-renderer/src/avatars-renderer/Head.h index aea6a41528..39331500b5 100644 --- a/libraries/avatars-renderer/src/avatars-renderer/Head.h +++ b/libraries/avatars-renderer/src/avatars-renderer/Head.h @@ -83,7 +83,10 @@ public: float getTimeWithoutTalking() const { return _timeWithoutTalking; } protected: - glm::vec3 calculateAverageEyePosition() const { return _leftEyePosition + (_rightEyePosition - _leftEyePosition ) * 0.5f; } + void computeAudioLoudness(float deltaTime); + void computeEyeMovement(float deltaTime); + void computeFaceMovement(float deltaTime); + void computeEyePosition(); // disallow copies of the Head, copy of owning Avatar is disallowed too Head(const Head&); diff --git a/libraries/avatars/src/HeadData.cpp b/libraries/avatars/src/HeadData.cpp index 2e4eec73a8..2704b6539c 100644 --- a/libraries/avatars/src/HeadData.cpp +++ b/libraries/avatars/src/HeadData.cpp @@ -28,12 +28,6 @@ HeadData::HeadData(AvatarData* owningAvatar) : _basePitch(0.0f), _baseRoll(0.0f), _lookAtPosition(0.0f, 0.0f, 0.0f), - _isFaceTrackerConnected(false), - _isEyeTrackerConnected(false), - _leftEyeBlink(0.0f), - _rightEyeBlink(0.0f), - _averageLoudness(0.0f), - _browAudioLift(0.0f), _blendshapeCoefficients(QVector(0, 0.0f)), _transientBlendshapeCoefficients(QVector(0, 0.0f)), _summedBlendshapeCoefficients(QVector(0, 0.0f)), diff --git a/libraries/avatars/src/HeadData.h b/libraries/avatars/src/HeadData.h index 9b28616b3f..be9d54e93e 100644 --- a/libraries/avatars/src/HeadData.h +++ b/libraries/avatars/src/HeadData.h @@ -63,7 +63,7 @@ public: void setBlendshapeCoefficients(const QVector& blendshapeCoefficients) { _blendshapeCoefficients = blendshapeCoefficients; } const glm::vec3& getLookAtPosition() const { return _lookAtPosition; } - void setLookAtPosition(const glm::vec3& lookAtPosition) { + void setLookAtPosition(const glm::vec3& lookAtPosition) { if (_lookAtPosition != lookAtPosition) { _lookAtPositionChanged = usecTimestampNow(); } @@ -85,12 +85,12 @@ protected: glm::vec3 _lookAtPosition; quint64 _lookAtPositionChanged { 0 }; - bool _isFaceTrackerConnected; - bool _isEyeTrackerConnected; - float _leftEyeBlink; - float _rightEyeBlink; - float _averageLoudness; - float _browAudioLift; + bool _isFaceTrackerConnected { false }; + bool _isEyeTrackerConnected { false }; + float _leftEyeBlink { 0.0f }; + float _rightEyeBlink { 0.0f }; + float _averageLoudness { 0.0f }; + float _browAudioLift { 0.0f }; QVector _blendshapeCoefficients; QVector _transientBlendshapeCoefficients; From 04827c3a2a39544cdddda68d0bde52d9beb2a512 Mon Sep 17 00:00:00 2001 From: Andrew Meadows Date: Thu, 11 May 2017 14:51:52 -0700 Subject: [PATCH 2/5] use const auto& where appropritate --- libraries/avatars/src/AvatarData.cpp | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/libraries/avatars/src/AvatarData.cpp b/libraries/avatars/src/AvatarData.cpp index 51b69711dc..e2188478ed 100644 --- a/libraries/avatars/src/AvatarData.cpp +++ b/libraries/avatars/src/AvatarData.cpp @@ -445,7 +445,7 @@ QByteArray AvatarData::toByteArray(AvatarDataDetail dataDetail, quint64 lastSent if (hasFaceTrackerInfo) { auto startSection = destinationBuffer; auto faceTrackerInfo = reinterpret_cast(destinationBuffer); - auto blendshapeCoefficients = _headData->getSummedBlendshapeCoefficients(); + const auto& blendshapeCoefficients = _headData->getSummedBlendshapeCoefficients(); faceTrackerInfo->leftEyeBlink = _headData->_leftEyeBlink; faceTrackerInfo->rightEyeBlink = _headData->_rightEyeBlink; From a43447310df5174942bc864bd1ed8333568ae2a9 Mon Sep 17 00:00:00 2001 From: Andrew Meadows Date: Thu, 11 May 2017 15:22:22 -0700 Subject: [PATCH 3/5] remove physics dependency from avatars-render lib --- interface/src/avatar/AvatarManager.h | 2 +- .../src/avatar}/AvatarMotionState.cpp | 0 .../src/avatar}/AvatarMotionState.h | 2 +- libraries/avatars-renderer/CMakeLists.txt | 2 +- 4 files changed, 3 insertions(+), 3 deletions(-) rename {libraries/avatars-renderer/src/avatars-renderer => interface/src/avatar}/AvatarMotionState.cpp (100%) rename {libraries/avatars-renderer/src/avatars-renderer => interface/src/avatar}/AvatarMotionState.h (98%) diff --git a/interface/src/avatar/AvatarManager.h b/interface/src/avatar/AvatarManager.h index 9df1639853..f1e71f7367 100644 --- a/interface/src/avatar/AvatarManager.h +++ b/interface/src/avatar/AvatarManager.h @@ -21,9 +21,9 @@ #include #include #include -#include #include +#include "AvatarMotionState.h" #include "MyAvatar.h" class AudioInjector; diff --git a/libraries/avatars-renderer/src/avatars-renderer/AvatarMotionState.cpp b/interface/src/avatar/AvatarMotionState.cpp similarity index 100% rename from libraries/avatars-renderer/src/avatars-renderer/AvatarMotionState.cpp rename to interface/src/avatar/AvatarMotionState.cpp diff --git a/libraries/avatars-renderer/src/avatars-renderer/AvatarMotionState.h b/interface/src/avatar/AvatarMotionState.h similarity index 98% rename from libraries/avatars-renderer/src/avatars-renderer/AvatarMotionState.h rename to interface/src/avatar/AvatarMotionState.h index f8801ddf2f..90bd2a60ac 100644 --- a/libraries/avatars-renderer/src/avatars-renderer/AvatarMotionState.h +++ b/interface/src/avatar/AvatarMotionState.h @@ -14,10 +14,10 @@ #include +#include #include #include -#include "Avatar.h" class AvatarMotionState : public ObjectMotionState { public: diff --git a/libraries/avatars-renderer/CMakeLists.txt b/libraries/avatars-renderer/CMakeLists.txt index b13bc0a4a6..2ac5e6766d 100644 --- a/libraries/avatars-renderer/CMakeLists.txt +++ b/libraries/avatars-renderer/CMakeLists.txt @@ -1,6 +1,6 @@ set(TARGET_NAME avatars-renderer) AUTOSCRIBE_SHADER_LIB(gpu model render render-utils) setup_hifi_library(Widgets Network Script) -link_hifi_libraries(shared gpu model animation physics model-networking script-engine render image render-utils) +link_hifi_libraries(shared gpu model animation model-networking script-engine render image render-utils) target_bullet() From 20c27fc1338593498cb851498ddb38d004337589 Mon Sep 17 00:00:00 2001 From: Vladyslav Stelmakhovskyi Date: Sat, 13 May 2017 14:20:28 +0200 Subject: [PATCH 4/5] Fix crash on save Map data settings from scripts --- libraries/shared/src/SettingHelpers.cpp | 11 ++++++++++- 1 file changed, 10 insertions(+), 1 deletion(-) diff --git a/libraries/shared/src/SettingHelpers.cpp b/libraries/shared/src/SettingHelpers.cpp index 9e2d15fcd0..cbda4e4096 100644 --- a/libraries/shared/src/SettingHelpers.cpp +++ b/libraries/shared/src/SettingHelpers.cpp @@ -126,7 +126,16 @@ QJsonDocument variantMapToJsonDocument(const QSettings::SettingsMap& map) { } switch (variantType) { - case QVariant::Map: + case QVariant::Map: { + auto varmap = variant.toMap(); + for (auto mapit = varmap.cbegin(); mapit != varmap.cend(); ++mapit) { + auto& mapkey = mapit.key(); + auto& mapvariant = mapit.value(); + object.insert(key + "/" + mapkey, QJsonValue::fromVariant(mapvariant)); + } + break; + } + case QVariant::List: case QVariant::Hash: { qCritical() << "Unsupported variant type" << variant.typeName(); From cc10fc81b79cdd7139e683b3967dd844f30f2e7c Mon Sep 17 00:00:00 2001 From: Vladyslav Stelmakhovskyi Date: Sat, 13 May 2017 20:57:04 +0200 Subject: [PATCH 5/5] Fix crash in sit script --- interface/src/avatar/MyAvatar.cpp | 5 +++++ 1 file changed, 5 insertions(+) diff --git a/interface/src/avatar/MyAvatar.cpp b/interface/src/avatar/MyAvatar.cpp index fd547e39e0..3da9b8a214 100755 --- a/interface/src/avatar/MyAvatar.cpp +++ b/interface/src/avatar/MyAvatar.cpp @@ -291,6 +291,11 @@ QByteArray MyAvatar::toByteArrayStateful(AvatarDataDetail dataDetail) { } void MyAvatar::resetSensorsAndBody() { + if (QThread::currentThread() != thread()) { + QMetaObject::invokeMethod(this, "resetSensorsAndBody"); + return; + } + qApp->getActiveDisplayPlugin()->resetSensors(); reset(true, false, true); }