// // Head.cpp // interface/src/avatar // // Copyright 2013 High Fidelity, Inc. // // Distributed under the Apache License, Version 2.0. // See the accompanying file LICENSE or http://www.apache.org/licenses/LICENSE-2.0.html // #include #include #include #include #include "Application.h" #include "Avatar.h" #include "GeometryUtil.h" #include "Head.h" #include "Menu.h" #include "Util.h" #include "devices/DdeFaceTracker.h" #include "devices/Faceshift.h" #include "devices/OculusManager.h" using namespace std; Head::Head(Avatar* owningAvatar) : HeadData((AvatarData*)owningAvatar), _returnHeadToCenter(false), _position(0.0f, 0.0f, 0.0f), _rotation(0.0f, 0.0f, 0.0f), _leftEyePosition(0.0f, 0.0f, 0.0f), _rightEyePosition(0.0f, 0.0f, 0.0f), _eyePosition(0.0f, 0.0f, 0.0f), _scale(1.0f), _lastLoudness(0.0f), _longTermAverageLoudness(-1.0f), _audioAttack(0.0f), _audioJawOpen(0.0f), _mouth2(0.0f), _mouth3(0.0f), _mouth4(0.0f), _angularVelocity(0,0,0), _renderLookatVectors(false), _saccade(0.0f, 0.0f, 0.0f), _saccadeTarget(0.0f, 0.0f, 0.0f), _leftEyeBlinkVelocity(0.0f), _rightEyeBlinkVelocity(0.0f), _timeWithoutTalking(0.0f), _deltaPitch(0.0f), _deltaYaw(0.0f), _deltaRoll(0.0f), _deltaLeanSideways(0.0f), _deltaLeanForward(0.0f), _torsoTwist(0.0f), _isCameraMoving(false), _isLookingAtMe(false), _faceModel(this), _leftEyeLookAtID(DependencyManager::get()->allocateID()), _rightEyeLookAtID(DependencyManager::get()->allocateID()) { } void Head::init() { _faceModel.init(); } void Head::reset() { _baseYaw = _basePitch = _baseRoll = 0.0f; _leanForward = _leanSideways = 0.0f; _faceModel.reset(); } void Head::simulate(float deltaTime, bool isMine, bool billboard) { if (isMine) { MyAvatar* myAvatar = static_cast(_owningAvatar); // Only use face trackers when not playing back a recording. if (!myAvatar->isPlaying()) { FaceTracker* faceTracker = Application::getInstance()->getActiveFaceTracker(); DdeFaceTracker::SharedPointer dde = DependencyManager::get(); Faceshift::SharedPointer faceshift = DependencyManager::get(); if ((_isFaceshiftConnected = (faceshift == faceTracker))) { _blendshapeCoefficients = faceTracker->getBlendshapeCoefficients(); } else if (dde->isActive()) { faceTracker = dde.data(); _blendshapeCoefficients = faceTracker->getBlendshapeCoefficients(); } } // Twist the upper body to follow the rotation of the head, but only do this with my avatar, // since everyone else will see the full joint rotations for other people. const float BODY_FOLLOW_HEAD_YAW_RATE = 0.1f; const float BODY_FOLLOW_HEAD_FACTOR = 0.66f; float currentTwist = getTorsoTwist(); setTorsoTwist(currentTwist + (getFinalYaw() * BODY_FOLLOW_HEAD_FACTOR - currentTwist) * BODY_FOLLOW_HEAD_YAW_RATE); } // Update audio trailing average for rendering facial animations const float AUDIO_AVERAGING_SECS = 0.05f; const float AUDIO_LONG_TERM_AVERAGING_SECS = 30.0f; _averageLoudness = glm::mix(_averageLoudness, _audioLoudness, glm::min(deltaTime / AUDIO_AVERAGING_SECS, 1.0f)); if (_longTermAverageLoudness == -1.0) { _longTermAverageLoudness = _averageLoudness; } else { _longTermAverageLoudness = glm::mix(_longTermAverageLoudness, _averageLoudness, glm::min(deltaTime / AUDIO_LONG_TERM_AVERAGING_SECS, 1.0f)); } if (!(_isFaceshiftConnected || billboard)) { // Update eye saccades const float AVERAGE_MICROSACCADE_INTERVAL = 0.50f; const float AVERAGE_SACCADE_INTERVAL = 4.0f; const float MICROSACCADE_MAGNITUDE = 0.002f; const float SACCADE_MAGNITUDE = 0.04f; if (randFloat() < deltaTime / AVERAGE_MICROSACCADE_INTERVAL) { _saccadeTarget = MICROSACCADE_MAGNITUDE * randVector(); } else if (randFloat() < deltaTime / AVERAGE_SACCADE_INTERVAL) { _saccadeTarget = SACCADE_MAGNITUDE * randVector(); } _saccade += (_saccadeTarget - _saccade) * 0.50f; // Detect transition from talking to not; force blink after that and a delay bool forceBlink = false; const float TALKING_LOUDNESS = 100.0f; const float BLINK_AFTER_TALKING = 0.25f; if ((_averageLoudness - _longTermAverageLoudness) > TALKING_LOUDNESS) { _timeWithoutTalking = 0.0f; } else if (_timeWithoutTalking < BLINK_AFTER_TALKING && (_timeWithoutTalking += deltaTime) >= BLINK_AFTER_TALKING) { forceBlink = true; } // Update audio attack data for facial animation (eyebrows and mouth) const float AUDIO_ATTACK_AVERAGING_RATE = 0.9f; _audioAttack = AUDIO_ATTACK_AVERAGING_RATE * _audioAttack + (1.0f - AUDIO_ATTACK_AVERAGING_RATE) * fabs((_audioLoudness - _longTermAverageLoudness) - _lastLoudness); _lastLoudness = (_audioLoudness - _longTermAverageLoudness); const float BROW_LIFT_THRESHOLD = 100.0f; if (_audioAttack > BROW_LIFT_THRESHOLD) { _browAudioLift += sqrtf(_audioAttack) * 0.01f; } _browAudioLift = glm::clamp(_browAudioLift *= 0.7f, 0.0f, 1.0f); const float BLINK_SPEED = 10.0f; const float BLINK_SPEED_VARIABILITY = 1.0f; const float BLINK_START_VARIABILITY = 0.25f; const float FULLY_OPEN = 0.0f; const float FULLY_CLOSED = 1.0f; if (_leftEyeBlinkVelocity == 0.0f && _rightEyeBlinkVelocity == 0.0f) { // no blinking when brows are raised; blink less with increasing loudness const float BASE_BLINK_RATE = 15.0f / 60.0f; const float ROOT_LOUDNESS_TO_BLINK_INTERVAL = 0.25f; if (forceBlink || (_browAudioLift < EPSILON && shouldDo(glm::max(1.0f, sqrt(fabs(_averageLoudness - _longTermAverageLoudness)) * ROOT_LOUDNESS_TO_BLINK_INTERVAL) / BASE_BLINK_RATE, deltaTime))) { _leftEyeBlinkVelocity = BLINK_SPEED + randFloat() * BLINK_SPEED_VARIABILITY; _rightEyeBlinkVelocity = BLINK_SPEED + randFloat() * BLINK_SPEED_VARIABILITY; if (randFloat() < 0.5f) { _leftEyeBlink = BLINK_START_VARIABILITY; } else { _rightEyeBlink = BLINK_START_VARIABILITY; } } } else { _leftEyeBlink = glm::clamp(_leftEyeBlink + _leftEyeBlinkVelocity * deltaTime, FULLY_OPEN, FULLY_CLOSED); _rightEyeBlink = glm::clamp(_rightEyeBlink + _rightEyeBlinkVelocity * deltaTime, FULLY_OPEN, FULLY_CLOSED); if (_leftEyeBlink == FULLY_CLOSED) { _leftEyeBlinkVelocity = -BLINK_SPEED; } else if (_leftEyeBlink == FULLY_OPEN) { _leftEyeBlinkVelocity = 0.0f; } if (_rightEyeBlink == FULLY_CLOSED) { _rightEyeBlinkVelocity = -BLINK_SPEED; } else if (_rightEyeBlink == FULLY_OPEN) { _rightEyeBlinkVelocity = 0.0f; } } // use data to update fake Faceshift blendshape coefficients const float JAW_OPEN_SCALE = 0.015f; const float JAW_OPEN_RATE = 0.9f; const float JAW_CLOSE_RATE = 0.90f; float audioDelta = sqrtf(glm::max(_averageLoudness - _longTermAverageLoudness, 0.0f)) * JAW_OPEN_SCALE; if (audioDelta > _audioJawOpen) { _audioJawOpen += (audioDelta - _audioJawOpen) * JAW_OPEN_RATE; } else { _audioJawOpen *= JAW_CLOSE_RATE; } _audioJawOpen = glm::clamp(_audioJawOpen, 0.0f, 1.0f); // _mouth2 = "mmmm" shape // _mouth3 = "funnel" shape // _mouth4 = "smile" shape const float FUNNEL_PERIOD = 0.985f; const float FUNNEL_RANDOM_PERIOD = 0.01f; const float MMMM_POWER = 0.25f; const float MMMM_PERIOD = 0.91f; const float MMMM_RANDOM_PERIOD = 0.15f; const float SMILE_PERIOD = 0.925f; const float SMILE_RANDOM_PERIOD = 0.05f; _mouth3 = glm::mix(_audioJawOpen, _mouth3, FUNNEL_PERIOD + randFloat() * FUNNEL_RANDOM_PERIOD); _mouth2 = glm::mix(_audioJawOpen * MMMM_POWER, _mouth2, MMMM_PERIOD + randFloat() * MMMM_RANDOM_PERIOD); _mouth4 = glm::mix(_audioJawOpen, _mouth4, SMILE_PERIOD + randFloat() * SMILE_RANDOM_PERIOD); DependencyManager::get()->updateFakeCoefficients(_leftEyeBlink, _rightEyeBlink, _browAudioLift, _audioJawOpen, _mouth2, _mouth3, _mouth4, _blendshapeCoefficients); } else { _saccade = glm::vec3(); } if (!isMine) { _faceModel.setLODDistance(static_cast(_owningAvatar)->getLODDistance()); } _leftEyePosition = _rightEyePosition = getPosition(); if (!billboard) { _faceModel.simulate(deltaTime); if (!_faceModel.getEyePositions(_leftEyePosition, _rightEyePosition)) { static_cast(_owningAvatar)->getSkeletonModel().getEyePositions(_leftEyePosition, _rightEyePosition); } } _eyePosition = calculateAverageEyePosition(); } void Head::relaxLean(float deltaTime) { // restore rotation, lean to neutral positions const float LEAN_RELAXATION_PERIOD = 0.25f; // seconds float relaxationFactor = 1.0f - glm::min(deltaTime / LEAN_RELAXATION_PERIOD, 1.0f); _deltaYaw *= relaxationFactor; _deltaPitch *= relaxationFactor; _deltaRoll *= relaxationFactor; _leanSideways *= relaxationFactor; _leanForward *= relaxationFactor; _deltaLeanSideways *= relaxationFactor; _deltaLeanForward *= relaxationFactor; } void Head::render(float alpha, Model::RenderMode mode, bool postLighting) { if (postLighting) { if (_renderLookatVectors) { renderLookatVectors(_leftEyePosition, _rightEyePosition, getCorrectedLookAtPosition()); } } else { _faceModel.render(alpha, mode); } } void Head::setScale (float scale) { if (_scale == scale) { return; } _scale = scale; } glm::quat Head::getFinalOrientationInWorldFrame() const { return _owningAvatar->getOrientation() * getFinalOrientationInLocalFrame(); } glm::quat Head::getFinalOrientationInLocalFrame() const { return glm::quat(glm::radians(glm::vec3(getFinalPitch(), getFinalYaw(), getFinalRoll() ))); } glm::vec3 Head::getCorrectedLookAtPosition() { if (_isLookingAtMe) { return _correctedLookAtPosition; } else { return getLookAtPosition(); } } void Head::setCorrectedLookAtPosition(glm::vec3 correctedLookAtPosition) { _isLookingAtMe = true; _correctedLookAtPosition = correctedLookAtPosition; } glm::quat Head::getCameraOrientation() const { // NOTE: Head::getCameraOrientation() is not used for orienting the camera "view" while in Oculus mode, so // you may wonder why this code is here. This method will be called while in Oculus mode to determine how // to change the driving direction while in Oculus mode. It is used to support driving toward where you're // head is looking. Note that in oculus mode, your actual camera view and where your head is looking is not // always the same. if (OculusManager::isConnected()) { return getOrientation(); } Avatar* owningAvatar = static_cast(_owningAvatar); return owningAvatar->getWorldAlignedOrientation() * glm::quat(glm::radians(glm::vec3(_basePitch, 0.0f, 0.0f))); } glm::quat Head::getEyeRotation(const glm::vec3& eyePosition) const { glm::quat orientation = getOrientation(); return rotationBetween(orientation * IDENTITY_FRONT, _lookAtPosition + _saccade - eyePosition) * orientation; } glm::vec3 Head::getScalePivot() const { return _faceModel.isActive() ? _faceModel.getTranslation() : _position; } void Head::setFinalPitch(float finalPitch) { _deltaPitch = glm::clamp(finalPitch, MIN_HEAD_PITCH, MAX_HEAD_PITCH) - _basePitch; } void Head::setFinalYaw(float finalYaw) { _deltaYaw = glm::clamp(finalYaw, MIN_HEAD_YAW, MAX_HEAD_YAW) - _baseYaw; } void Head::setFinalRoll(float finalRoll) { _deltaRoll = glm::clamp(finalRoll, MIN_HEAD_ROLL, MAX_HEAD_ROLL) - _baseRoll; } float Head::getFinalYaw() const { return glm::clamp(_baseYaw + _deltaYaw, MIN_HEAD_YAW, MAX_HEAD_YAW); } float Head::getFinalPitch() const { return glm::clamp(_basePitch + _deltaPitch, MIN_HEAD_PITCH, MAX_HEAD_PITCH); } float Head::getFinalRoll() const { return glm::clamp(_baseRoll + _deltaRoll, MIN_HEAD_ROLL, MAX_HEAD_ROLL); } void Head::addLeanDeltas(float sideways, float forward) { _deltaLeanSideways += sideways; _deltaLeanForward += forward; } void Head::renderLookatVectors(glm::vec3 leftEyePosition, glm::vec3 rightEyePosition, glm::vec3 lookatPosition) { GeometryCache::SharedPointer geometryCache = DependencyManager::get(); DependencyManager::get()->begin(); glLineWidth(2.0); glm::vec4 startColor(0.2f, 0.2f, 0.2f, 1.0f); glm::vec4 endColor(1.0f, 1.0f, 1.0f, 0.0f); geometryCache->renderLine(leftEyePosition, lookatPosition, startColor, endColor, _leftEyeLookAtID); geometryCache->renderLine(rightEyePosition, lookatPosition, startColor, endColor, _rightEyeLookAtID); DependencyManager::get()->end(); }