From fc018257e1c5cec851da80c0b5366db03c6db04d Mon Sep 17 00:00:00 2001 From: luiscuenca Date: Wed, 9 Oct 2019 12:20:35 -0700 Subject: [PATCH 1/6] Select avatar to look using API. Blink when look at change --- interface/src/avatar/MyAvatar.cpp | 28 +++++++++++++------ interface/src/avatar/MyAvatar.h | 2 ++ .../src/avatars-renderer/Head.cpp | 7 +++-- libraries/avatars/src/HeadData.cpp | 16 +++++++++++ libraries/avatars/src/HeadData.h | 15 ++++++---- 5 files changed, 52 insertions(+), 16 deletions(-) diff --git a/interface/src/avatar/MyAvatar.cpp b/interface/src/avatar/MyAvatar.cpp index ea0efe29cb..f529f168f4 100644 --- a/interface/src/avatar/MyAvatar.cpp +++ b/interface/src/avatar/MyAvatar.cpp @@ -2188,15 +2188,20 @@ void MyAvatar::computeMyLookAtTarget(const AvatarHash& hash) { foreach (const AvatarSharedPointer& avatarData, hash) { std::shared_ptr avatar = std::static_pointer_cast(avatarData); if (!avatar->isMyAvatar() && avatar->isInitialized()) { - glm::vec3 otherForward = avatar->getHead()->getForwardDirection(); - glm::vec3 otherPosition = avatar->getHead()->getEyePosition(); - const float TIME_WITHOUT_TALKING_THRESHOLD = 1.0f; - bool otherIsTalking = avatar->getHead()->getTimeWithoutTalking() <= TIME_WITHOUT_TALKING_THRESHOLD; - bool lookingAtOtherAlready = _lookAtTargetAvatar.lock().get() == avatar.get(); - float cost = lookAtCostFunction(myForward, myPosition, otherForward, otherPosition, otherIsTalking, lookingAtOtherAlready); - if (cost < bestCost) { - bestCost = cost; + if (_forceTargetAvatarID.isNull() || avatar->getID() != _forceTargetAvatarID) { + glm::vec3 otherForward = avatar->getHead()->getForwardDirection(); + glm::vec3 otherPosition = avatar->getHead()->getEyePosition(); + const float TIME_WITHOUT_TALKING_THRESHOLD = 1.0f; + bool otherIsTalking = avatar->getHead()->getTimeWithoutTalking() <= TIME_WITHOUT_TALKING_THRESHOLD; + bool lookingAtOtherAlready = _lookAtTargetAvatar.lock().get() == avatar.get(); + float cost = lookAtCostFunction(myForward, myPosition, otherForward, otherPosition, otherIsTalking, lookingAtOtherAlready); + if (cost < bestCost) { + bestCost = cost; + bestAvatar = avatar; + } + } else { bestAvatar = avatar; + break; } } } @@ -6838,3 +6843,10 @@ void MyAvatar::resetPointAt() { } } +void MyAvatar::setLookAtAvatarID(const QUuid& avatarID) { + if (QThread::currentThread() != thread()) { + BLOCKING_INVOKE_METHOD(this, "setLookAtAvatarID", + Q_ARG(const QUuid&, avatarID)); + } + _forceTargetAvatarID = avatarID; +} \ No newline at end of file diff --git a/interface/src/avatar/MyAvatar.h b/interface/src/avatar/MyAvatar.h index a7ba639461..d93ee776be 100644 --- a/interface/src/avatar/MyAvatar.h +++ b/interface/src/avatar/MyAvatar.h @@ -971,6 +971,7 @@ public: * @param {Uuid} entityID - The entity that the hand touch effect will be enabled for. */ Q_INVOKABLE void enableHandTouchForID(const QUuid& entityID); + Q_INVOKABLE void setLookAtAvatarID(const QUuid& avatarID); bool useAdvancedMovementControls() const { return _useAdvancedMovementControls.get(); } void setUseAdvancedMovementControls(bool useAdvancedMovementControls) @@ -2656,6 +2657,7 @@ private: AvatarWeakPointer _lookAtTargetAvatar; glm::vec3 _targetAvatarPosition; + QUuid _forceTargetAvatarID; bool _shouldRender { true }; float _oculusYawOffset; diff --git a/libraries/avatars-renderer/src/avatars-renderer/Head.cpp b/libraries/avatars-renderer/src/avatars-renderer/Head.cpp index 63d8e2981c..a9dde201af 100644 --- a/libraries/avatars-renderer/src/avatars-renderer/Head.cpp +++ b/libraries/avatars-renderer/src/avatars-renderer/Head.cpp @@ -96,10 +96,13 @@ void Head::simulate(float deltaTime) { // no blinking when brows are raised; blink less with increasing loudness const float BASE_BLINK_RATE = 15.0f / 60.0f; const float ROOT_LOUDNESS_TO_BLINK_INTERVAL = 0.25f; - if (forceBlink || (_browAudioLift < EPSILON && shouldDo(glm::max(1.0f, sqrt(fabs(_averageLoudness - _longTermAverageLoudness)) * + if (_forceBlink || forceBlink || (_browAudioLift < EPSILON && shouldDo(glm::max(1.0f, sqrt(fabs(_averageLoudness - _longTermAverageLoudness)) * ROOT_LOUDNESS_TO_BLINK_INTERVAL) / BASE_BLINK_RATE, deltaTime))) { float randSpeedVariability = randFloat(); float eyeBlinkVelocity = BLINK_SPEED + randSpeedVariability * BLINK_SPEED_VARIABILITY; + if (_forceBlink) { + eyeBlinkVelocity = 0.5f * eyeBlinkVelocity; + } _leftEyeBlinkVelocity = eyeBlinkVelocity; _rightEyeBlinkVelocity = eyeBlinkVelocity; if (randFloat() < 0.5f) { @@ -114,7 +117,7 @@ void Head::simulate(float deltaTime) { if (_leftEyeBlink == FULLY_CLOSED) { _leftEyeBlinkVelocity = -BLINK_SPEED; - + updateEyeLookAt(); } else if (_leftEyeBlink == FULLY_OPEN) { _leftEyeBlinkVelocity = 0.0f; } diff --git a/libraries/avatars/src/HeadData.cpp b/libraries/avatars/src/HeadData.cpp index c86e534929..f8a09905ef 100644 --- a/libraries/avatars/src/HeadData.cpp +++ b/libraries/avatars/src/HeadData.cpp @@ -233,3 +233,19 @@ void HeadData::setHasProceduralEyeMovement(bool hasProceduralEyeMovement) { void HeadData::setFaceTrackerConnected(bool value) { _isFaceTrackerConnected = value; } + +void HeadData::setLookAtPosition(const glm::vec3& lookAtPosition) { + if (_requestLookAtPosition != lookAtPosition) { + _lookAtPositionChanged = usecTimestampNow(); + glm::vec3 oldAvatarLookAtVector = _requestLookAtPosition - _owningAvatar->getWorldPosition(); + glm::vec3 newAvatarLookAtVector = lookAtPosition - _owningAvatar->getWorldPosition(); + const float MIN_BLINK_ANGLE = 0.35f; // 20 degrees + _forceBlink = angleBetween(oldAvatarLookAtVector, newAvatarLookAtVector) > MIN_BLINK_ANGLE; + _lookAtUpdated = false; + } + _requestLookAtPosition = lookAtPosition; + if (_lookAtUpdated) { + _forceBlink = false; + _lookAtPosition = lookAtPosition; + } +} \ No newline at end of file diff --git a/libraries/avatars/src/HeadData.h b/libraries/avatars/src/HeadData.h index dc5aaf2595..a8005b5659 100644 --- a/libraries/avatars/src/HeadData.h +++ b/libraries/avatars/src/HeadData.h @@ -64,12 +64,7 @@ public: void setBlendshapeCoefficients(const QVector& blendshapeCoefficients) { _blendshapeCoefficients = blendshapeCoefficients; } const glm::vec3& getLookAtPosition() const { return _lookAtPosition; } - void setLookAtPosition(const glm::vec3& lookAtPosition) { - if (_lookAtPosition != lookAtPosition) { - _lookAtPositionChanged = usecTimestampNow(); - } - _lookAtPosition = lookAtPosition; - } + void setLookAtPosition(const glm::vec3& lookAtPosition); bool lookAtPositionChangedSince(quint64 time) { return _lookAtPositionChanged >= time; } bool getHasProceduralEyeFaceMovement() const; @@ -84,6 +79,11 @@ public: void setFaceTrackerConnected(bool value); bool getFaceTrackerConnected() const { return _isFaceTrackerConnected; } + void updateEyeLookAt() { + _lookAtPosition = _requestLookAtPosition; + _lookAtUpdated = true; + } + friend class AvatarData; QJsonObject toJson() const; @@ -95,6 +95,7 @@ protected: float _basePitch; float _baseRoll; + glm::vec3 _requestLookAtPosition; glm::vec3 _lookAtPosition; quint64 _lookAtPositionChanged { 0 }; @@ -115,6 +116,8 @@ protected: QVector _summedBlendshapeCoefficients; QMap _blendshapeLookupMap; AvatarData* _owningAvatar; + bool _forceBlink { false }; + bool _lookAtUpdated { false }; private: // privatize copy ctor and assignment operator so copies of this object cannot be made From 505813b3c356bfee7835a96c89f8e5d02a3db9bf Mon Sep 17 00:00:00 2001 From: luiscuenca Date: Mon, 14 Oct 2019 10:44:55 -0700 Subject: [PATCH 2/6] Fix blink async and other is talking bug --- interface/src/avatar/MyAvatar.cpp | 15 +++++---------- interface/src/avatar/MyAvatar.h | 1 - .../src/avatars-renderer/Head.cpp | 8 +++++--- libraries/avatars/src/AvatarData.cpp | 9 +++++++++ libraries/avatars/src/AvatarData.h | 1 + libraries/avatars/src/HeadData.cpp | 16 ++++++++++------ libraries/avatars/src/HeadData.h | 7 ++----- 7 files changed, 32 insertions(+), 25 deletions(-) diff --git a/interface/src/avatar/MyAvatar.cpp b/interface/src/avatar/MyAvatar.cpp index f529f168f4..0abbd4e324 100644 --- a/interface/src/avatar/MyAvatar.cpp +++ b/interface/src/avatar/MyAvatar.cpp @@ -2150,7 +2150,7 @@ static float lookAtCostFunction(const glm::vec3& myForward, const glm::vec3& myP const float DISTANCE_FACTOR = 3.14f; const float MY_ANGLE_FACTOR = 1.0f; const float OTHER_ANGLE_FACTOR = 1.0f; - const float OTHER_IS_TALKING_TERM = otherIsTalking ? 1.0f : 0.0f; + const float OTHER_IS_TALKING_TERM = otherIsTalking ? -1.0f : 0.0f; const float LOOKING_AT_OTHER_ALREADY_TERM = lookingAtOtherAlready ? -0.2f : 0.0f; const float GREATEST_LOOKING_AT_DISTANCE = 10.0f; // meters @@ -2176,6 +2176,9 @@ static float lookAtCostFunction(const glm::vec3& myForward, const glm::vec3& myP void MyAvatar::computeMyLookAtTarget(const AvatarHash& hash) { glm::vec3 myForward = _lookAtYaw * IDENTITY_FORWARD; + if (_skeletonModel->isLoaded()) { + myForward = getHeadJointFrontVector(); + } glm::vec3 myPosition = getHead()->getEyePosition(); CameraMode mode = qApp->getCamera().getMode(); if (mode == CAMERA_MODE_FIRST_PERSON) { @@ -2189,7 +2192,7 @@ void MyAvatar::computeMyLookAtTarget(const AvatarHash& hash) { std::shared_ptr avatar = std::static_pointer_cast(avatarData); if (!avatar->isMyAvatar() && avatar->isInitialized()) { if (_forceTargetAvatarID.isNull() || avatar->getID() != _forceTargetAvatarID) { - glm::vec3 otherForward = avatar->getHead()->getForwardDirection(); + glm::vec3 otherForward = avatar->getHeadJointFrontVector(); glm::vec3 otherPosition = avatar->getHead()->getEyePosition(); const float TIME_WITHOUT_TALKING_THRESHOLD = 1.0f; bool otherIsTalking = avatar->getHead()->getTimeWithoutTalking() <= TIME_WITHOUT_TALKING_THRESHOLD; @@ -6842,11 +6845,3 @@ void MyAvatar::resetPointAt() { POINT_BLEND_LINEAR_ALPHA_NAME, POINT_ALPHA_BLENDING); } } - -void MyAvatar::setLookAtAvatarID(const QUuid& avatarID) { - if (QThread::currentThread() != thread()) { - BLOCKING_INVOKE_METHOD(this, "setLookAtAvatarID", - Q_ARG(const QUuid&, avatarID)); - } - _forceTargetAvatarID = avatarID; -} \ No newline at end of file diff --git a/interface/src/avatar/MyAvatar.h b/interface/src/avatar/MyAvatar.h index d93ee776be..3f91222392 100644 --- a/interface/src/avatar/MyAvatar.h +++ b/interface/src/avatar/MyAvatar.h @@ -971,7 +971,6 @@ public: * @param {Uuid} entityID - The entity that the hand touch effect will be enabled for. */ Q_INVOKABLE void enableHandTouchForID(const QUuid& entityID); - Q_INVOKABLE void setLookAtAvatarID(const QUuid& avatarID); bool useAdvancedMovementControls() const { return _useAdvancedMovementControls.get(); } void setUseAdvancedMovementControls(bool useAdvancedMovementControls) diff --git a/libraries/avatars-renderer/src/avatars-renderer/Head.cpp b/libraries/avatars-renderer/src/avatars-renderer/Head.cpp index a9dde201af..8184804e1f 100644 --- a/libraries/avatars-renderer/src/avatars-renderer/Head.cpp +++ b/libraries/avatars-renderer/src/avatars-renderer/Head.cpp @@ -96,12 +96,15 @@ void Head::simulate(float deltaTime) { // no blinking when brows are raised; blink less with increasing loudness const float BASE_BLINK_RATE = 15.0f / 60.0f; const float ROOT_LOUDNESS_TO_BLINK_INTERVAL = 0.25f; - if (_forceBlink || forceBlink || (_browAudioLift < EPSILON && shouldDo(glm::max(1.0f, sqrt(fabs(_averageLoudness - _longTermAverageLoudness)) * + if (_blinkToRetarget || forceBlink || + (_browAudioLift < EPSILON && shouldDo(glm::max(1.0f, sqrt(fabs(_averageLoudness - _longTermAverageLoudness)) * ROOT_LOUDNESS_TO_BLINK_INTERVAL) / BASE_BLINK_RATE, deltaTime))) { float randSpeedVariability = randFloat(); float eyeBlinkVelocity = BLINK_SPEED + randSpeedVariability * BLINK_SPEED_VARIABILITY; - if (_forceBlink) { + if (_blinkToRetarget) { + // Slow down by half the blink if reseting eye target eyeBlinkVelocity = 0.5f * eyeBlinkVelocity; + _blinkToRetarget = false; } _leftEyeBlinkVelocity = eyeBlinkVelocity; _rightEyeBlinkVelocity = eyeBlinkVelocity; @@ -123,7 +126,6 @@ void Head::simulate(float deltaTime) { } if (_rightEyeBlink == FULLY_CLOSED) { _rightEyeBlinkVelocity = -BLINK_SPEED; - } else if (_rightEyeBlink == FULLY_OPEN) { _rightEyeBlinkVelocity = 0.0f; } diff --git a/libraries/avatars/src/AvatarData.cpp b/libraries/avatars/src/AvatarData.cpp index a91154ff15..69694a00c3 100755 --- a/libraries/avatars/src/AvatarData.cpp +++ b/libraries/avatars/src/AvatarData.cpp @@ -3214,3 +3214,12 @@ void AvatarData::clearAvatarGrabData(const QUuid& grabID) { } }); } + +glm::vec3 AvatarData::getHeadJointFrontVector() const { + int headJointIndex = getJointIndex("Head"); + glm::quat headJointRotation = Quaternions::Y_180 * getAbsoluteJointRotationInObjectFrame(headJointIndex);// getAbsoluteJointRotationInRigFrame(headJointIndex, headJointRotation); + headJointRotation = getWorldOrientation() * headJointRotation; + float headYaw = safeEulerAngles(headJointRotation).y; + glm::quat headYawRotation = glm::angleAxis(headYaw, Vectors::UP); + return headYawRotation * IDENTITY_FORWARD; +} diff --git a/libraries/avatars/src/AvatarData.h b/libraries/avatars/src/AvatarData.h index 59a2e2a53e..bfe775e972 100755 --- a/libraries/avatars/src/AvatarData.h +++ b/libraries/avatars/src/AvatarData.h @@ -1484,6 +1484,7 @@ public: std::vector getSkeletonData() const; void sendSkeletonData() const; QVector getJointData() const; + glm::vec3 getHeadJointFrontVector() const; signals: diff --git a/libraries/avatars/src/HeadData.cpp b/libraries/avatars/src/HeadData.cpp index f8a09905ef..7d05a50143 100644 --- a/libraries/avatars/src/HeadData.cpp +++ b/libraries/avatars/src/HeadData.cpp @@ -240,12 +240,16 @@ void HeadData::setLookAtPosition(const glm::vec3& lookAtPosition) { glm::vec3 oldAvatarLookAtVector = _requestLookAtPosition - _owningAvatar->getWorldPosition(); glm::vec3 newAvatarLookAtVector = lookAtPosition - _owningAvatar->getWorldPosition(); const float MIN_BLINK_ANGLE = 0.35f; // 20 degrees - _forceBlink = angleBetween(oldAvatarLookAtVector, newAvatarLookAtVector) > MIN_BLINK_ANGLE; + _blinkToRetarget = angleBetween(oldAvatarLookAtVector, newAvatarLookAtVector) > MIN_BLINK_ANGLE; _lookAtUpdated = false; + } + if (_lookAtUpdated) { + _lookAtPosition = lookAtPosition; } _requestLookAtPosition = lookAtPosition; - if (_lookAtUpdated) { - _forceBlink = false; - _lookAtPosition = lookAtPosition; - } -} \ No newline at end of file +} + +void HeadData::updateEyeLookAt() { + _lookAtPosition = _requestLookAtPosition; + _lookAtUpdated = true; +} diff --git a/libraries/avatars/src/HeadData.h b/libraries/avatars/src/HeadData.h index a8005b5659..7b73f57113 100644 --- a/libraries/avatars/src/HeadData.h +++ b/libraries/avatars/src/HeadData.h @@ -79,10 +79,7 @@ public: void setFaceTrackerConnected(bool value); bool getFaceTrackerConnected() const { return _isFaceTrackerConnected; } - void updateEyeLookAt() { - _lookAtPosition = _requestLookAtPosition; - _lookAtUpdated = true; - } + void updateEyeLookAt(); friend class AvatarData; @@ -116,7 +113,7 @@ protected: QVector _summedBlendshapeCoefficients; QMap _blendshapeLookupMap; AvatarData* _owningAvatar; - bool _forceBlink { false }; + bool _blinkToRetarget { false }; bool _lookAtUpdated { false }; private: From 9f0e82e1e65deeaf372645bac81443deb402f70a Mon Sep 17 00:00:00 2001 From: luiscuenca Date: Wed, 16 Oct 2019 15:29:28 -0700 Subject: [PATCH 3/6] Improve eye's look at and blinking. Fix look at update --- interface/src/avatar/MyAvatar.cpp | 5 ++-- .../src/avatars-renderer/Head.cpp | 6 ++-- libraries/avatars/src/AvatarData.h | 1 + libraries/avatars/src/HeadData.cpp | 30 ++++++++++++------- libraries/avatars/src/HeadData.h | 4 +-- 5 files changed, 28 insertions(+), 18 deletions(-) diff --git a/interface/src/avatar/MyAvatar.cpp b/interface/src/avatar/MyAvatar.cpp index 0abbd4e324..2bc0ae32d4 100644 --- a/interface/src/avatar/MyAvatar.cpp +++ b/interface/src/avatar/MyAvatar.cpp @@ -6709,8 +6709,9 @@ void MyAvatar::updateLookAtPosition(FaceTracker* faceTracker, Camera& myCamera) if (headPose.isValid()) { lookAtSpot = transformPoint(headPose.getMatrix(), glm::vec3(0.0f, 0.0f, TREE_SCALE)); } else { - lookAtSpot = myHead->getEyePosition() + - (getHead()->getFinalOrientationInWorldFrame() * glm::vec3(0.0f, 0.0f, -TREE_SCALE)); + lookAtSpot = _shouldTurnToFaceCamera ? + myHead->getLookAtPosition() : + myHead->getEyePosition() + getHeadJointFrontVector() * (float)TREE_SCALE; } } diff --git a/libraries/avatars-renderer/src/avatars-renderer/Head.cpp b/libraries/avatars-renderer/src/avatars-renderer/Head.cpp index 8184804e1f..e2f5ef0c60 100644 --- a/libraries/avatars-renderer/src/avatars-renderer/Head.cpp +++ b/libraries/avatars-renderer/src/avatars-renderer/Head.cpp @@ -96,15 +96,15 @@ void Head::simulate(float deltaTime) { // no blinking when brows are raised; blink less with increasing loudness const float BASE_BLINK_RATE = 15.0f / 60.0f; const float ROOT_LOUDNESS_TO_BLINK_INTERVAL = 0.25f; - if (_blinkToRetarget || forceBlink || + if (_forceBlinkToRetarget || forceBlink || (_browAudioLift < EPSILON && shouldDo(glm::max(1.0f, sqrt(fabs(_averageLoudness - _longTermAverageLoudness)) * ROOT_LOUDNESS_TO_BLINK_INTERVAL) / BASE_BLINK_RATE, deltaTime))) { float randSpeedVariability = randFloat(); float eyeBlinkVelocity = BLINK_SPEED + randSpeedVariability * BLINK_SPEED_VARIABILITY; - if (_blinkToRetarget) { + if (_forceBlinkToRetarget) { // Slow down by half the blink if reseting eye target eyeBlinkVelocity = 0.5f * eyeBlinkVelocity; - _blinkToRetarget = false; + _forceBlinkToRetarget = false; } _leftEyeBlinkVelocity = eyeBlinkVelocity; _rightEyeBlinkVelocity = eyeBlinkVelocity; diff --git a/libraries/avatars/src/AvatarData.h b/libraries/avatars/src/AvatarData.h index bfe775e972..61729f8db3 100755 --- a/libraries/avatars/src/AvatarData.h +++ b/libraries/avatars/src/AvatarData.h @@ -1480,6 +1480,7 @@ public: void setIsNewAvatar(bool isNewAvatar) { _isNewAvatar = isNewAvatar; } bool getIsNewAvatar() { return _isNewAvatar; } void setIsClientAvatar(bool isClientAvatar) { _isClientAvatar = isClientAvatar; } + bool getIsClientAvatar() const { return _isClientAvatar; } void setSkeletonData(const std::vector& skeletonData); std::vector getSkeletonData() const; void sendSkeletonData() const; diff --git a/libraries/avatars/src/HeadData.cpp b/libraries/avatars/src/HeadData.cpp index 7d05a50143..d9998883d7 100644 --- a/libraries/avatars/src/HeadData.cpp +++ b/libraries/avatars/src/HeadData.cpp @@ -235,21 +235,29 @@ void HeadData::setFaceTrackerConnected(bool value) { } void HeadData::setLookAtPosition(const glm::vec3& lookAtPosition) { - if (_requestLookAtPosition != lookAtPosition) { - _lookAtPositionChanged = usecTimestampNow(); - glm::vec3 oldAvatarLookAtVector = _requestLookAtPosition - _owningAvatar->getWorldPosition(); - glm::vec3 newAvatarLookAtVector = lookAtPosition - _owningAvatar->getWorldPosition(); - const float MIN_BLINK_ANGLE = 0.35f; // 20 degrees - _blinkToRetarget = angleBetween(oldAvatarLookAtVector, newAvatarLookAtVector) > MIN_BLINK_ANGLE; - _lookAtUpdated = false; - } - if (_lookAtUpdated) { + if (_owningAvatar->getIsClientAvatar() || _owningAvatar->isMyAvatar()) { + if (_isEyeLookAtUpdated && _requestLookAtPosition != lookAtPosition) { + _lookAtPositionChanged = usecTimestampNow(); + glm::vec3 oldAvatarLookAtVector = _requestLookAtPosition - _owningAvatar->getWorldPosition(); + glm::vec3 newAvatarLookAtVector = lookAtPosition - _owningAvatar->getWorldPosition(); + const float MIN_BLINK_ANGLE = 0.35f; // 20 degrees + _forceBlinkToRetarget = angleBetween(oldAvatarLookAtVector, newAvatarLookAtVector) > MIN_BLINK_ANGLE; + if (_forceBlinkToRetarget) { + _isEyeLookAtUpdated = false; + } else { + _lookAtPosition = lookAtPosition; + } + } + _requestLookAtPosition = lookAtPosition; + } else { + if (_lookAtPosition != lookAtPosition) { + _lookAtPositionChanged = usecTimestampNow(); + } _lookAtPosition = lookAtPosition; } - _requestLookAtPosition = lookAtPosition; } void HeadData::updateEyeLookAt() { _lookAtPosition = _requestLookAtPosition; - _lookAtUpdated = true; + _isEyeLookAtUpdated = true; } diff --git a/libraries/avatars/src/HeadData.h b/libraries/avatars/src/HeadData.h index 7b73f57113..190b179f25 100644 --- a/libraries/avatars/src/HeadData.h +++ b/libraries/avatars/src/HeadData.h @@ -113,8 +113,8 @@ protected: QVector _summedBlendshapeCoefficients; QMap _blendshapeLookupMap; AvatarData* _owningAvatar; - bool _blinkToRetarget { false }; - bool _lookAtUpdated { false }; + bool _forceBlinkToRetarget { false }; + bool _isEyeLookAtUpdated { false }; private: // privatize copy ctor and assignment operator so copies of this object cannot be made From a4fcb2c39e1fcc93df6c0d3883929580704b1a10 Mon Sep 17 00:00:00 2001 From: luiscuenca Date: Mon, 21 Oct 2019 14:31:15 -0700 Subject: [PATCH 4/6] Add eyesLookAtTarget set/get API methods --- interface/src/Application.cpp | 6 +++--- interface/src/Application.h | 2 +- interface/src/avatar/MyAvatar.cpp | 26 +++++++++++++++++++++++--- interface/src/avatar/MyAvatar.h | 23 +++++++++++++++++++++-- 4 files changed, 48 insertions(+), 9 deletions(-) diff --git a/interface/src/Application.cpp b/interface/src/Application.cpp index 4766353eba..7c0e822bed 100644 --- a/interface/src/Application.cpp +++ b/interface/src/Application.cpp @@ -5814,14 +5814,14 @@ void Application::pushPostUpdateLambda(void* key, const std::function& f // to everyone. // The principal result is to call updateLookAtTargetAvatar() and then setLookAtPosition(). // Note that it is called BEFORE we update position or joints based on sensors, etc. -void Application::updateMyAvatarLookAtPosition() { +void Application::updateMyAvatarLookAtPosition(float deltaTime) { PerformanceTimer perfTimer("lookAt"); bool showWarnings = Menu::getInstance()->isOptionChecked(MenuOption::PipelineWarnings); PerformanceWarning warn(showWarnings, "Application::updateMyAvatarLookAtPosition()"); auto myAvatar = getMyAvatar(); FaceTracker* faceTracker = getActiveFaceTracker(); - myAvatar->updateLookAtPosition(faceTracker, _myCamera); + myAvatar->updateEyesLookAtPosition(faceTracker, _myCamera, deltaTime); } void Application::updateThreads(float deltaTime) { @@ -6605,7 +6605,7 @@ void Application::update(float deltaTime) { { PROFILE_RANGE(simulation, "MyAvatar"); PerformanceTimer perfTimer("MyAvatar"); - qApp->updateMyAvatarLookAtPosition(); + qApp->updateMyAvatarLookAtPosition(deltaTime); avatarManager->updateMyAvatar(deltaTime); } } diff --git a/interface/src/Application.h b/interface/src/Application.h index af2348d1e9..2bb6c5ddb4 100644 --- a/interface/src/Application.h +++ b/interface/src/Application.h @@ -298,7 +298,7 @@ public: virtual void pushPostUpdateLambda(void* key, const std::function& func) override; - void updateMyAvatarLookAtPosition(); + void updateMyAvatarLookAtPosition(float deltaTime); float getGameLoopRate() const { return _gameLoopCounter.rate(); } diff --git a/interface/src/avatar/MyAvatar.cpp b/interface/src/avatar/MyAvatar.cpp index 2bc0ae32d4..e152902b47 100644 --- a/interface/src/avatar/MyAvatar.cpp +++ b/interface/src/avatar/MyAvatar.cpp @@ -2284,7 +2284,9 @@ void MyAvatar::updateLookAtTargetAvatar() { AvatarHash hash = DependencyManager::get()->getHashCopy(); // determine what the best look at target for my avatar should be. - computeMyLookAtTarget(hash); + if (!_scriptControlsEyesLookAt) { + computeMyLookAtTarget(hash); + } // snap look at position for avatars that are looking at me. snapOtherAvatarLookAtTargetsToMe(hash); @@ -6617,7 +6619,7 @@ bool MyAvatar::getIsJointOverridden(int jointIndex) const { return _skeletonModel->getIsJointOverridden(jointIndex); } -void MyAvatar::updateLookAtPosition(FaceTracker* faceTracker, Camera& myCamera) { +void MyAvatar::updateEyesLookAtPosition(FaceTracker* faceTracker, Camera& myCamera, float deltaTime) { updateLookAtTargetAvatar(); @@ -6647,6 +6649,13 @@ void MyAvatar::updateLookAtPosition(FaceTracker* faceTracker, Camera& myCamera) } else { lookAtSpot = myHead->getEyePosition() + glm::normalize(leftVec) * 1000.0f; } + } else if (_scriptControlsEyesLookAt) { + if (_scriptEyesControlTimer < MAX_LOOK_AT_TIME_SCRIPT_CONTROL) { + _scriptEyesControlTimer += deltaTime; + lookAtSpot = _eyesLookAtTarget; + } else { + _scriptControlsEyesLookAt = false; + } } else { controller::Pose leftEyePose = getControllerPoseInAvatarFrame(controller::Action::LEFT_EYE); controller::Pose rightEyePose = getControllerPoseInAvatarFrame(controller::Action::RIGHT_EYE); @@ -6731,7 +6740,7 @@ void MyAvatar::updateLookAtPosition(FaceTracker* faceTracker, Camera& myCamera) } } } - + _eyesLookAtTarget = lookAtSpot; getHead()->setLookAtPosition(lookAtSpot); } @@ -6814,6 +6823,17 @@ void MyAvatar::setHeadLookAt(const glm::vec3& lookAtTarget) { _lookAtScriptTarget = lookAtTarget; } +void MyAvatar::setEyesLookAt(const glm::vec3& lookAtTarget) { + if (QThread::currentThread() != thread()) { + BLOCKING_INVOKE_METHOD(this, "setEyesLookAt", + Q_ARG(const glm::vec3&, lookAtTarget)); + return; + } + _eyesLookAtTarget = lookAtTarget; + _scriptEyesControlTimer = 0.0f; + _scriptControlsEyesLookAt = true; +} + glm::vec3 MyAvatar::getLookAtPivotPoint() { glm::vec3 avatarUp = getWorldOrientation() * Vectors::UP; glm::vec3 yAxisEyePosition = getWorldPosition() + avatarUp * glm::dot(avatarUp, _skeletonModel->getDefaultEyeModelPosition()); diff --git a/interface/src/avatar/MyAvatar.h b/interface/src/avatar/MyAvatar.h index 3f91222392..90c4157c3c 100644 --- a/interface/src/avatar/MyAvatar.h +++ b/interface/src/avatar/MyAvatar.h @@ -1762,10 +1762,26 @@ public: /**jsdoc * Returns the current head look at target point in world coordinates. * @function MyAvatar.getHeadLookAt - * @returns {Vec3} Default position between your avatar's eyes in world coordinates. + * @returns {Vec3} The head's look at target in world coordinates. */ Q_INVOKABLE glm::vec3 getHeadLookAt() { return _lookAtCameraTarget; } + /**jsdoc + * Force the avatar's eyes to look to the specified location. + * Once this method is called, API calls will have full control of the eyes for a limited time. + * If this method is not called for two seconds, the engine will regain control of the eyes. + * @function MyAvatar.setEyesLookAt + * @param {Vec3} lookAtTarget - The target point in world coordinates. + */ + Q_INVOKABLE void setEyesLookAt(const glm::vec3& lookAtTarget); + + /**jsdoc + * Returns the current eyes look at target point in world coordinates. + * @function MyAvatar.getEyesLookAt + * @returns {Vec3} The eyes's look at target in world coordinates. + */ + Q_INVOKABLE glm::vec3 getEyesLookAt() { return _eyesLookAtTarget; } + /**jsdoc * Aims the pointing directional blending towards the provided target point. * The "point" reaction should be triggered before using this method. @@ -1903,7 +1919,7 @@ public: bool getFlowActive() const; bool getNetworkGraphActive() const; - void updateLookAtPosition(FaceTracker* faceTracker, Camera& myCamera); + void updateEyesLookAtPosition(FaceTracker* faceTracker, Camera& myCamera, float deltaTime); // sets the reaction enabled and triggered parameters of the passed in params // also clears internal reaction triggers @@ -2662,6 +2678,9 @@ private: eyeContactTarget _eyeContactTarget; float _eyeContactTargetTimer { 0.0f }; + glm::vec3 _eyesLookAtTarget; + bool _scriptControlsEyesLookAt{ false }; + float _scriptEyesControlTimer{ 0.0f }; glm::vec3 _trackedHeadPosition; From bb0849d3a0295735159ff2b1cb9b0fdd7712bda1 Mon Sep 17 00:00:00 2001 From: luiscuenca Date: Wed, 23 Oct 2019 14:23:29 -0700 Subject: [PATCH 5/6] Move changes to Head and remove leftover code --- interface/src/avatar/MyAvatar.cpp | 27 ++++++++----------- interface/src/avatar/MyAvatar.h | 5 ++-- .../src/avatars-renderer/Head.cpp | 21 +++++++++++++++ .../src/avatars-renderer/Head.h | 7 +++++ libraries/avatars/src/AvatarData.h | 1 - libraries/avatars/src/HeadData.cpp | 27 ------------------- libraries/avatars/src/HeadData.h | 12 ++++----- 7 files changed, 47 insertions(+), 53 deletions(-) diff --git a/interface/src/avatar/MyAvatar.cpp b/interface/src/avatar/MyAvatar.cpp index e152902b47..9c9505323f 100644 --- a/interface/src/avatar/MyAvatar.cpp +++ b/interface/src/avatar/MyAvatar.cpp @@ -2191,20 +2191,15 @@ void MyAvatar::computeMyLookAtTarget(const AvatarHash& hash) { foreach (const AvatarSharedPointer& avatarData, hash) { std::shared_ptr avatar = std::static_pointer_cast(avatarData); if (!avatar->isMyAvatar() && avatar->isInitialized()) { - if (_forceTargetAvatarID.isNull() || avatar->getID() != _forceTargetAvatarID) { - glm::vec3 otherForward = avatar->getHeadJointFrontVector(); - glm::vec3 otherPosition = avatar->getHead()->getEyePosition(); - const float TIME_WITHOUT_TALKING_THRESHOLD = 1.0f; - bool otherIsTalking = avatar->getHead()->getTimeWithoutTalking() <= TIME_WITHOUT_TALKING_THRESHOLD; - bool lookingAtOtherAlready = _lookAtTargetAvatar.lock().get() == avatar.get(); - float cost = lookAtCostFunction(myForward, myPosition, otherForward, otherPosition, otherIsTalking, lookingAtOtherAlready); - if (cost < bestCost) { - bestCost = cost; - bestAvatar = avatar; - } - } else { + glm::vec3 otherForward = avatar->getHeadJointFrontVector(); + glm::vec3 otherPosition = avatar->getHead()->getEyePosition(); + const float TIME_WITHOUT_TALKING_THRESHOLD = 1.0f; + bool otherIsTalking = avatar->getHead()->getTimeWithoutTalking() <= TIME_WITHOUT_TALKING_THRESHOLD; + bool lookingAtOtherAlready = _lookAtTargetAvatar.lock().get() == avatar.get(); + float cost = lookAtCostFunction(myForward, myPosition, otherForward, otherPosition, otherIsTalking, lookingAtOtherAlready); + if (cost < bestCost) { + bestCost = cost; bestAvatar = avatar; - break; } } } @@ -6652,7 +6647,7 @@ void MyAvatar::updateEyesLookAtPosition(FaceTracker* faceTracker, Camera& myCame } else if (_scriptControlsEyesLookAt) { if (_scriptEyesControlTimer < MAX_LOOK_AT_TIME_SCRIPT_CONTROL) { _scriptEyesControlTimer += deltaTime; - lookAtSpot = _eyesLookAtTarget; + lookAtSpot = _eyesLookAtTarget.get(); } else { _scriptControlsEyesLookAt = false; } @@ -6740,7 +6735,7 @@ void MyAvatar::updateEyesLookAtPosition(FaceTracker* faceTracker, Camera& myCame } } } - _eyesLookAtTarget = lookAtSpot; + _eyesLookAtTarget.set(lookAtSpot); getHead()->setLookAtPosition(lookAtSpot); } @@ -6829,7 +6824,7 @@ void MyAvatar::setEyesLookAt(const glm::vec3& lookAtTarget) { Q_ARG(const glm::vec3&, lookAtTarget)); return; } - _eyesLookAtTarget = lookAtTarget; + _eyesLookAtTarget.set(lookAtTarget); _scriptEyesControlTimer = 0.0f; _scriptControlsEyesLookAt = true; } diff --git a/interface/src/avatar/MyAvatar.h b/interface/src/avatar/MyAvatar.h index 90c4157c3c..c0fa15b624 100644 --- a/interface/src/avatar/MyAvatar.h +++ b/interface/src/avatar/MyAvatar.h @@ -1780,7 +1780,7 @@ public: * @function MyAvatar.getEyesLookAt * @returns {Vec3} The eyes's look at target in world coordinates. */ - Q_INVOKABLE glm::vec3 getEyesLookAt() { return _eyesLookAtTarget; } + Q_INVOKABLE glm::vec3 getEyesLookAt() { return _eyesLookAtTarget.get(); } /**jsdoc * Aims the pointing directional blending towards the provided target point. @@ -2672,13 +2672,12 @@ private: AvatarWeakPointer _lookAtTargetAvatar; glm::vec3 _targetAvatarPosition; - QUuid _forceTargetAvatarID; bool _shouldRender { true }; float _oculusYawOffset; eyeContactTarget _eyeContactTarget; float _eyeContactTargetTimer { 0.0f }; - glm::vec3 _eyesLookAtTarget; + ThreadSafeValueCache _eyesLookAtTarget { glm::vec3() }; bool _scriptControlsEyesLookAt{ false }; float _scriptEyesControlTimer{ 0.0f }; diff --git a/libraries/avatars-renderer/src/avatars-renderer/Head.cpp b/libraries/avatars-renderer/src/avatars-renderer/Head.cpp index e2f5ef0c60..aa6dc779d5 100644 --- a/libraries/avatars-renderer/src/avatars-renderer/Head.cpp +++ b/libraries/avatars-renderer/src/avatars-renderer/Head.cpp @@ -355,3 +355,24 @@ float Head::getFinalPitch() const { float Head::getFinalRoll() const { return glm::clamp(_baseRoll + _deltaRoll, MIN_HEAD_ROLL, MAX_HEAD_ROLL); } + +void Head::setLookAtPosition(const glm::vec3& lookAtPosition) { + if (_isEyeLookAtUpdated && _requestLookAtPosition != lookAtPosition) { + _lookAtPositionChanged = usecTimestampNow(); + glm::vec3 oldAvatarLookAtVector = _requestLookAtPosition - _owningAvatar->getWorldPosition(); + glm::vec3 newAvatarLookAtVector = lookAtPosition - _owningAvatar->getWorldPosition(); + const float MIN_BLINK_ANGLE = 0.35f; // 20 degrees + _forceBlinkToRetarget = angleBetween(oldAvatarLookAtVector, newAvatarLookAtVector) > MIN_BLINK_ANGLE; + if (_forceBlinkToRetarget) { + _isEyeLookAtUpdated = false; + } else { + _lookAtPosition = lookAtPosition; + } + } + _requestLookAtPosition = lookAtPosition; +} + +void Head::updateEyeLookAt() { + _lookAtPosition = _requestLookAtPosition; + _isEyeLookAtUpdated = true; +} diff --git a/libraries/avatars-renderer/src/avatars-renderer/Head.h b/libraries/avatars-renderer/src/avatars-renderer/Head.h index e3c8d7d2b5..98b2fbb3a0 100644 --- a/libraries/avatars-renderer/src/avatars-renderer/Head.h +++ b/libraries/avatars-renderer/src/avatars-renderer/Head.h @@ -79,6 +79,9 @@ public: float getTimeWithoutTalking() const { return _timeWithoutTalking; } + virtual void setLookAtPosition(const glm::vec3& lookAtPosition) override; + void updateEyeLookAt(); + protected: // disallow copies of the Head, copy of owning Avatar is disallowed too Head(const Head&); @@ -123,6 +126,10 @@ protected: int _leftEyeLookAtID; int _rightEyeLookAtID; + glm::vec3 _requestLookAtPosition; + bool _forceBlinkToRetarget { false }; + bool _isEyeLookAtUpdated { false }; + // private methods void calculateMouthShapes(float timeRatio); void applyEyelidOffset(glm::quat headOrientation); diff --git a/libraries/avatars/src/AvatarData.h b/libraries/avatars/src/AvatarData.h index 61729f8db3..bfe775e972 100755 --- a/libraries/avatars/src/AvatarData.h +++ b/libraries/avatars/src/AvatarData.h @@ -1480,7 +1480,6 @@ public: void setIsNewAvatar(bool isNewAvatar) { _isNewAvatar = isNewAvatar; } bool getIsNewAvatar() { return _isNewAvatar; } void setIsClientAvatar(bool isClientAvatar) { _isClientAvatar = isClientAvatar; } - bool getIsClientAvatar() const { return _isClientAvatar; } void setSkeletonData(const std::vector& skeletonData); std::vector getSkeletonData() const; void sendSkeletonData() const; diff --git a/libraries/avatars/src/HeadData.cpp b/libraries/avatars/src/HeadData.cpp index d9998883d7..7e1d13511e 100644 --- a/libraries/avatars/src/HeadData.cpp +++ b/libraries/avatars/src/HeadData.cpp @@ -234,30 +234,3 @@ void HeadData::setFaceTrackerConnected(bool value) { _isFaceTrackerConnected = value; } -void HeadData::setLookAtPosition(const glm::vec3& lookAtPosition) { - if (_owningAvatar->getIsClientAvatar() || _owningAvatar->isMyAvatar()) { - if (_isEyeLookAtUpdated && _requestLookAtPosition != lookAtPosition) { - _lookAtPositionChanged = usecTimestampNow(); - glm::vec3 oldAvatarLookAtVector = _requestLookAtPosition - _owningAvatar->getWorldPosition(); - glm::vec3 newAvatarLookAtVector = lookAtPosition - _owningAvatar->getWorldPosition(); - const float MIN_BLINK_ANGLE = 0.35f; // 20 degrees - _forceBlinkToRetarget = angleBetween(oldAvatarLookAtVector, newAvatarLookAtVector) > MIN_BLINK_ANGLE; - if (_forceBlinkToRetarget) { - _isEyeLookAtUpdated = false; - } else { - _lookAtPosition = lookAtPosition; - } - } - _requestLookAtPosition = lookAtPosition; - } else { - if (_lookAtPosition != lookAtPosition) { - _lookAtPositionChanged = usecTimestampNow(); - } - _lookAtPosition = lookAtPosition; - } -} - -void HeadData::updateEyeLookAt() { - _lookAtPosition = _requestLookAtPosition; - _isEyeLookAtUpdated = true; -} diff --git a/libraries/avatars/src/HeadData.h b/libraries/avatars/src/HeadData.h index 190b179f25..020827c4d9 100644 --- a/libraries/avatars/src/HeadData.h +++ b/libraries/avatars/src/HeadData.h @@ -64,7 +64,12 @@ public: void setBlendshapeCoefficients(const QVector& blendshapeCoefficients) { _blendshapeCoefficients = blendshapeCoefficients; } const glm::vec3& getLookAtPosition() const { return _lookAtPosition; } - void setLookAtPosition(const glm::vec3& lookAtPosition); + virtual void setLookAtPosition(const glm::vec3& lookAtPosition) { + if (_lookAtPosition != lookAtPosition) { + _lookAtPositionChanged = usecTimestampNow(); + } + _lookAtPosition = lookAtPosition; + } bool lookAtPositionChangedSince(quint64 time) { return _lookAtPositionChanged >= time; } bool getHasProceduralEyeFaceMovement() const; @@ -79,8 +84,6 @@ public: void setFaceTrackerConnected(bool value); bool getFaceTrackerConnected() const { return _isFaceTrackerConnected; } - void updateEyeLookAt(); - friend class AvatarData; QJsonObject toJson() const; @@ -92,7 +95,6 @@ protected: float _basePitch; float _baseRoll; - glm::vec3 _requestLookAtPosition; glm::vec3 _lookAtPosition; quint64 _lookAtPositionChanged { 0 }; @@ -113,8 +115,6 @@ protected: QVector _summedBlendshapeCoefficients; QMap _blendshapeLookupMap; AvatarData* _owningAvatar; - bool _forceBlinkToRetarget { false }; - bool _isEyeLookAtUpdated { false }; private: // privatize copy ctor and assignment operator so copies of this object cannot be made From 9432a375675e6aa5b87ca26e87d22e80c0d837b3 Mon Sep 17 00:00:00 2001 From: luiscuenca Date: Wed, 23 Oct 2019 14:26:53 -0700 Subject: [PATCH 6/6] Undo changes in HeadData.cpp --- libraries/avatars/src/HeadData.cpp | 1 - 1 file changed, 1 deletion(-) diff --git a/libraries/avatars/src/HeadData.cpp b/libraries/avatars/src/HeadData.cpp index 7e1d13511e..c86e534929 100644 --- a/libraries/avatars/src/HeadData.cpp +++ b/libraries/avatars/src/HeadData.cpp @@ -233,4 +233,3 @@ void HeadData::setHasProceduralEyeMovement(bool hasProceduralEyeMovement) { void HeadData::setFaceTrackerConnected(bool value) { _isFaceTrackerConnected = value; } -